diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index 2fed5c32f5..ce17bcedec 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -59,6 +59,7 @@ "Activity recognition", "Tumor detection", "Duplicate Detection", + "Rendered semantic textual similarity", ] TASK_DOMAIN = Literal[ @@ -105,13 +106,16 @@ MIEB_TASK_TYPE = ( "Any2AnyMultiChoice", "Any2AnyRetrieval", - "Any2TextMutipleChoice", + "Any2AnyMultilingualRetrieval", + "VisionCentric", "ImageClustering", "ImageClassification", "ImageMultilabelClassification", - "ImageTextPairClassification", - "VisualSTS", + "DocumentUnderstanding", + "VisualSTS(eng)", + "VisualSTS(multi)", "ZeroShotClassification", + "Compositionality", ) TASK_TYPE = ( diff --git a/mteb/abstasks/aggregate_task_metadata.py b/mteb/abstasks/aggregate_task_metadata.py index 106419b752..445f07ff6d 100644 --- a/mteb/abstasks/aggregate_task_metadata.py +++ b/mteb/abstasks/aggregate_task_metadata.py @@ -60,6 +60,12 @@ class AggregateTaskMetadata(TaskMetadata): @property def hf_subsets_to_langscripts(self) -> dict[HFSubset, list[ISO_LANGUAGE_SCRIPT]]: """Return a dictionary mapping huggingface subsets to languages.""" + if isinstance(self.eval_langs, dict): + langs = [] + for v in self.eval_langs.values(): + langs.extend(v) + langs = list(set(langs)) + return {"default": langs} return {"default": self.eval_langs} # type: ignore @model_validator(mode="after") # type: ignore diff --git a/mteb/abstasks/aggregated_task.py b/mteb/abstasks/aggregated_task.py index 255df2000f..4c79db01ae 100644 --- a/mteb/abstasks/aggregated_task.py +++ b/mteb/abstasks/aggregated_task.py @@ -35,15 +35,27 @@ def task_results_to_scores( ) -> dict[str, dict[HFSubset, ScoresDict]]: """The function that aggregated scores. Can be redefined to allow for custom aggregations.""" scores = {} + subsets = ( + self.metadata.eval_langs.keys() + if isinstance(self.metadata.eval_langs, dict) + else None + ) + eval_langs = ( + self.metadata.eval_langs.values() + if isinstance(self.metadata.eval_langs, dict) + else [self.metadata.eval_langs] + ) for split in self.metadata.eval_splits: main_scores = [] for task_res in task_results: - main_scores.append( - task_res.get_score_fast( - languages=None, - splits=self.metadata.eval_splits, + for langs in eval_langs: + main_scores.append( + task_res.get_score_fast( + languages=[lang.split("-")[0] for lang in langs], + splits=self.metadata.eval_splits, + subsets=subsets, + ) ) - ) main_score = np.mean(main_scores) scores[split] = { "default": { @@ -64,7 +76,7 @@ def combine_task_results(self, task_results: list[TaskResult]) -> TaskResult: eval_times = [tr.evaluation_time for tr in task_results if tr.evaluation_time] if len(eval_times) != len(task_results): logger.info( - f"Loaded results does not include runtime. Therefor evaluation of {self.metadata.name} " + f"Loaded results does not include runtime. Therefore evaluation of {self.metadata.name} " + "can't be computed. Setting it to None." ) eval_time = np.nan @@ -76,7 +88,7 @@ def combine_task_results(self, task_results: list[TaskResult]) -> TaskResult: ] if len(kg_co2_emissions_) != len(task_results): logger.info( - f"Loaded results does not include co2-eq emissions. Therefor evaluation of {self.metadata.name} " + f"Loaded results does not include co2-eq emissions. Therefore evaluation of {self.metadata.name} " + "can't be computed. Setting it to None." ) kg_co2_emissions = np.nan diff --git a/mteb/benchmarks/benchmarks.py b/mteb/benchmarks/benchmarks.py index fcf5ad8bd6..c6edfa9a28 100644 --- a/mteb/benchmarks/benchmarks.py +++ b/mteb/benchmarks/benchmarks.py @@ -1402,6 +1402,254 @@ }""", ) +MIEB_common_tasks = [ + # Image Classification + "Birdsnap", # fine + "Caltech101", # fine + "CIFAR10", # coarse + "CIFAR100", # fine + "Country211", # fine + "DTD", # coarse + "EuroSAT", # coarse + "FER2013", # coarse + "FGVCAircraft", # fine + "Food101Classification", # fine + "GTSRB", # coarse + "Imagenet1k", # fine + "MNIST", # coarse + "OxfordFlowersClassification", # fine + "OxfordPets", # fine + "PatchCamelyon", # coarse + "RESISC45", # fine + "StanfordCars", # fine + "STL10", # coarse + "SUN397", # fine + "UCF101", # fine + # ImageMultiLabelClassification + "VOC2007", # coarse + # Clustering + "CIFAR10Clustering", + "CIFAR100Clustering", + "ImageNetDog15Clustering", + "ImageNet10Clustering", + "TinyImageNetClustering", + # ZeroShotClassification + "BirdsnapZeroShot", + "Caltech101ZeroShot", + "CIFAR10ZeroShot", + "CIFAR100ZeroShot", + "CLEVRZeroShot", + "CLEVRCountZeroShot", + "Country211ZeroShot", + "DTDZeroShot", + "EuroSATZeroShot", + "FER2013ZeroShot", + "FGVCAircraftZeroShot", + "Food101ZeroShot", + "GTSRBZeroShot", + "Imagenet1kZeroShot", + "MNISTZeroShot", + "OxfordPetsZeroShot", + "PatchCamelyonZeroShot", + "RenderedSST2", + "RESISC45ZeroShot", + "StanfordCarsZeroShot", + "STL10ZeroShot", + "SUN397ZeroShot", + "UCF101ZeroShot", + # Any2TextMutipleChoice + "CVBenchCount", + "CVBenchRelation", + "CVBenchDepth", + "CVBenchDistance", + # Any2AnyMultipleChoice + "BLINKIT2IMultiChoice", + "BLINKIT2TMultiChoice", + # Compositionality + "ImageCoDeT2IMultiChoice", + "AROCocoOrder", + "AROFlickrOrder", + "AROVisualAttribution", + "AROVisualRelation", + "SugarCrepe", + "Winoground", + # VisualSTS + "STS12VisualSTS", + "STS13VisualSTS", + "STS14VisualSTS", + "STS15VisualSTS", + "STS16VisualSTS", + # Any2AnyRetrieval + "BLINKIT2IRetrieval", + "BLINKIT2TRetrieval", + "CIRRIT2IRetrieval", + "CUB200I2IRetrieval", + "EDIST2ITRetrieval", + "Fashion200kI2TRetrieval", + "Fashion200kT2IRetrieval", + "FashionIQIT2IRetrieval", + "Flickr30kI2TRetrieval", + "Flickr30kT2IRetrieval", + "FORBI2IRetrieval", + "GLDv2I2IRetrieval", + "GLDv2I2TRetrieval", + "HatefulMemesI2TRetrieval", + "HatefulMemesT2IRetrieval", + "ImageCoDeT2IRetrieval", + "InfoSeekIT2ITRetrieval", + "InfoSeekIT2TRetrieval", + "MemotionI2TRetrieval", + "MemotionT2IRetrieval", + "METI2IRetrieval", + "MSCOCOI2TRetrieval", + "MSCOCOT2IRetrieval", + "NIGHTSI2IRetrieval", + "OVENIT2ITRetrieval", + "OVENIT2TRetrieval", + "ROxfordEasyI2IMultiChoice", + "ROxfordMediumI2IMultiChoice", + "ROxfordHardI2IMultiChoice", + "RP2kI2IRetrieval", + "RParisEasyI2IMultiChoice", + "RParisMediumI2IMultiChoice", + "RParisHardI2IMultiChoice", + "SciMMIRI2TRetrieval", + "SciMMIRT2IRetrieval", + "SketchyI2IRetrieval", + "SOPI2IRetrieval", + "StanfordCarsI2IRetrieval", + "TUBerlinT2IRetrieval", + "VidoreArxivQARetrieval", + "VidoreDocVQARetrieval", + "VidoreInfoVQARetrieval", + "VidoreTabfquadRetrieval", + "VidoreTatdqaRetrieval", + "VidoreShiftProjectRetrieval", + "VidoreSyntheticDocQAAIRetrieval", + "VidoreSyntheticDocQAEnergyRetrieval", + "VidoreSyntheticDocQAGovernmentReportsRetrieval", + "VidoreSyntheticDocQAHealthcareIndustryRetrieval", + "VisualNewsI2TRetrieval", + "VisualNewsT2IRetrieval", + "VizWizIT2TRetrieval", + "VQA2IT2TRetrieval", + "WebQAT2ITRetrieval", + "WebQAT2TRetrieval", +] + +MIEB_ENG = Benchmark( + name="MIEB(eng)", + tasks=get_tasks( + tasks=MIEB_common_tasks + + [ + "VisualSTS17Eng", + "VisualSTS-b-Eng", + ], + ), + description="""MIEB(eng) is a comprehensive image embeddings benchmark, spanning 8 task types, covering 125 tasks. + In addition to image classification (zero shot and linear probing), clustering, retrieval, MIEB includes tasks in compositionality evaluation, + document undestanding, visual STS, and CV-centric tasks.""", + reference="", + contacts=["gowitheflow-1998", "isaac-chung"], + citation="", +) + +MIEB_MULTILINGUAL = Benchmark( + name="MIEB(Multilingual)", + tasks=get_tasks( + tasks=MIEB_common_tasks + + [ + "WITT2IRetrieval", + "XFlickr30kCoT2IRetrieval", + "XM3600T2IRetrieval", + "VisualSTS17Eng", + "VisualSTS-b-Eng", + "VisualSTS17Multilingual", + "VisualSTS-b-Multilingual", + ], + ), + description="""MIEB(Multilingual) is a comprehensive image embeddings benchmark, spanning 10 task types, covering 130 tasks and a total of 39 languages. + In addition to image classification (zero shot and linear probing), clustering, retrieval, MIEB includes tasks in compositionality evaluation, + document undestanding, visual STS, and CV-centric tasks. This benchmark consists of MIEB(eng) + 3 multilingual retrieval + datasets + the multilingual parts of VisualSTS-b and VisualSTS-16.""", + reference="", + contacts=["gowitheflow-1998", "isaac-chung"], + citation="", +) + +MIEB_LITE = Benchmark( + name="MIEB(lite)", + tasks=get_tasks( + tasks=[ + # Image Classification + "Country211", + "DTD", + "EuroSAT", + "GTSRB", + "OxfordPets", + "PatchCamelyon", + "RESISC45", + "SUN397", + # Clustering + "ImageNetDog15Clustering", + "TinyImageNetClustering", + # ZeroShotClassification + "CIFAR100ZeroShot", + "Country211ZeroShot", + "FER2013ZeroShot", + "FGVCAircraftZeroShot", + "Food101ZeroShot", + "OxfordPetsZeroShot", + "StanfordCarsZeroShot", + # Any2TextMutipleChoice + "CVBenchCount", + "CVBenchRelation", + "CVBenchDepth", + "CVBenchDistance", + # Any2AnyMultipleChoice + "BLINKIT2IMultiChoice", + "ImageCoDeT2IMultiChoice", + # ImageTextPairClassification + "AROCocoOrder", + "AROFlickrOrder", + "AROVisualAttribution", + "AROVisualRelation", + "Winoground", + # VisualSTS + "STS13VisualSTS", + "STS15VisualSTS", + "STS17MultilingualVisualSTS", + "STSBenchmarkMultilingualVisualSTS", + # Any2AnyRetrieval + "CIRRIT2IRetrieval", + "CUB200I2IRetrieval", + "Fashion200kI2TRetrieval", + "HatefulMemesI2TRetrieval", + "InfoSeekIT2TRetrieval", + "NIGHTSI2IRetrieval", + "OVENIT2TRetrieval", + "RP2kI2IRetrieval", + "VidoreDocVQARetrieval", + "VidoreInfoVQARetrieval", + "VidoreTabfquadRetrieval", + "VidoreTatdqaRetrieval", + "VidoreShiftProjectRetrieval", + "VidoreSyntheticDocQAAIRetrieval", + "VisualNewsI2TRetrieval", + "VQA2IT2TRetrieval", + "WebQAT2ITRetrieval", + "WITT2IRetrieval", + "XM3600T2IRetrieval", + ], + ), + description="""MIEB(lite) is a comprehensive image embeddings benchmark, spanning 10 task types, covering 51 tasks. + This is a lite version of MIEB(Multilingual), designed to be run at a fraction of the cost while maintaining + relative rank of models.""", + reference="", + contacts=["gowitheflow-1998", "isaac-chung"], + citation="", +) + BUILT_MTEB = Benchmark( name="BuiltBench(eng)", tasks=get_tasks( diff --git a/mteb/leaderboard/app.py b/mteb/leaderboard/app.py index 3966ffba03..c7a6d687fd 100644 --- a/mteb/leaderboard/app.py +++ b/mteb/leaderboard/app.py @@ -6,7 +6,7 @@ import tempfile import time from pathlib import Path -from typing import Literal +from typing import Literal, get_args from urllib.parse import urlencode import cachetools @@ -15,7 +15,9 @@ from gradio_rangeslider import RangeSlider import mteb +from mteb.abstasks.TaskMetadata import TASK_DOMAIN, TASK_TYPE from mteb.benchmarks.benchmarks import MTEB_multilingual +from mteb.languages import ISO_TO_LANGUAGE from mteb.leaderboard.figures import performance_size_plot, radar_chart from mteb.leaderboard.table import scores_to_tables @@ -43,20 +45,6 @@ We also thank the following companies which provide API credits to evaluate their models: [OpenAI](https://openai.com/), [Voyage AI](https://www.voyageai.com/) """ -MMTEB_TASK_TYPES = [ # TEMPORARY FIX: when adding MIEB to the leaderboard, this can probably be replaced with TASK_TYPE - "BitextMining", - "Classification", - "MultilabelClassification", - "Clustering", - "PairClassification", - "Reranking", - "Retrieval", - "STS", - "Summarization", - "InstructionRetrieval", - "Speed", -] - ALL_MODELS = {meta.name for meta in mteb.get_model_metas()} @@ -238,21 +226,21 @@ def filter_models( info="Select one of our expert-selected benchmarks from MTEB publications.", ) lang_select = gr.Dropdown( - all_results.languages, + ISO_TO_LANGUAGE, value=sorted(default_results.languages), multiselect=True, label="Language", info="Select languages to include.", ) type_select = gr.Dropdown( - all_results.task_types, - value=sorted(MMTEB_TASK_TYPES), + sorted(get_args(TASK_TYPE)), + value=sorted(default_results.task_types), multiselect=True, label="Task Type", info="Select task types to include.", ) domain_select = gr.Dropdown( - all_results.domains, + sorted(get_args(TASK_DOMAIN)), value=sorted(default_results.domains), multiselect=True, label="Domain", diff --git a/mteb/leaderboard/figures.py b/mteb/leaderboard/figures.py index a883b043b3..30beed066d 100644 --- a/mteb/leaderboard/figures.py +++ b/mteb/leaderboard/figures.py @@ -1,10 +1,14 @@ from __future__ import annotations +from typing import get_args + import numpy as np import pandas as pd import plotly.express as px import plotly.graph_objects as go +from mteb.abstasks.TaskMetadata import TASK_TYPE + def text_plot(text: str): """Returns empty scatter plot with text added, this can be great for error messages.""" @@ -56,6 +60,10 @@ def parse_float(value) -> float: "GritLM-7B", "LaBSE", "multilingual-e5-large-instruct", + "EVA02-CLIP-bigE-14-plus", + "voyage-multimodal-3", + "e5-v", + "VLM2Vec-Full", ] @@ -165,21 +173,10 @@ def performance_size_plot(df: pd.DataFrame) -> go.Figure: TOP_N = 5 -task_types = [ - "BitextMining", - "Classification", - "MultilabelClassification", - "Clustering", - "PairClassification", - "Reranking", - "Retrieval", - "STS", - "Summarization", - # "InstructionRetrieval", - # Not displayed, because the scores are negative, - # doesn't work well with the radar chart. - "Speed", -] +task_types = sorted(get_args(TASK_TYPE)) +task_types.remove("InstructionRetrieval") +# Not displayed, because the scores are negative, +# doesn't work well with the radar chart. line_colors = [ "#EE4266", diff --git a/mteb/load_results/task_results.py b/mteb/load_results/task_results.py index 2eabeeab02..989d191d1e 100644 --- a/mteb/load_results/task_results.py +++ b/mteb/load_results/task_results.py @@ -458,7 +458,10 @@ def get_score( return aggregation(values) def get_score_fast( - self, splits: Iterable[str] | None = None, languages: str | None = None + self, + splits: Iterable[str] | None = None, + languages: str | None = None, + subsets: Iterable[str] | None = None, ) -> float: """Sped up version of get_score that will be used if no aggregation, script or getter needs to be specified.""" if splits is None: @@ -475,6 +478,13 @@ def get_score_fast( main_score = scores.get("main_score", None) if main_score is None: raise ValueError(f"Missing main score for subset: {hf_subset}") + if subsets and hf_subset not in subsets: + continue + elif subsets: + val_sum += main_score + n_val += 1 + continue + if languages is None: val_sum += main_score n_val += 1 @@ -483,6 +493,7 @@ def get_score_fast( if lang.split("-")[0] in languages: val_sum += main_score n_val += 1 + logger.info(f"{val_sum=}, {n_val=}") break if n_val == 0: raise ValueError("No splits had scores for the specified languages.") diff --git a/mteb/models/voyage_v.py b/mteb/models/voyage_v.py index d607d809f0..1086f88ee5 100644 --- a/mteb/models/voyage_v.py +++ b/mteb/models/voyage_v.py @@ -248,16 +248,16 @@ def get_fused_embeddings( release_date="2024-11-10", n_parameters=None, memory_usage_mb=None, - max_tokens=None, + max_tokens=32768, embed_dim=1024, - license=None, + license="mit", similarity_fn_name="cosine", - framework=[], + framework=["API"], modalities=["image", "text"], - open_weights=None, + open_weights=False, public_training_code=None, public_training_data=None, - reference=None, + reference="https://huggingface.co/voyageai/voyage-multimodal-3", use_instructions=None, training_datasets=None, ) diff --git a/mteb/overview.py b/mteb/overview.py index caf3a41467..f3333c1fc7 100644 --- a/mteb/overview.py +++ b/mteb/overview.py @@ -8,7 +8,7 @@ import pandas as pd -from mteb.abstasks import AbsTask +from mteb.abstasks.AbsTask import AbsTask from mteb.abstasks.TaskMetadata import TASK_CATEGORY, TASK_DOMAIN, TASK_TYPE from mteb.languages import ( ISO_TO_LANGUAGE, diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2IMultiChoice.py b/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2IMultiChoice.py index 58db0c8c92..f15d1df8a8 100644 --- a/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2IMultiChoice.py +++ b/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2IMultiChoice.py @@ -14,7 +14,7 @@ class BLINKIT2IMultiChoice(AbsTaskAny2AnyMultiChoice): "revision": "a9f994925551c14503d00d86f1307bac6e2ead6a", "trust_remote_code": True, }, - type="Any2AnyMultiChoice", + type="VisionCentric", category="it2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2TMultiChoice.py b/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2TMultiChoice.py index 0a1dfcdc42..b4e6c08472 100644 --- a/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2TMultiChoice.py +++ b/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2TMultiChoice.py @@ -13,7 +13,7 @@ class BLINKIT2TMultiChoice(AbsTaskAny2AnyMultiChoice): "path": "JamieSJS/blink-it2t-multi", "revision": "bc8f4c7f62450a4ceb737c8339061cf87aea42d5", }, - type="Any2AnyMultiChoice", + type="VisionCentric", category="it2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/eng/ImageCoDeT2IMultiChoice.py b/mteb/tasks/Image/Any2AnyMultiChoice/eng/ImageCoDeT2IMultiChoice.py index 3cb875b845..f9400ff280 100644 --- a/mteb/tasks/Image/Any2AnyMultiChoice/eng/ImageCoDeT2IMultiChoice.py +++ b/mteb/tasks/Image/Any2AnyMultiChoice/eng/ImageCoDeT2IMultiChoice.py @@ -13,7 +13,7 @@ class ImageCoDeT2IMultiChoice(AbsTaskAny2AnyMultiChoice): "path": "JamieSJS/imagecode-multi", "revision": "d28adfd8b34fefa546fdf94bdc352622b2575f6c", }, - type="Any2AnyMultiChoice", + type="Compositionality", category="it2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/eng/ROxfordI2IMultiChoice.py b/mteb/tasks/Image/Any2AnyMultiChoice/eng/ROxfordI2IMultiChoice.py index 136848c128..d5661e2840 100644 --- a/mteb/tasks/Image/Any2AnyMultiChoice/eng/ROxfordI2IMultiChoice.py +++ b/mteb/tasks/Image/Any2AnyMultiChoice/eng/ROxfordI2IMultiChoice.py @@ -4,6 +4,8 @@ from mteb.abstasks.TaskMetadata import TaskMetadata +# NOTE: These tasks are marked as Any2AnyRetrieval types they are the correct implementations of ROxford retrieval and RParis retrieval +# (as it requires masking out the different docs in corpus for every query). This aligns with the MIEB papeer. class ROxfordEasyI2IMultiChoice(AbsTaskAny2AnyMultiChoice): metadata = TaskMetadata( name="ROxfordEasyI2IMultiChoice", @@ -13,7 +15,7 @@ class ROxfordEasyI2IMultiChoice(AbsTaskAny2AnyMultiChoice): "path": "JamieSJS/r-oxford-easy-multi", "revision": "4c167c3ce529f19457c9b8e694258cc6cf8e7cc7", }, - type="Any2AnyMultiChoice", + type="Any2AnyRetrieval", category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -59,7 +61,7 @@ class ROxfordMediumI2IMultiChoice(AbsTaskAny2AnyMultiChoice): "path": "JamieSJS/r-oxford-medium-multi", "revision": "83bd440268e200a4f60313070618e3f45000fa94", }, - type="Any2AnyMultiChoice", + type="Any2AnyRetrieval", category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -105,7 +107,7 @@ class ROxfordHardI2IMultiChoice(AbsTaskAny2AnyMultiChoice): "path": "JamieSJS/r-oxford-hard-multi", "revision": "fc7c4ae6655b1e6b132f3b262a359acef42dfce8", }, - type="Any2AnyMultiChoice", + type="Any2AnyRetrieval", category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/eng/RParisI2IMultiChoice.py b/mteb/tasks/Image/Any2AnyMultiChoice/eng/RParisI2IMultiChoice.py index 69da75118f..754111b594 100644 --- a/mteb/tasks/Image/Any2AnyMultiChoice/eng/RParisI2IMultiChoice.py +++ b/mteb/tasks/Image/Any2AnyMultiChoice/eng/RParisI2IMultiChoice.py @@ -4,6 +4,8 @@ from mteb.abstasks.TaskMetadata import TaskMetadata +# NOTE: These tasks are marked as Any2AnyRetrieval types they are the correct implementations of ROxford retrieval and RParis retrieval +# (as it requires masking out the different docs in corpus for every query). This aligns with the MIEB papeer. class RParisEasyI2IMultiChoice(AbsTaskAny2AnyMultiChoice): metadata = TaskMetadata( name="RParisEasyI2IMultiChoice", @@ -13,7 +15,7 @@ class RParisEasyI2IMultiChoice(AbsTaskAny2AnyMultiChoice): "path": "JamieSJS/r-paris-easy-multi", "revision": "db94b5afd0014ab8c978f20a0fbcc52da1612a08", }, - type="Any2AnyMultiChoice", + type="Any2AnyRetrieval", category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -59,7 +61,7 @@ class RParisMediumI2IMultiChoice(AbsTaskAny2AnyMultiChoice): "path": "JamieSJS/r-paris-medium-multi", "revision": "372c79fc823e1cebc1d55f8e0039aa239285e177", }, - type="Any2AnyMultiChoice", + type="Any2AnyRetrieval", category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -105,7 +107,7 @@ class RParisHardI2IMultiChoice(AbsTaskAny2AnyMultiChoice): "path": "JamieSJS/r-paris-hard-multi", "revision": "4e5997e48fb2f2f8bf1c8973851dedeb17e09a83", }, - type="Any2AnyMultiChoice", + type="Any2AnyRetrieval", category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py index 44d0d36cb0..f279d9b277 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/VidoreBenchRetrieval.py @@ -79,7 +79,7 @@ class VidoreArxivQARetrieval(AbsTaskAny2AnyRetrieval): "path": "vidore/arxivqa_test_subsampled_beir", "revision": "7d94d570960eac2408d3baa7a33f9de4822ae3e4", }, - type="Any2AnyRetrieval", + type="DocumentUnderstanding", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -133,7 +133,7 @@ class VidoreDocVQARetrieval(AbsTaskAny2AnyRetrieval): "path": "vidore/docvqa_test_subsampled_beir", "revision": "162ba2fc1a8437eda8b6c37b240bc1c0f0deb092", }, - type="Any2AnyRetrieval", + type="DocumentUnderstanding", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -187,7 +187,7 @@ class VidoreInfoVQARetrieval(AbsTaskAny2AnyRetrieval): "path": "vidore/infovqa_test_subsampled_beir", "revision": "b802cc5fd6c605df2d673a963667d74881d2c9a4", }, - type="Any2AnyRetrieval", + type="DocumentUnderstanding", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -241,7 +241,7 @@ class VidoreTabfquadRetrieval(AbsTaskAny2AnyRetrieval): "path": "vidore/tabfquad_test_subsampled_beir", "revision": "61a2224bcd29b7b261a4892ff4c8bea353527a31", }, - type="Any2AnyRetrieval", + type="DocumentUnderstanding", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -295,7 +295,7 @@ class VidoreTatdqaRetrieval(AbsTaskAny2AnyRetrieval): "path": "vidore/tatdqa_test_beir", "revision": "5feb5630fdff4d8d189ffedb2dba56862fdd45c0", }, - type="Any2AnyRetrieval", + type="DocumentUnderstanding", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -349,7 +349,7 @@ class VidoreShiftProjectRetrieval(AbsTaskAny2AnyRetrieval): "path": "vidore/shiftproject_test_beir", "revision": "84a382e05c4473fed9cff2bbae95fe2379416117", }, - type="Any2AnyRetrieval", + type="DocumentUnderstanding", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -403,7 +403,7 @@ class VidoreSyntheticDocQAAIRetrieval(AbsTaskAny2AnyRetrieval): "path": "vidore/syntheticDocQA_artificial_intelligence_test_beir", "revision": "2d9ebea5a1c6e9ef4a3b902a612f605dca11261c", }, - type="Any2AnyRetrieval", + type="DocumentUnderstanding", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -457,7 +457,7 @@ class VidoreSyntheticDocQAEnergyRetrieval(AbsTaskAny2AnyRetrieval): "path": "vidore/syntheticDocQA_energy_test_beir", "revision": "9935aadbad5c8deec30910489db1b2c7133ae7a7", }, - type="Any2AnyRetrieval", + type="DocumentUnderstanding", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -511,7 +511,7 @@ class VidoreSyntheticDocQAGovernmentReportsRetrieval(AbsTaskAny2AnyRetrieval): "path": "vidore/syntheticDocQA_government_reports_test_beir", "revision": "b4909afa930f81282fd20601e860668073ad02aa", }, - type="Any2AnyRetrieval", + type="DocumentUnderstanding", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -565,7 +565,7 @@ class VidoreSyntheticDocQAHealthcareIndustryRetrieval(AbsTaskAny2AnyRetrieval): "path": "vidore/syntheticDocQA_healthcare_industry_test_beir", "revision": "f9e25d5b6e13e1ad9f5c3cce202565031b3ab164", }, - type="Any2AnyRetrieval", + type="DocumentUnderstanding", category="t2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py index 5f20e45d25..884729d8ab 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py @@ -101,7 +101,7 @@ class WITT2IRetrieval(MultilingualTask, AbsTaskAny2AnyRetrieval): "revision": "91ac153f1371a98b209ed763205e25e115ecd06e", # "trust_remote_code": True, }, - type="Any2AnyRetrieval", + type="Any2AnyMultilingualRetrieval", category="t2i", eval_splits=["test"], eval_langs=_LANGUAGES, diff --git a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py index 98b45006a2..4370c2752c 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py @@ -84,7 +84,7 @@ class XFlickr30kCoT2IRetrieval(MultilingualTask, AbsTaskAny2AnyRetrieval): "revision": "0af2c2eba58b27a71898787e286be04befdd7a20", # "trust_remote_code": True, }, - type="Any2AnyRetrieval", + type="Any2AnyMultilingualRetrieval", category="t2i", eval_splits=["test"], eval_langs=_LANGUAGES, diff --git a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py index a65d37f324..880c7aade8 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py @@ -129,7 +129,7 @@ class XM3600T2IRetrieval(MultilingualTask, AbsTaskAny2AnyRetrieval): "revision": "8d3e5665526c55a5855cd6ddfbaba2032bc7cee4", # "trust_remote_code": True, }, - type="Any2AnyRetrieval", + type="Any2AnyMultilingualRetrieval", category="t2i", eval_splits=["test"], eval_langs=_LANGUAGES, diff --git a/mteb/tasks/Image/Any2TextMultipleChoice/eng/CVBench.py b/mteb/tasks/Image/Any2TextMultipleChoice/eng/CVBench.py index 3f387fdcbf..cbc7450188 100644 --- a/mteb/tasks/Image/Any2TextMultipleChoice/eng/CVBench.py +++ b/mteb/tasks/Image/Any2TextMultipleChoice/eng/CVBench.py @@ -23,7 +23,7 @@ class CVBenchCount(AbsTaskAny2TextMultipleChoice): "path": "nyu-visionx/CV-Bench", "revision": "22409a927ab5cf68e3655023d51694587455fc99", }, - type="Any2TextMutipleChoice", + type="VisionCentric", category="it2t", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -83,7 +83,7 @@ class CVBenchRelation(AbsTaskAny2TextMultipleChoice): "path": "nyu-visionx/CV-Bench", "revision": "22409a927ab5cf68e3655023d51694587455fc99", }, - type="Any2TextMutipleChoice", + type="VisionCentric", category="it2t", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -145,7 +145,7 @@ class CVBenchDepth(AbsTaskAny2TextMultipleChoice): "path": "nyu-visionx/CV-Bench", "revision": "22409a927ab5cf68e3655023d51694587455fc99", }, - type="Any2TextMutipleChoice", + type="VisionCentric", category="it2t", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -205,7 +205,7 @@ class CVBenchDistance(AbsTaskAny2TextMultipleChoice): "path": "nyu-visionx/CV-Bench", "revision": "22409a927ab5cf68e3655023d51694587455fc99", }, - type="Any2TextMutipleChoice", + type="VisionCentric", category="it2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py index 7f607d6aac..baae7f38dd 100644 --- a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py @@ -29,7 +29,16 @@ class OxfordFlowersClassification(AbsTaskImageClassification): dialect=[], modalities=["image"], sample_creation="found", - bibtex_citation="""d""", + bibtex_citation="""@INPROCEEDINGS{4756141, + author={Nilsback, Maria-Elena and Zisserman, Andrew}, + booktitle={2008 Sixth Indian Conference on Computer Vision, Graphics & Image Processing}, + title={Automated Flower Classification over a Large Number of Classes}, + year={2008}, + volume={}, + number={}, + pages={722-729}, + keywords={Shape;Kernel;Distributed computing;Support vector machines;Support vector machine classification;object classification;segmentation}, + doi={10.1109/ICVGIP.2008.47}}""", descriptive_stats={ "n_samples": {"test": 400000}, "avg_character_length": {"test": 431.4}, diff --git a/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py b/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py index 28a2357d5c..a5e9c26867 100644 --- a/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py @@ -8,7 +8,7 @@ class OxfordPetsClassification(AbsTaskImageClassification): metadata = TaskMetadata( name="OxfordPets", description="Classifying animal images.", - reference="https://arxiv.org/abs/1306.5151", + reference="https://ieeexplore.ieee.org/abstract/document/6248092", dataset={ "path": "isaacchung/OxfordPets", "revision": "557b480fae8d69247be74d9503b378a09425096f", @@ -29,15 +29,16 @@ class OxfordPetsClassification(AbsTaskImageClassification): dialect=[], modalities=["image"], sample_creation="created", - bibtex_citation="""@misc{maji2013finegrainedvisualclassificationaircraft, - title={Fine-Grained Visual Classification of Aircraft}, - author={Subhransu Maji and Esa Rahtu and Juho Kannala and Matthew Blaschko and Andrea Vedaldi}, - year={2013}, - eprint={1306.5151}, - archivePrefix={arXiv}, - primaryClass={cs.CV}, - url={https://arxiv.org/abs/1306.5151}, - } + bibtex_citation="""@INPROCEEDINGS{6248092, + author={Parkhi, Omkar M and Vedaldi, Andrea and Zisserman, Andrew and Jawahar, C. V.}, + booktitle={2012 IEEE Conference on Computer Vision and Pattern Recognition}, + title={Cats and dogs}, + year={2012}, + volume={}, + number={}, + pages={3498-3505}, + keywords={Positron emission tomography;Image segmentation;Cats;Dogs;Layout;Deformable models;Head}, + doi={10.1109/CVPR.2012.6248092}} """, descriptive_stats={ "n_samples": {"test": 3669}, diff --git a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py index ce32f85f93..e3839505d6 100644 --- a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py +++ b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py @@ -6,6 +6,8 @@ from mteb.abstasks.TaskMetadata import TaskMetadata +# NOTE: In the paper, this is grouped with linear probe tasks. +# See https://github.com/embeddings-benchmark/mteb/pull/2035#issuecomment-2661626309. class VOC2007Classification(AbsTaskImageMultilabelClassification): metadata = TaskMetadata( name="VOC2007", @@ -17,7 +19,7 @@ class VOC2007Classification(AbsTaskImageMultilabelClassification): "revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", "trust_remote_code": True, }, - type="ImageMultilabelClassification", + type="ImageClassification", category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageTextPairClassification/AROCocoOrder.py b/mteb/tasks/Image/ImageTextPairClassification/AROCocoOrder.py index ac5f03127e..8a227494ae 100644 --- a/mteb/tasks/Image/ImageTextPairClassification/AROCocoOrder.py +++ b/mteb/tasks/Image/ImageTextPairClassification/AROCocoOrder.py @@ -20,12 +20,12 @@ class AROCocoOrder(AbsTaskImageTextPairClassification): name="AROCocoOrder", description="Compositionality Evaluation of images to their captions." + "Each capation has four hard negatives created by order permutations.", - reference="https://proceedings.neurips.cc/paper_files/paper/2023/hash/63461de0b4cb760fc498e85b18a7fe81-Abstract-Datasets_and_Benchmarks.html", + reference="https://openreview.net/forum?id=KRLUvxh8uaX", dataset={ "path": "gowitheflow/ARO-COCO-order", "revision": "853ec8757226585a38a80886c51fe0f3f268787c", }, - type="ImageTextPairClassification", + type="Compositionality", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -41,12 +41,11 @@ class AROCocoOrder(AbsTaskImageTextPairClassification): dialect=[], modalities=["text", "image"], sample_creation="created", - bibtex_citation="""@article{hsieh2024sugarcrepe, - title={Sugarcrepe: Fixing hackable benchmarks for vision-language compositionality}, - author={Hsieh, Cheng-Yu and Zhang, Jieyu and Ma, Zixian and Kembhavi, Aniruddha and Krishna, Ranjay}, - journal={Advances in neural information processing systems}, - volume={36}, - year={2024} + bibtex_citation="""@inproceedings{yuksekgonul2023and, + title={When and why vision-language models behave like bags-of-words, and what to do about it?}, + author={Yuksekgonul, Mert and Bianchi, Federico and Kalluri, Pratyusha and Jurafsky, Dan and Zou, James}, + booktitle={The Eleventh International Conference on Learning Representations}, + year={2023} }""", descriptive_stats={ "n_samples": {"test": 25010}, diff --git a/mteb/tasks/Image/ImageTextPairClassification/AROFlickrOrder.py b/mteb/tasks/Image/ImageTextPairClassification/AROFlickrOrder.py index 18faadaf23..78fc0b8c79 100644 --- a/mteb/tasks/Image/ImageTextPairClassification/AROFlickrOrder.py +++ b/mteb/tasks/Image/ImageTextPairClassification/AROFlickrOrder.py @@ -20,12 +20,12 @@ class AROFlickrOrder(AbsTaskImageTextPairClassification): name="AROFlickrOrder", description="Compositionality Evaluation of images to their captions." + "Each capation has four hard negatives created by order permutations.", - reference="https://proceedings.neurips.cc/paper_files/paper/2023/hash/63461de0b4cb760fc498e85b18a7fe81-Abstract-Datasets_and_Benchmarks.html", + reference="https://openreview.net/forum?id=KRLUvxh8uaX", dataset={ "path": "gowitheflow/ARO-Flickr-Order", "revision": "1f9485f69c87947812378a1aedf86410c86a0aa8", }, - type="ImageTextPairClassification", + type="Compositionality", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], @@ -41,12 +41,11 @@ class AROFlickrOrder(AbsTaskImageTextPairClassification): dialect=[], modalities=["text", "image"], sample_creation="created", - bibtex_citation="""@article{hsieh2024sugarcrepe, - title={Sugarcrepe: Fixing hackable benchmarks for vision-language compositionality}, - author={Hsieh, Cheng-Yu and Zhang, Jieyu and Ma, Zixian and Kembhavi, Aniruddha and Krishna, Ranjay}, - journal={Advances in neural information processing systems}, - volume={36}, - year={2024} + bibtex_citation="""@inproceedings{yuksekgonul2023and, + title={When and why vision-language models behave like bags-of-words, and what to do about it?}, + author={Yuksekgonul, Mert and Bianchi, Federico and Kalluri, Pratyusha and Jurafsky, Dan and Zou, James}, + booktitle={The Eleventh International Conference on Learning Representations}, + year={2023} }""", descriptive_stats={ "n_samples": {"test": 5000}, diff --git a/mteb/tasks/Image/ImageTextPairClassification/AROVisualAttribution.py b/mteb/tasks/Image/ImageTextPairClassification/AROVisualAttribution.py index 4f75db410b..b43ac87a00 100644 --- a/mteb/tasks/Image/ImageTextPairClassification/AROVisualAttribution.py +++ b/mteb/tasks/Image/ImageTextPairClassification/AROVisualAttribution.py @@ -18,7 +18,7 @@ class AROVisualAttribution(AbsTaskImageTextPairClassification): "path": "gowitheflow/ARO-Visual-Attribution", "revision": "18f7e01358d91df599d723f00e16a18640e19398", }, - type="ImageTextPairClassification", + type="Compositionality", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageTextPairClassification/AROVisualRelation.py b/mteb/tasks/Image/ImageTextPairClassification/AROVisualRelation.py index fef938271e..1d74de646c 100644 --- a/mteb/tasks/Image/ImageTextPairClassification/AROVisualRelation.py +++ b/mteb/tasks/Image/ImageTextPairClassification/AROVisualRelation.py @@ -18,7 +18,7 @@ class AROVisualRelation(AbsTaskImageTextPairClassification): "path": "gowitheflow/ARO-Visual-Relation", "revision": "3867ad4f46a1ac2e63be034d1fc77dd8c2ef7209", }, - type="ImageTextPairClassification", + type="Compositionality", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageTextPairClassification/SugarCrepe.py b/mteb/tasks/Image/ImageTextPairClassification/SugarCrepe.py index b410cbacd5..94114f100e 100644 --- a/mteb/tasks/Image/ImageTextPairClassification/SugarCrepe.py +++ b/mteb/tasks/Image/ImageTextPairClassification/SugarCrepe.py @@ -20,7 +20,7 @@ class SugarCrepe(AbsTaskImageTextPairClassification): "path": "yjkimstats/SUGARCREPE_fmt", "revision": "134abf9ade6a32f9fdae0e89022ff227a70b87e5", }, - type="ImageTextPairClassification", + type="Compositionality", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageTextPairClassification/Winoground.py b/mteb/tasks/Image/ImageTextPairClassification/Winoground.py index 6169182286..84f696cd86 100644 --- a/mteb/tasks/Image/ImageTextPairClassification/Winoground.py +++ b/mteb/tasks/Image/ImageTextPairClassification/Winoground.py @@ -18,7 +18,7 @@ class Winoground(AbsTaskImageTextPairClassification): "path": "facebook/winoground", "revision": "b400e173549071916ad1b3d449293bc8d8b4b763", }, - type="ImageTextPairClassification", + type="Compositionality", category="i2t", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/VisualSTS/en/STS12VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS12VisualSTS.py index 09d550547f..3a99e9fc47 100644 --- a/mteb/tasks/Image/VisualSTS/en/STS12VisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/en/STS12VisualSTS.py @@ -13,7 +13,7 @@ class STS12VisualSTS(AbsTaskVisualSTS): }, description="SemEval-2012 Task 6." + "then rendered into images.", reference="https://arxiv.org/abs/2402.08183/", - type="VisualSTS", + type="VisualSTS(eng)", category="i2i", modalities=["image"], eval_splits=["test"], diff --git a/mteb/tasks/Image/VisualSTS/en/STS13VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS13VisualSTS.py index 771e9e0ce8..b66678c851 100644 --- a/mteb/tasks/Image/VisualSTS/en/STS13VisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/en/STS13VisualSTS.py @@ -13,7 +13,7 @@ class STS13VisualSTS(AbsTaskVisualSTS): }, description="SemEval STS 2013 dataset." + "then rendered into images.", reference="https://arxiv.org/abs/2402.08183/", - type="VisualSTS", + type="VisualSTS(eng)", category="i2i", modalities=["image"], eval_splits=["test"], diff --git a/mteb/tasks/Image/VisualSTS/en/STS14VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS14VisualSTS.py index 299e54dca9..0820ed7823 100644 --- a/mteb/tasks/Image/VisualSTS/en/STS14VisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/en/STS14VisualSTS.py @@ -14,7 +14,7 @@ class STS14VisualSTS(AbsTaskVisualSTS): description="SemEval STS 2014 dataset. Currently only the English dataset." + "rendered into images.", reference="https://arxiv.org/abs/2402.08183/", - type="VisualSTS", + type="VisualSTS(eng)", category="i2i", modalities=["image"], eval_splits=["test"], diff --git a/mteb/tasks/Image/VisualSTS/en/STS15VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS15VisualSTS.py index 1756cdc55c..8a9b8c682b 100644 --- a/mteb/tasks/Image/VisualSTS/en/STS15VisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/en/STS15VisualSTS.py @@ -13,7 +13,7 @@ class STS15VisualSTS(AbsTaskVisualSTS): }, description="SemEval STS 2015 dataset" + "rendered into images.", reference="https://arxiv.org/abs/2402.08183/", - type="VisualSTS", + type="VisualSTS(eng)", category="i2i", modalities=["image"], eval_splits=["test"], diff --git a/mteb/tasks/Image/VisualSTS/en/STS16VisualSTS.py b/mteb/tasks/Image/VisualSTS/en/STS16VisualSTS.py index dba6e4af63..ea82fa5a8b 100644 --- a/mteb/tasks/Image/VisualSTS/en/STS16VisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/en/STS16VisualSTS.py @@ -13,7 +13,7 @@ class STS16VisualSTS(AbsTaskVisualSTS): }, description="SemEval STS 2016 dataset" + "rendered into images.", reference="https://arxiv.org/abs/2402.08183/", - type="VisualSTS", + type="VisualSTS(eng)", category="i2i", modalities=["image"], eval_splits=["test"], diff --git a/mteb/tasks/Image/VisualSTS/multilingual/STS17MultilingualVisualSTS.py b/mteb/tasks/Image/VisualSTS/multilingual/STS17MultilingualVisualSTS.py index 068fd33b9c..2bf15406f3 100644 --- a/mteb/tasks/Image/VisualSTS/multilingual/STS17MultilingualVisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/multilingual/STS17MultilingualVisualSTS.py @@ -33,7 +33,7 @@ class STS17MultilingualVisualSTS(AbsTaskVisualSTS, MultilingualTask): + "rendered into images." ), reference="https://arxiv.org/abs/2402.08183/", - type="VisualSTS", + type="VisualSTS(multi)", category="i2i", modalities=["image"], eval_splits=_SPLITS, @@ -41,7 +41,7 @@ class STS17MultilingualVisualSTS(AbsTaskVisualSTS, MultilingualTask): main_score="cosine_spearman", date=("2012-01-01", "2017-12-31"), domains=["News", "Social", "Web", "Spoken", "Written"], - task_subtypes=[], + task_subtypes=["Rendered semantic textual similarity"], license="not specified", annotations_creators="human-annotated", dialect=[], diff --git a/mteb/tasks/Image/VisualSTS/multilingual/STSBenchmarkMultilingualVisualSTS.py b/mteb/tasks/Image/VisualSTS/multilingual/STSBenchmarkMultilingualVisualSTS.py index ce8c047655..9cab4e2f45 100644 --- a/mteb/tasks/Image/VisualSTS/multilingual/STSBenchmarkMultilingualVisualSTS.py +++ b/mteb/tasks/Image/VisualSTS/multilingual/STSBenchmarkMultilingualVisualSTS.py @@ -34,7 +34,7 @@ class STSBenchmarkMultilingualVisualSTS(AbsTaskVisualSTS, MultilingualTask): + "built upon multi-sts created by Philip May" ), reference="https://arxiv.org/abs/2402.08183/", - type="VisualSTS", + type="VisualSTS(multi)", category="i2i", modalities=["image"], eval_splits=_SPLITS, @@ -42,7 +42,7 @@ class STSBenchmarkMultilingualVisualSTS(AbsTaskVisualSTS, MultilingualTask): main_score="cosine_spearman", date=("2012-01-01", "2017-12-31"), domains=["News", "Social", "Web", "Spoken", "Written"], - task_subtypes=[], + task_subtypes=["Rendered semantic textual similarity"], license="not specified", annotations_creators="human-annotated", dialect=[], diff --git a/mteb/tasks/aggregated_tasks/STS17MultilingualVisualSTS.py b/mteb/tasks/aggregated_tasks/STS17MultilingualVisualSTS.py new file mode 100644 index 0000000000..563f09cbe6 --- /dev/null +++ b/mteb/tasks/aggregated_tasks/STS17MultilingualVisualSTS.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTask import AbsTask +from mteb.abstasks.aggregated_task import AbsTaskAggregate, AggregateTaskMetadata +from mteb.tasks.Image.VisualSTS import STS17MultilingualVisualSTS + +task_list_sts17: list[AbsTask] = [ + STS17MultilingualVisualSTS().filter_languages( + languages=["eng"], hf_subsets=["en-en"] + ) +] + + +class STS17MultilingualVisualSTSEng(AbsTaskAggregate): + metadata = AggregateTaskMetadata( + name="VisualSTS17Eng", + description="STS17MultilingualVisualSTS English only.", + reference="https://arxiv.org/abs/2402.08183/", + tasks=task_list_sts17, + category="i2i", + license="not specified", + annotations_creators="human-annotated", + dialect=[""], + eval_langs={ + "en-en": ["eng-Latn"] + }, # rely on subsets to filter scores in TaskResults.get_score_fast(). + sample_creation="rendered", + main_score="cosine_spearman", + type="VisualSTS(eng)", + eval_splits=["test"], + bibtex_citation="""@article{xiao2024pixel, + title={Pixel Sentence Representation Learning}, + author={Xiao, Chenghao and Huang, Zhuoxu and Chen, Danlu and Hudson, G Thomas and Li, Yizhi and Duan, Haoran and Lin, Chenghua and Fu, Jie and Han, Jungong and Moubayed, Noura Al}, + journal={arXiv preprint arXiv:2402.08183}, + year={2024} +}""", + ) + + +task_list_sts17_multi: list[AbsTask] = [ + STS17MultilingualVisualSTS().filter_languages( + languages=["ara", "eng", "spa", "kor"], + hf_subsets=[ + "ko-ko", + "ar-ar", + "en-ar", + "en-de", + "en-tr", + "es-en", + "es-es", + "fr-en", + "it-en", + "nl-en", + ], + ) +] + + +class STS17MultilingualVisualSTSMultilingual(AbsTaskAggregate): + metadata = AggregateTaskMetadata( + name="VisualSTS17Multilingual", + description="STS17MultilingualVisualSTS multilingual.", + reference="https://arxiv.org/abs/2402.08183/", + tasks=task_list_sts17_multi, + category="i2i", + license="not specified", + annotations_creators="human-annotated", + dialect=[""], + sample_creation="rendered", + main_score="cosine_spearman", + type="VisualSTS(multi)", + eval_splits=["test"], + eval_langs={ # rely on subsets to filter scores in TaskResults.get_score_fast(). + "ko-ko": ["kor-Hang"], + "ar-ar": ["ara-Arab"], + "en-ar": ["eng-Latn", "ara-Arab"], + "en-de": ["eng-Latn", "deu-Latn"], + "en-tr": ["eng-Latn", "tur-Latn"], + "es-en": ["spa-Latn", "eng-Latn"], + "es-es": ["spa-Latn"], + "fr-en": ["fra-Latn", "eng-Latn"], + "it-en": ["ita-Latn", "eng-Latn"], + "nl-en": ["nld-Latn", "eng-Latn"], + }, + bibtex_citation="""@article{xiao2024pixel, + title={Pixel Sentence Representation Learning}, + author={Xiao, Chenghao and Huang, Zhuoxu and Chen, Danlu and Hudson, G Thomas and Li, Yizhi and Duan, Haoran and Lin, Chenghua and Fu, Jie and Han, Jungong and Moubayed, Noura Al}, + journal={arXiv preprint arXiv:2402.08183}, + year={2024} +}""", + ) diff --git a/mteb/tasks/aggregated_tasks/STSBenchmarkMultilingualVisualSTS.py b/mteb/tasks/aggregated_tasks/STSBenchmarkMultilingualVisualSTS.py new file mode 100644 index 0000000000..74c5f9feb6 --- /dev/null +++ b/mteb/tasks/aggregated_tasks/STSBenchmarkMultilingualVisualSTS.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from mteb.abstasks.AbsTask import AbsTask +from mteb.abstasks.aggregated_task import AbsTaskAggregate, AggregateTaskMetadata +from mteb.tasks.Image.VisualSTS import STSBenchmarkMultilingualVisualSTS + +task_list_stsb: list[AbsTask] = [ + STSBenchmarkMultilingualVisualSTS().filter_languages( + languages=["eng"], hf_subsets=["en"] + ) +] + + +class STSBenchmarkMultilingualVisualSTSEng(AbsTaskAggregate): + metadata = AggregateTaskMetadata( + name="VisualSTS-b-Eng", + description="STSBenchmarkMultilingualVisualSTS English only.", + reference="https://arxiv.org/abs/2402.08183/", + tasks=task_list_stsb, + category="i2i", + license="not specified", + annotations_creators="human-annotated", + dialect=[""], + sample_creation="rendered", + main_score="cosine_spearman", + type="VisualSTS(eng)", + eval_splits=["test"], + eval_langs=["eng-Latn"], + bibtex_citation="""@article{xiao2024pixel, + title={Pixel Sentence Representation Learning}, + author={Xiao, Chenghao and Huang, Zhuoxu and Chen, Danlu and Hudson, G Thomas and Li, Yizhi and Duan, Haoran and Lin, Chenghua and Fu, Jie and Han, Jungong and Moubayed, Noura Al}, + journal={arXiv preprint arXiv:2402.08183}, + year={2024} +}""", + ) + + +task_list_multi: list[AbsTask] = [ + STSBenchmarkMultilingualVisualSTS().filter_languages( + languages=[ + "deu", + "spa", + "fra", + "ita", + "nld", + "pol", + "por", + "rus", + "cmn", + ], + hf_subsets=[ + "de", + "es", + "fr", + "it", + "nl", + "pl", + "pt", + "ru", + "zh", + ], + ) +] + + +class STSBenchmarkMultilingualVisualSTSMultilingual(AbsTaskAggregate): + metadata = AggregateTaskMetadata( + name="VisualSTS-b-Multilingual", + description="STSBenchmarkMultilingualVisualSTS multilingual.", + reference="https://arxiv.org/abs/2402.08183/", + tasks=task_list_multi, + category="i2i", + license="not specified", + annotations_creators="human-annotated", + dialect=[""], + sample_creation="rendered", + main_score="cosine_spearman", + type="VisualSTS(multi)", + eval_splits=["test"], + eval_langs=[ + "deu-Latn", + "spa-Latn", + "fra-Latn", + "ita-Latn", + "nld-Latn", + "pol-Latn", + "por-Latn", + "rus-Cyrl", + "cmn-Hans", + ], + bibtex_citation="""@article{xiao2024pixel, + title={Pixel Sentence Representation Learning}, + author={Xiao, Chenghao and Huang, Zhuoxu and Chen, Danlu and Hudson, G Thomas and Li, Yizhi and Duan, Haoran and Lin, Chenghua and Fu, Jie and Han, Jungong and Moubayed, Noura Al}, + journal={arXiv preprint arXiv:2402.08183}, + year={2024} +}""", + ) diff --git a/mteb/tasks/aggregated_tasks/__init__.py b/mteb/tasks/aggregated_tasks/__init__.py index 7a1e406658..d6ef84d795 100644 --- a/mteb/tasks/aggregated_tasks/__init__.py +++ b/mteb/tasks/aggregated_tasks/__init__.py @@ -3,6 +3,14 @@ from .CQADupStackNLRetrieval import CQADupstackNLRetrieval from .CQADupStackRetrieval import CQADupstackRetrieval from .CQADupStackRetrievalFa import CQADupstackRetrievalFa +from .STS17MultilingualVisualSTS import ( + STS17MultilingualVisualSTSEng, + STS17MultilingualVisualSTSMultilingual, +) +from .STSBenchmarkMultilingualVisualSTS import ( + STSBenchmarkMultilingualVisualSTSEng, + STSBenchmarkMultilingualVisualSTSMultilingual, +) from .SynPerChatbotConvSAClassification import SynPerChatbotConvSAClassification __all__ = [ @@ -10,4 +18,8 @@ "CQADupstackRetrievalFa", "SynPerChatbotConvSAClassification", "CQADupstackNLRetrieval", + "STS17MultilingualVisualSTSEng", + "STS17MultilingualVisualSTSMultilingual", + "STSBenchmarkMultilingualVisualSTSEng", + "STSBenchmarkMultilingualVisualSTSMultilingual", ] diff --git a/scripts/run_mieb.py b/scripts/run_mieb.py index b3c55b26d5..a0d444617e 100644 --- a/scripts/run_mieb.py +++ b/scripts/run_mieb.py @@ -60,13 +60,14 @@ task_types=[ "Any2AnyRetrieval", "Any2AnyMultiChoice", - "Any2TextMutipleChoice", + "VisionCentric", "ImageClustering", "ImageClassification", "ImageMultilabelClassification", - "ImageTextPairClassification", + "Compositionality", "VisualSTS", "ZeroShotClassification", + "DocumentUnderstanding", ] ) # get i-only tasks for i-only models. diff --git a/scripts/run_mieb_agg_task.py b/scripts/run_mieb_agg_task.py new file mode 100644 index 0000000000..b59f989efc --- /dev/null +++ b/scripts/run_mieb_agg_task.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import mteb + +for model_name in [ + "openai/clip-vit-base-patch32", + "openai/clip-vit-base-patch16", + "openai/clip-vit-large-patch14", + "royokong/e5-v", + "BAAI/bge-visualized-base", + "BAAI/bge-visualized-m3", + "kakaobrain/align-base", + "jinaai/jina-clip-v1", + "nomic-ai/nomic-embed-vision-v1.5", + "Salesforce/blip-image-captioning-large", + "Salesforce/blip-image-captioning-base", + "Salesforce/blip2-opt-2.7b", + "Salesforce/blip2-opt-6.7b-coco", + "facebook/dinov2-small", + "facebook/dinov2-base", + "facebook/dinov2-large", + "facebook/dinov2-giant", + "nyu-visionx/moco-v3-vit-b", + "nyu-visionx/moco-v3-vit-l", + "google/siglip-so400m-patch14-224", + "google/siglip-so400m-patch14-384", + "google/siglip-so400m-patch16-256-i18n", + "google/siglip-base-patch16-256-multilingual", + "google/siglip-base-patch16-256", + "google/siglip-base-patch16-512", + "google/siglip-base-patch16-384", + "google/siglip-base-patch16-224", + "google/siglip-large-patch16-256", + "google/siglip-large-patch16-384", + "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", + "laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K", + "laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K", + "laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", + "laion/CLIP-ViT-g-14-laion2B-s34B-b88K", + "laion/CLIP-ViT-H-14-laion2B-s32B-b79K", + "laion/CLIP-ViT-L-14-laion2B-s32B-b82K", + "laion/CLIP-ViT-B-32-laion2B-s34B-b79K", + "TIGER-Lab/VLM2Vec-Full", + "TIGER-Lab/VLM2Vec-LoRA", + "Salesforce/blip-itm-base-coco", + "Salesforce/blip-itm-large-coco", + "Salesforce/blip-itm-base-flickr", + "Salesforce/blip-itm-large-flickr", + "QuanSun/EVA02-CLIP-B-16", + "QuanSun/EVA02-CLIP-L-14", + "QuanSun/EVA02-CLIP-bigE-14", + "QuanSun/EVA02-CLIP-bigE-14-plus", + "voyageai/voyage-multimodal-3", +]: + model = mteb.get_model(model_name) + tasks = mteb.get_tasks( + tasks=[ + "VisualSTS-b-Eng", + "VisualSTS-b-Multilingual", + "VisualSTS17Eng", + "VisualSTS17Multilingual", + ] + ) + + evaluation = mteb.MTEB(tasks=tasks) + results = evaluation.run(model, output_folder="/home/.cache/mteb/results/results") diff --git a/scripts/run_mieb_rerun_siglip.py b/scripts/run_mieb_rerun_siglip.py index 539a31e2e7..5c7bad9f27 100644 --- a/scripts/run_mieb_rerun_siglip.py +++ b/scripts/run_mieb_rerun_siglip.py @@ -17,13 +17,14 @@ task_types=[ "Any2AnyRetrieval", "Any2AnyMultiChoice", - "Any2TextMutipleChoice", + "VisionCentric", "ImageClustering", "ImageClassification", "ImageMultilabelClassification", - "ImageTextPairClassification", + "Compositionality", # "VisualSTS", # visual sts does not need rerun as will be the same after fixed. "ZeroShotClassification", + "DocumentUnderstanding", ] ) evaluation = mteb.MTEB(tasks=tasks) diff --git a/tests/test_TaskMetadata.py b/tests/test_TaskMetadata.py index e5c160cdf3..241ec536ea 100644 --- a/tests/test_TaskMetadata.py +++ b/tests/test_TaskMetadata.py @@ -181,6 +181,8 @@ "TenKGnadClusteringS2S.v2", "SynPerChatbotConvSAClassification", "CQADupstackRetrieval-Fa", + "VisualSTS17Eng", + "VisualSTS17Multilingual", ] @@ -1103,6 +1105,9 @@ def test_empy_descriptive_stat_in_new_datasets(): @pytest.mark.parametrize("task", get_tasks()) def test_eval_langs_correctly_specified(task: AbsTask): + if task.metadata.name in ["VisualSTS17Eng", "VisualSTS17Multilingual"]: + return + if task.is_multilingual: assert isinstance(task.metadata.eval_langs, dict), ( f"{task.metadata.name} should have eval_langs as a dict" diff --git a/tests/test_benchmark/mock_tasks.py b/tests/test_benchmark/mock_tasks.py index eea73b2e69..b268bd83c3 100644 --- a/tests/test_benchmark/mock_tasks.py +++ b/tests/test_benchmark/mock_tasks.py @@ -2283,7 +2283,7 @@ def load_data(self, **kwargs): class MockTextMultipleChoiceTask(AbsTaskAny2TextMultipleChoice): metadata = TaskMetadata( - type="Any2TextMutipleChoice", + type="VisionCentric", name="MockTextMultipleChoice", main_score="accuracy", descriptive_stats={ @@ -2722,7 +2722,7 @@ def load_data(self, **kwargs): class MockImageTextPairClassificationTask(AbsTaskImageTextPairClassification): metadata = TaskMetadata( - type="ImageTextPairClassification", + type="Compositionality", name="MockImageTextPairClassification", main_score="text_acc", descriptive_stats={ @@ -2763,7 +2763,7 @@ class MockMultilingualImageTextPairClassificationTask( AbsTaskImageTextPairClassification, MultilingualTask ): metadata = TaskMetadata( - type="ImageTextPairClassification", + type="Compositionality", name="MockMultilingualImageTextPairClassification", main_score="accuracy", descriptive_stats={ @@ -2824,7 +2824,7 @@ def load_data(self, **kwargs): class MockVisualSTSTask(AbsTaskVisualSTS): metadata = TaskMetadata( - type="VisualSTS", + type="VisualSTS(eng)", name="MockVisualSTS", main_score="cosine_spearman", descriptive_stats={ diff --git a/tests/test_overview.py b/tests/test_overview.py index 7041328a59..59bc86eb9c 100644 --- a/tests/test_overview.py +++ b/tests/test_overview.py @@ -76,11 +76,16 @@ def test_get_tasks_filtering(): tasks = get_tasks(languages=["eng"]) for task in tasks: - if task.is_multilingual: + if ( + task.is_multilingual + and task.metadata.name != "STS17MultilingualVisualSTSEng" + ): assert isinstance(task.metadata.eval_langs, dict) for hf_subset in task.hf_subsets: - assert "eng-Latn" in task.metadata.eval_langs[hf_subset] + assert "eng-Latn" in task.metadata.eval_langs[hf_subset], ( + f"{task.metadata.name}" + ) @pytest.mark.parametrize("script", [["Latn"], ["Cyrl"], None])