Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions mteb/benchmarks/benchmarks.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ class Benchmark:
description: A description of the benchmark, should include its intended goal and potentially a description of its construction
reference: A link reference, to a source containing additional information typically to a paper, leaderboard or github.
citation: A bibtex citation
contacts: The people to contact in case of a problem in the benchmark, preferably a GitHub handle.

Example:
>>> Benchmark(
Expand All @@ -44,6 +45,7 @@ class Benchmark:
description: str | None = None
reference: UrlString | None = None
citation: str | None = None
contacts: list[str] | None = None

def __iter__(self):
return iter(self.tasks)
Expand Down Expand Up @@ -128,6 +130,7 @@ def load_results(
),
description="English benchmarks from MTEB",
citation="",
contacts=["KennethEnevoldsen", "Muennighoff"],
)

MTEB_ENG_CLASSIC = Benchmark(
Expand Down Expand Up @@ -232,6 +235,7 @@ def load_results(
pages = "2014--2037",
}
""",
contacts=["Muennighoff"],
)

MTEB_MAIN_RU = Benchmark(
Expand Down Expand Up @@ -422,6 +426,7 @@ def load_results(
archivePrefix={arXiv},
primaryClass={cs.CL}
}""",
contacts=["KennethEnevoldsen", "x-tabdeveloping", "Samoed"],
)

CoIR = Benchmark(
Expand Down Expand Up @@ -484,6 +489,7 @@ def load_results(
journal={arXiv preprint arXiv:2404.06347},
year={2024}
}""",
contacts=["gowitheflow-1998"],
)

MTEB_FRA = Benchmark(
Expand Down Expand Up @@ -538,6 +544,7 @@ def load_results(
primaryClass={cs.CL},
url={https://arxiv.org/abs/2405.20468},
}""",
contacts=["imenelydiaker"],
)


Expand Down Expand Up @@ -833,6 +840,7 @@ def load_results(
description="The Multilingual benchmarks from MMTEB. Currently under development.",
reference=None,
citation=None,
contacts=["KennethEnevoldsen"],
)

MTEB_JPN = Benchmark(
Expand Down Expand Up @@ -944,6 +952,7 @@ def load_results(
description="Main Indic benchmark from MMTEB",
reference=None,
citation=None,
contacts=["KennethEnevoldsen"],
)


Expand Down Expand Up @@ -1075,6 +1084,7 @@ def load_results(
description="Main European benchmark from MMTEB",
reference=None,
citation=None,
contacts=["KennethEnevoldsen"],
)

LONG_EMBED = Benchmark(
Expand Down
Loading