diff --git a/docs/overview/create_available_benchmarks.py b/docs/overview/create_available_benchmarks.py index b5ee8956af..01bb873e6f 100644 --- a/docs/overview/create_available_benchmarks.py +++ b/docs/overview/create_available_benchmarks.py @@ -3,6 +3,9 @@ from pathlib import Path from typing import cast +from prettify_list import pretty_long_list +from slugify import slugify_anchor + import mteb from mteb.get_tasks import MTEBTasks @@ -32,12 +35,6 @@ """ -def pretty_long_list(items: list[str], max_items: int = 5) -> str: - if len(items) <= max_items: - return ", ".join(items) - return ", ".join(items[:max_items]) + f", ... ({len(items)})" - - def create_table(benchmark: mteb.Benchmark) -> str: """Create a markdown table of tasks in the benchmark.""" tasks = benchmark.tasks @@ -47,7 +44,10 @@ def create_table(benchmark: mteb.Benchmark) -> str: # add links to task names: # format: http://127.0.0.1:8000/overview/available_tasks/retrieval/#treccovid df["name"] = df.apply( - lambda row: f"[{row['name']}](./available_tasks/{row['type'].lower()}.md#{row['name'].lower()})", + lambda row: ( + f"[{row['name']}](./available_tasks/{row['type'].lower()}.md" + f"#{slugify_anchor(row['name'])})" + ), axis=1, ) df["modalities"] = df["modalities"].apply(lambda x: pretty_long_list(x)) diff --git a/docs/overview/create_available_models.py b/docs/overview/create_available_models.py index c320865da4..eb4b6a976f 100644 --- a/docs/overview/create_available_models.py +++ b/docs/overview/create_available_models.py @@ -2,6 +2,8 @@ from pathlib import Path +from prettify_list import pretty_long_list + import mteb from mteb.models import ModelMeta @@ -62,12 +64,6 @@ def human_readable_number(num: int) -> str: return f"{num:.2f}P" -def pretty_long_list(items: list[str], max_items: int = 5) -> str: - if len(items) <= max_items: - return ", ".join(items) - return ", ".join(items[:max_items]) + f", ... ({len(items)})" - - def modality_to_string(modality: tuple[str, ...]) -> str: return ("-".join(modality)).capitalize() diff --git a/docs/overview/create_available_tasks.py b/docs/overview/create_available_tasks.py index 3846dbc461..444d2fac4f 100644 --- a/docs/overview/create_available_tasks.py +++ b/docs/overview/create_available_tasks.py @@ -1,20 +1,31 @@ """Updates the available tasks markdown files.""" from pathlib import Path +from typing import cast + +from prettify_list import pretty_long_list +from slugify import slugify_anchor import mteb +from mteb.abstasks.aggregated_task import AbsTaskAggregate +from mteb.get_tasks import MTEBTasks task_entry = """ #### {task_name} {description} -**Dataset:** [`{dataset_name}`](https://huggingface.co/datasets/{dataset_name}) • **License:** {license} • [Learn more →]({reference}) +{dataset_line} | Task category | Score | Languages | Domains | Annotations Creators | Sample Creation | |-------|-------|-------|-------|-------|-------| | {task_category_string} ({task_category}) | {main_score} | {languages} | {domains} | {annotation_creators} | {sample_creation} | +""" +aggregated_tasks_section = """ +??? info "Tasks" + +{task_table} """ task_type_section = """ @@ -43,12 +54,6 @@ """ -def pretty_long_list(items: list[str], max_items: int = 5) -> str: - if len(items) <= max_items: - return ", ".join(items) - return ", ".join(items[:max_items]) + f", ... ({len(items)})" - - def task_category_to_string(category: str) -> str: """Convert task category to a more readable string, @@ -77,16 +82,34 @@ def task_category_to_string(category: str) -> str: return f"{', '.join(input_strings)} to {', '.join(output_strings)}" +def create_aggregate_table(task: AbsTaskAggregate) -> str: + tasks = cast(MTEBTasks, MTEBTasks(task.metadata.tasks)) + df = tasks.to_dataframe(["name", "type", "modalities", "languages"]) + df["name"] = df.apply( + lambda row: ( + f"[{row['name']}](./{row['type'].lower()}.md#{slugify_anchor(row['name'])})" + ), + axis=1, + ) + df["modalities"] = df["modalities"].apply(lambda x: pretty_long_list(x)) + df["languages"] = df["languages"].apply(lambda x: pretty_long_list(x)) + return df.to_markdown(index=False) + + def format_task_entry(task: mteb.AbsTask) -> str: description = task.metadata.description - dataset_name = task.metadata.dataset["path"] license = task.metadata.license or "not specified" - reference = ( - task.metadata.reference or f"https://huggingface.co/datasets/{dataset_name}" - ) + reference = task.metadata.reference + dataset_name = task.metadata.dataset["path"] + if not reference and not isinstance(task, AbsTaskAggregate): + reference = f"https://huggingface.co/datasets/{dataset_name}" main_score = task.metadata.main_score - task_category = task.metadata.category - task_category_string = task_category_to_string(task_category) + task_category = task.metadata.category or "not specified" + task_category_string = ( + task_category_to_string(task_category) + if task.metadata.category + else "not specified" + ) languages = pretty_long_list(task.metadata.languages) domains = ( pretty_long_list(sorted(task.metadata.domains)) @@ -96,12 +119,23 @@ def format_task_entry(task: mteb.AbsTask) -> str: annotation_creators = task.metadata.annotations_creators or "not specified" sample_creation = task.metadata.sample_creation or "not specified" + if reference: + learn_more = f"[Learn more →]({reference})" + else: + learn_more = "Learn more → not specified" + + if not isinstance(task, AbsTaskAggregate): + dataset_line = ( + f"**Dataset:** [`{dataset_name}`](https://huggingface.co/datasets/{dataset_name}) " + f"• **License:** {license} • {learn_more}" + ) + else: + dataset_line = f"**License:** {license} • {learn_more}" + entry = task_entry.format( task_name=task.metadata.name, description=description, - dataset_name=dataset_name, - license=license, - reference=reference, + dataset_line=dataset_line, main_score=main_score, task_category=task_category, task_category_string=task_category_string, @@ -115,13 +149,22 @@ def format_task_entry(task: mteb.AbsTask) -> str: citation = "\n".join([f" {line}" for line in citation.split("\n")]) # indent entry += citation_admonition.format(citation_chunk=citation) + if isinstance(task, AbsTaskAggregate): + task_table = create_aggregate_table(task) + task_table = "\n".join([f" {line}" for line in task_table.split("\n")]) + entry += aggregated_tasks_section.format(task_table=task_table) + return entry def main(folder: Path) -> None: folder.mkdir(exist_ok=True) - tasks = mteb.get_tasks(exclude_superseded=False, exclude_aggregate=True) + tasks = mteb.get_tasks( + exclude_superseded=False, + exclude_aggregate=False, + exclude_private=False, + ) task_types = sorted({task.metadata.type for task in tasks}) task_types2tasks = {task_type: [] for task_type in task_types} diff --git a/docs/overview/prettify_list.py b/docs/overview/prettify_list.py new file mode 100644 index 0000000000..4fede05d71 --- /dev/null +++ b/docs/overview/prettify_list.py @@ -0,0 +1,4 @@ +def pretty_long_list(items: list[str], max_items: int = 5) -> str: + if len(items) <= max_items: + return ", ".join(items) + return ", ".join(items[:max_items]) + f", ... ({len(items)})" diff --git a/docs/overview/slugify.py b/docs/overview/slugify.py new file mode 100644 index 0000000000..20658ac808 --- /dev/null +++ b/docs/overview/slugify.py @@ -0,0 +1,6 @@ +import re + + +def slugify_anchor(value: str) -> str: + slug = re.sub(r"[^\w\s-]", "", value).strip().lower() + return re.sub(r"[\s]+", "-", slug) diff --git a/docs/whats_new.md b/docs/whats_new.md index da234779b9..2352a2967e 100644 --- a/docs/whats_new.md +++ b/docs/whats_new.md @@ -3,7 +3,7 @@ This section goes through new features added in v2. Below we give an overview of **Overview of changes:** -- [New in v2.0 🎉](#new-in-v20-🎉) +- [New in v2.0 🎉](#new-in-v20) - [Easier evaluation](#easier-evaluation) - [Better local and online caching](#better-local-and-online-caching) - [Multimodal Input format](#multimodal-input-format)