diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index ad0d2bd6be..aa4658f513 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -93,6 +93,8 @@ "Emotional Speech Retrieval", "Environment Sound Retrieval", "Speech Retrieval", + "Question Answering Retrieval", + "Reading Comprehension", ] TASK_DOMAIN = Literal[ diff --git a/mteb/tasks/Audio/Any2AnyRetrieval/SpokenSQuAD.py b/mteb/tasks/Audio/Any2AnyRetrieval/SpokenSQuAD.py new file mode 100644 index 0000000000..01115ca206 --- /dev/null +++ b/mteb/tasks/Audio/Any2AnyRetrieval/SpokenSQuAD.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyRetrieval import AbsTaskAny2AnyRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class SpokenSQuADT2ARetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="SpokenSQuADT2ARetrieval", + description="Text-to-audio retrieval task based on SpokenSQuAD dataset. Given a text question, retrieve relevant audio segments that contain the answer. Questions are derived from SQuAD reading comprehension dataset with corresponding spoken passages.", + reference="https://github.com/chiuwy/Spoken-SQuAD", + dataset={ + "path": "arteemg/spoken-squad-t2a", + "revision": "63c5c8e4cf762e52e18ecde8baf5733eab114ae9", + }, + type="Any2AnyRetrieval", + category="t2a", + modalities=["text", "audio"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cv_recall_at_5", + date=("2018-03-01", "2018-03-01"), + domains=["Academic", "Encyclopaedic", "Non-fiction"], + task_subtypes=["Question Answering Retrieval", "Reading Comprehension"], + license="cc-by-sa-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=r""" +@inproceedings{li2018spokensquad, + author = {Li, Chia-Hsuan and Ma, Szu-Lin and Zhang, Hsin-Wei and Lee, Hung-yi and Lee, Lin-shan}, + booktitle = {Interspeech}, + pages = {3459--3463}, + title = {Spoken SQuAD: A Study of Mitigating the Impact of Speech Recognition Errors on Listening Comprehension}, + year = {2018}, +} +""", + ) diff --git a/mteb/tasks/Audio/Any2AnyRetrieval/__init__.py b/mteb/tasks/Audio/Any2AnyRetrieval/__init__.py index 545c92164e..74f9bc2432 100644 --- a/mteb/tasks/Audio/Any2AnyRetrieval/__init__.py +++ b/mteb/tasks/Audio/Any2AnyRetrieval/__init__.py @@ -14,4 +14,5 @@ from .multilingual.JamAlt import * from .MusicCaps import * from .SoundDescs import * +from .SpokenSQuAD import * from .UrbanSound8kRetrieval import *