Skip to content

[mieb] run tasks small fix #1310

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gowitheflow-1998 merged 14 commits into from
Oct 22, 2024
Merged

[mieb] run tasks small fix #1310

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
91ad565
fix ImageTextPair dataloading for large datasets; more compositionali…
gowitheflow-1998 Sep 23, 2024
a5011e3
Merge branch 'mieb' of https://github.com/embeddings-benchmark/mteb i…
gowitheflow-1998 Oct 12, 2024
d7a1e25
Merge branch 'mieb' of https://github.com/embeddings-benchmark/mteb i…
gowitheflow-1998 Oct 20, 2024
bc8aa01
fix e5v&vista
gowitheflow-1998 Oct 21, 2024
7aa5bd2
Merge branch 'mieb' of https://github.com/embeddings-benchmark/mteb i…
gowitheflow-1998 Oct 21, 2024
53065b8
task type fix for running tasks
gowitheflow-1998 Oct 21, 2024
3b93440
fix wrong meta
gowitheflow-1998 Oct 21, 2024
3998ccd
run mieb script
gowitheflow-1998 Oct 21, 2024
cd63dd6
script
gowitheflow-1998 Oct 22, 2024
0a172ae
lint
gowitheflow-1998 Oct 22, 2024
b854159
align
gowitheflow-1998 Oct 22, 2024
5846727
fix
gowitheflow-1998 Oct 22, 2024
755349f
Merge branch 'mieb' of https://github.com/embeddings-benchmark/mteb i…
gowitheflow-1998 Oct 22, 2024
47284d7
linting
gowitheflow-1998 Oct 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ class ROxfordHardI2IRetrieval(AbsTaskAny2AnyRetrieval):
"path": "JamieSJS/r-oxford-hard",
"revision": "f20b30211b7ba3fc64a02bd83998fe75f3023719",
},
type="Retrieval",
type="Any2AnyRetrieval",
category="i2i",
eval_splits=["test"],
eval_langs=["eng-Latn"],
Expand Down
2 changes: 1 addition & 1 deletion mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ class RParisHardI2IRetrieval(AbsTaskAny2AnyRetrieval):
"path": "JamieSJS/r-paris-hard",
"revision": "fd121b6592fe946616fa85116703b94a4c61fd63",
},
type="Retrieval",
type="Any2AnyRetrieval",
category="i2i",
eval_splits=["test"],
eval_langs=["eng-Latn"],
Expand Down
1 change: 1 addition & 0 deletions mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ class Caltech101Classification(AbsTaskImageClassification):
"path": "HuggingFaceM4/Caltech-101",
"name": "with_background_category",
"revision": "851374102055782c84f89b1b4e9d128a6568847b",
"trust_remote_code": True,
},
type="ImageClassification",
category="i2t",
Expand Down
1 change: 1 addition & 0 deletions mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ class FGVCAircraftClassification(AbsTaskImageClassification):
dataset={
"path": "HuggingFaceM4/FGVC-Aircraft",
"revision": "91860adfc9a09aabca5cddb5247442109b38e213",
"trust_remote_code": True,
},
type="ImageClassification",
category="i2t",
Expand Down
2 changes: 1 addition & 1 deletion mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ class MNISTClassification(AbsTaskImageClassification):
reference="https://en.wikipedia.org/wiki/MNIST_database",
dataset={
"path": "ylecun/mnist",
"revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574",
"revision": "77f3279092a1c1579b2250db8eafed0ad422088c",
},
type="ImageClassification",
category="i2t",
Expand Down
1 change: 1 addition & 0 deletions mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ class VOC2007Classification(AbsTaskImageMultilabelClassification):
"path": "HuggingFaceM4/pascal_voc",
"name": "voc2007_main",
"revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50",
"trust_remote_code": True,
},
type="ImageMultilabelClassification",
category="i2t",
Expand Down
4 changes: 2 additions & 2 deletions mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

class CLEVR(AbsTaskZeroshotClassification):
metadata = TaskMetadata(
name="CLEVR",
name="CLEVRZeroShot",
description="CLEVR closest object distance identification task.",
reference="https://openaccess.thecvf.com/content_cvpr_2017/html/Johnson_CLEVR_A_Diagnostic_CVPR_2017_paper.html",
dataset={
Expand Down Expand Up @@ -61,7 +61,7 @@ def get_candidate_labels(self) -> list[str]:

class CLEVRCount(AbsTaskZeroshotClassification):
metadata = TaskMetadata(
name="CLEVRCount",
name="CLEVRCountZeroShot",
description="CLEVR count objects task.",
reference="https://openaccess.thecvf.com/content_cvpr_2017/html/Johnson_CLEVR_A_Diagnostic_CVPR_2017_paper.html",
dataset={
Expand Down
1 change: 1 addition & 0 deletions mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ class Caltech101Classification(AbsTaskZeroshotClassification):
"path": "HuggingFaceM4/Caltech-101",
"name": "with_background_category",
"revision": "851374102055782c84f89b1b4e9d128a6568847b",
"trust_remote_code": True,
},
type="ZeroShotClassification",
category="i2t",
Expand Down
1 change: 1 addition & 0 deletions mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ class FGVCAircraftClassification(AbsTaskZeroshotClassification):
dataset={
"path": "HuggingFaceM4/FGVC-Aircraft",
"revision": "91860adfc9a09aabca5cddb5247442109b38e213",
"trust_remote_code": True,
},
type="ZeroShotClassification",
category="i2t",
Expand Down
2 changes: 1 addition & 1 deletion mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ class MNISTClassification(AbsTaskZeroshotClassification):
reference="https://en.wikipedia.org/wiki/MNIST_database",
dataset={
"path": "ylecun/mnist",
"revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574",
"revision": "77f3279092a1c1579b2250db8eafed0ad422088c",
},
type="ZeroShotClassification",
category="i2t",
Expand Down
2 changes: 1 addition & 1 deletion scripts/run_mieb.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
"royokong/e5-v",
"BAAI/bge-visualized-base",
"BAAI/bge-visualized-m3",
"google/siglip-so400m-patch14-384",
"kakaobrain/align-base",
"jinaai/jina-clip-v1",
"nomic-ai/nomic-embed-vision-v1.5",
Expand All @@ -26,6 +25,7 @@
"laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K",
"nyu-visionx/moco-v3-vit-b",
"nyu-visionx/moco-v3-vit-l",
# "google/siglip-so400m-patch14-384",# haven't pushed
]:
model = mteb.get_model(model_name)
tasks = mteb.get_tasks(
Expand Down