From a269a5aae0c38edce3e43f9ed970abf26cc81875 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Fri, 1 Nov 2024 18:31:32 +0000 Subject: [PATCH 1/6] e5_v device arg --- mteb/models/e5_v.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mteb/models/e5_v.py b/mteb/models/e5_v.py index 70bc20cabf..5647bee380 100644 --- a/mteb/models/e5_v.py +++ b/mteb/models/e5_v.py @@ -21,6 +21,8 @@ def __init__( ): self.model_name = model_name self.processor = LlavaNextProcessor.from_pretrained(model_name) + if "device" in kwargs: + self.device = kwargs.pop("device") self.model = LlavaNextForConditionalGeneration.from_pretrained( model_name, **kwargs ) From 2bbdaf62df1d335241fb73531f1a7b295409420e Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Fri, 1 Nov 2024 18:33:20 +0000 Subject: [PATCH 2/6] dataloader num_workers --- .../evaluators/Image/Any2AnyMultiChoiceEvaluator.py | 5 +++-- .../evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py | 5 +++-- mteb/evaluation/evaluators/Image/ClassificationEvaluator.py | 1 + .../evaluators/Image/ImageTextPairClassificationEvaluator.py | 2 +- mteb/evaluation/evaluators/Image/VisualSTSEvaluator.py | 4 ++-- .../evaluators/Image/ZeroshotClassificationEvaluator.py | 4 +++- 6 files changed, 13 insertions(+), 8 deletions(-) diff --git a/mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py b/mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py index a391af472b..953a5e3c36 100644 --- a/mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py +++ b/mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py @@ -11,6 +11,7 @@ import numpy as np import pytrec_eval import torch +import math from datasets import Dataset from PIL import Image from torch.utils.data import DataLoader @@ -132,7 +133,7 @@ def search( batch_size=self.encode_kwargs["batch_size"], shuffle=False, collate_fn=custom_collate_fn, - num_workers=max(1, os.cpu_count() // 2), + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) if q_modality == "image": query_embeddings = self.model.get_image_embeddings( @@ -182,7 +183,7 @@ def search( batch_size=self.encode_kwargs["batch_size"], shuffle=False, collate_fn=custom_collate_fn, - num_workers=max(1, os.cpu_count() // 2), + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) if corpus_modality == "image": sub_corpus_embeddings = self.model.get_image_embeddings( diff --git a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py index cb0119ea6e..257a1c939c 100644 --- a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py @@ -5,6 +5,7 @@ import json import logging import os +import math from collections import defaultdict from typing import Any @@ -131,7 +132,7 @@ def search( batch_size=self.encode_kwargs["batch_size"], shuffle=False, collate_fn=custom_collate_fn, - num_workers=max(1, os.cpu_count() // 2), + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) if q_modality == "image": query_embeddings = self.model.get_image_embeddings( @@ -181,7 +182,7 @@ def search( batch_size=self.encode_kwargs["batch_size"], shuffle=False, collate_fn=custom_collate_fn, - num_workers=max(1, os.cpu_count() // 2), + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) if corpus_modality == "image": sub_corpus_embeddings = self.model.get_image_embeddings( diff --git a/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py index 2f60b5c690..647853cc32 100644 --- a/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py @@ -7,6 +7,7 @@ import numpy as np import torch +import math from sklearn.linear_model import LogisticRegression from sklearn.metrics import ( accuracy_score, diff --git a/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py index b548da365e..edaf779f4c 100644 --- a/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py @@ -106,7 +106,7 @@ def __call__( shuffle=False, # collate_fn=lambda x: x, # Identity collate function collate_fn=custom_collate_fn, - num_workers=4, + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) num_images_per_sample = ( diff --git a/mteb/evaluation/evaluators/Image/VisualSTSEvaluator.py b/mteb/evaluation/evaluators/Image/VisualSTSEvaluator.py index a442eb6a9a..a042d22f5a 100644 --- a/mteb/evaluation/evaluators/Image/VisualSTSEvaluator.py +++ b/mteb/evaluation/evaluators/Image/VisualSTSEvaluator.py @@ -78,14 +78,14 @@ def __call__( batch_size=encode_kwargs["batch_size"], shuffle=False, collate_fn=custom_collate_fn, - num_workers=math.floor(os.cpu_count() / 2), + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) sentence2_dataloader = DataLoader( self.sentence2_dataset, batch_size=encode_kwargs["batch_size"], shuffle=False, collate_fn=custom_collate_fn, - num_workers=math.floor(os.cpu_count() / 2), + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) embeddings1 = model.get_image_embeddings( diff --git a/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py index d082f13517..6f4d9ae623 100644 --- a/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py @@ -4,6 +4,8 @@ from typing import Any import torch +import math +import os from sklearn import metrics from torch.utils.data import DataLoader from torchvision import transforms @@ -66,7 +68,7 @@ def __call__(self, model: Encoder, *, encode_kwargs: dict[str, Any] = {}): batch_size=encode_kwargs["batch_size"], shuffle=False, collate_fn=custom_collate_fn, - num_workers=16, + num_workers=min(math.floor(os.cpu_count() / 2), 16), ) text_embeddings = model.get_text_embeddings( From 3acc73e99857029a71362d469d3da8be0851defe Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Fri, 1 Nov 2024 18:33:37 +0000 Subject: [PATCH 3/6] vista doc --- docs/mieb-docs/run_vista.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 docs/mieb-docs/run_vista.md diff --git a/docs/mieb-docs/run_vista.md b/docs/mieb-docs/run_vista.md new file mode 100644 index 0000000000..0d5fb0de82 --- /dev/null +++ b/docs/mieb-docs/run_vista.md @@ -0,0 +1,21 @@ +## set up VISTA + +the latest FlagEmbedding repo doesn't support VISTA anymore so we use a old version. +``` +git clone --no-checkout https://github.com/FlagOpen/FlagEmbedding.git +cd FlagEmbedding +git checkout 5c9260277977f8f8e256e56a8e12387552693af9 +pip install -e . +pip install torchvision timm einops ftfy +``` +download the vision tower for bge-base +``` +wget https://huggingface.co/BAAI/bge-visualized/resolve/main/Visualized_base_en_v1.5.pth?download=true +``` +rename it to `visualized_base_en_V1.5.pth` + +download the vision tower for bge-m3 +``` +wget https://huggingface.co/BAAI/bge-visualized/resolve/main/Visualized_m3.pth?download=true +``` +rename it to `visualized_m3.pth` \ No newline at end of file From c9f02f883670654a8ce070a0d8d07accbcf0b3c9 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Fri, 1 Nov 2024 19:15:54 +0000 Subject: [PATCH 4/6] vista doc --- docs/mieb-docs/run_vista.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/mieb-docs/run_vista.md b/docs/mieb-docs/run_vista.md index 0d5fb0de82..038b44bb92 100644 --- a/docs/mieb-docs/run_vista.md +++ b/docs/mieb-docs/run_vista.md @@ -13,9 +13,14 @@ download the vision tower for bge-base wget https://huggingface.co/BAAI/bge-visualized/resolve/main/Visualized_base_en_v1.5.pth?download=true ``` rename it to `visualized_base_en_V1.5.pth` - +``` +mv Visualized_base_en_v1.5.pth?download=true visualized_base_en_V1.5.pth +``` download the vision tower for bge-m3 ``` wget https://huggingface.co/BAAI/bge-visualized/resolve/main/Visualized_m3.pth?download=true ``` -rename it to `visualized_m3.pth` \ No newline at end of file +rename it to `visualized_m3.pth` +``` +mv Visualized_m3.pth?download=true visualized_m3.pth +``` \ No newline at end of file From 29694d1a3cc6502cb6f50576c0f74752f443c622 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Fri, 1 Nov 2024 19:16:25 +0000 Subject: [PATCH 5/6] run mieb --- scripts/run_mieb.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/run_mieb.py b/scripts/run_mieb.py index 3fd3e3c81b..676fd966a7 100644 --- a/scripts/run_mieb.py +++ b/scripts/run_mieb.py @@ -43,6 +43,14 @@ "laion/CLIP-ViT-B-32-laion2B-s34B-b79K", "TIGER-Lab/VLM2Vec-LoRA", "TIGER-Lab/VLM2Vec-Full", + "Salesforce/blip-itm-base-coco", + "Salesforce/blip-itm-large-coco", + "Salesforce/blip-itm-base-flickr", + "Salesforce/blip-itm-large-flickr", + "EVA02-CLIP-B-16", + "EVA02-CLIP-L-14", + "EVA02-CLIP-bigE-14", + "EVA02-CLIP-bigE-14-plus", # "embed-english-v3.0-v", # not feasible to run due to the 40 images/min constraint ]: model = mteb.get_model(model_name) From 89c7f744d35acdfc2c3f2b66fd0b10b11e426f0e Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Fri, 1 Nov 2024 19:20:33 +0000 Subject: [PATCH 6/6] fix --- .../evaluators/Image/Any2AnyMultiChoiceEvaluator.py | 2 +- .../evaluators/Image/Any2AnyRetrievalEvaluator.py | 2 +- .../evaluators/Image/ClassificationEvaluator.py | 1 - .../Image/ImageTextPairClassificationEvaluator.py | 2 ++ .../evaluators/Image/ZeroshotClassificationEvaluator.py | 4 ++-- mteb/models/cohere_v.py | 9 +++++---- mteb/models/evaclip_models.py | 2 +- 7 files changed, 12 insertions(+), 10 deletions(-) diff --git a/mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py b/mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py index 953a5e3c36..5fdbb112f3 100644 --- a/mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py +++ b/mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py @@ -4,6 +4,7 @@ import io import json import logging +import math import os from collections import defaultdict from typing import Any @@ -11,7 +12,6 @@ import numpy as np import pytrec_eval import torch -import math from datasets import Dataset from PIL import Image from torch.utils.data import DataLoader diff --git a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py index 257a1c939c..a321979d26 100644 --- a/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py +++ b/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py @@ -4,8 +4,8 @@ import io import json import logging -import os import math +import os from collections import defaultdict from typing import Any diff --git a/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py index 647853cc32..2f60b5c690 100644 --- a/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ClassificationEvaluator.py @@ -7,7 +7,6 @@ import numpy as np import torch -import math from sklearn.linear_model import LogisticRegression from sklearn.metrics import ( accuracy_score, diff --git a/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py index edaf779f4c..7e3d84bb87 100644 --- a/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ImageTextPairClassificationEvaluator.py @@ -1,6 +1,8 @@ from __future__ import annotations import logging +import math +import os from typing import Any import torch diff --git a/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py b/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py index 6f4d9ae623..0b3b7f8f67 100644 --- a/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py +++ b/mteb/evaluation/evaluators/Image/ZeroshotClassificationEvaluator.py @@ -1,11 +1,11 @@ from __future__ import annotations import logging +import math +import os from typing import Any import torch -import math -import os from sklearn import metrics from torch.utils.data import DataLoader from torchvision import transforms diff --git a/mteb/models/cohere_v.py b/mteb/models/cohere_v.py index cf98f7dc3c..d53fd662e8 100644 --- a/mteb/models/cohere_v.py +++ b/mteb/models/cohere_v.py @@ -1,5 +1,9 @@ from __future__ import annotations +import base64 +import io +import os +import time from functools import partial from typing import Any @@ -8,11 +12,8 @@ from torch.utils.data import DataLoader from torchvision import transforms from tqdm import tqdm -import os -import io -import base64 + import mteb -import time from mteb.model_meta import ModelMeta api_key = os.getenv("COHERE_API_KEY") diff --git a/mteb/models/evaclip_models.py b/mteb/models/evaclip_models.py index 1e123096ca..015c965c07 100644 --- a/mteb/models/evaclip_models.py +++ b/mteb/models/evaclip_models.py @@ -13,8 +13,8 @@ def evaclip_loader(**kwargs): try: - import sys import os + import sys sys.path.insert(0, os.path.join(os.getcwd(), "EVA/EVA-CLIP/rei"))