From 5e3b45f54a1c121033dc31a461aae573b1fc376c Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Wed, 16 Oct 2024 12:59:03 +0000 Subject: [PATCH 1/4] add moco models first try --- mteb/models/__init__.py | 2 + mteb/models/moco_models.py | 133 +++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 mteb/models/moco_models.py diff --git a/mteb/models/__init__.py b/mteb/models/__init__.py index 33b83f9a11..d8512a5a23 100644 --- a/mteb/models/__init__.py +++ b/mteb/models/__init__.py @@ -25,6 +25,7 @@ gte_models, jina_clip, llm2vec_models, + moco_models, mxbai_models, nomic_models, nomic_models_vision, @@ -147,6 +148,7 @@ def model_meta_from_sentence_transformers(model: SentenceTransformer) -> ModelMe gte_models, jina_clip, llm2vec_models, + moco_models, mxbai_models, nomic_models, nomic_models_vision, diff --git a/mteb/models/moco_models.py b/mteb/models/moco_models.py new file mode 100644 index 0000000000..2c4c1c511e --- /dev/null +++ b/mteb/models/moco_models.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +from functools import partial +from typing import Any + +import torch +from PIL import Image +from torch.utils.data import DataLoader +from tqdm import tqdm +from transformers import ViTModel, AutoProcessor + +from mteb.model_meta import ModelMeta + + +class MOCOv3Wrapper: + """A wrapper class for MOCOv3 models that supports image encoding. + Text encoding and text-image fusion are not supported. + """ + def __init__( + self, + model_name: str = "nyu-visionx/moco-v3-vit-b", + device: str = "cuda" if torch.cuda.is_available() else "cpu", + **kwargs: Any, + ): + self.model_name = model_name + self.device = device + self.model = ViTModel.from_pretrained(model_name, trust_remote_code=True).to( + self.device + ) + self.processor = AutoProcessor.from_pretrained("laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K") + + @staticmethod + def get_text_embeddings(texts: list[str], batch_size: int = 32): + raise ValueError("MOCO models only support image encoding.") + + def get_image_embeddings( + self, + images: list[Image.Image] | DataLoader, + batch_size: int = 32, + pooling="cls", + ): + all_image_embeddings = [] + + if isinstance(images, DataLoader): + with torch.no_grad(): + for batch in tqdm(images): + inputs = self.processor(images=batch, return_tensors="pt") + inputs = {k: v.to(self.device) for k, v in inputs.items()} + image_outputs = self.model(**inputs) + features = image_outputs.last_hidden_state + if pooling == "cls": + features = features[:, 0, :] # TODO: confirm best practice + elif pooling == "mean": + features = features.mean(dim=1) + else: + raise ValueError( + "Pooling methods not implemented. Use cls or mean." + ) + all_image_embeddings.append(features.cpu()) + else: + with torch.no_grad(): + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + inputs = self.processor(images=batch_images, return_tensors="pt") + inputs = {k: v.to(self.device) for k, v in inputs.items()} + image_outputs = self.model(**inputs) + features = image_outputs.last_hidden_state + if pooling == "cls": + features = features[:, 0, :] + elif pooling == "mean": + features = features.mean(dim=1) + else: + raise ValueError( + "Pooling methods not implemented. Use cls or mean." + ) + all_image_embeddings.append(features.cpu()) + + all_image_embeddings = torch.cat(all_image_embeddings, dim=0) + return all_image_embeddings + + @staticmethod + def calculate_probs(text_embeddings, image_embeddings): + raise ValueError("MOCO models only support image encoding.") + + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] | DataLoader = None, + fusion_mode="sum", + batch_size: int = 32, + ): + if texts is None and images is None: + raise ValueError("images must be provided for MOCO models") + + text_embeddings = None + image_embeddings = None + + if texts is not None: + text_embeddings = self.get_text_embeddings(texts, batch_size) + + if images is not None: + image_embeddings = self.get_image_embeddings(images, batch_size) + + if text_embeddings is not None and image_embeddings is not None: + raise ValueError("MOCO models only support image encoding.") + elif text_embeddings is not None: + return text_embeddings + elif image_embeddings is not None: + return image_embeddings + +mocov3_vit_base = ModelMeta( + loader=partial( + MOCOv3Wrapper, + model_name="nyu-visionx/moco-v3-vit-b", + ), + name="nyu-visionx/moco-v3-vit-b", + languages=["eng_Latn"], + open_source=True, + revision="7d091cd70772c5c0ecf7f00b5f12ca609a99d69d", + release_date="2024-06-03", +) + +mocov3_vit_large = ModelMeta( + loader=partial( + MOCOv3Wrapper, + model_name="nyu-visionx/moco-v3-vit-l", + ), + name="nyu-visionx/moco-v3-vit-l", + languages=["eng_Latn"], + open_source=True, + revision="7bf75358d616f39b9716148bf4e3425f3bd35b47", + release_date="2024-06-03", +) From d2f6cc01f38d95368c38a1a922cd6c746d8c0020 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Wed, 16 Oct 2024 19:38:08 +0000 Subject: [PATCH 2/4] add as a timm model --- mteb/models/moco_models.py | 182 +++++++++--------- .../OxfordFlowersClassification.json | 48 +++++ .../model_meta.json | 1 + 3 files changed, 135 insertions(+), 96 deletions(-) create mode 100644 results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/OxfordFlowersClassification.json create mode 100644 results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/model_meta.json diff --git a/mteb/models/moco_models.py b/mteb/models/moco_models.py index 2c4c1c511e..4f695d3d95 100644 --- a/mteb/models/moco_models.py +++ b/mteb/models/moco_models.py @@ -7,110 +7,100 @@ from PIL import Image from torch.utils.data import DataLoader from tqdm import tqdm -from transformers import ViTModel, AutoProcessor from mteb.model_meta import ModelMeta -class MOCOv3Wrapper: - """A wrapper class for MOCOv3 models that supports image encoding. - Text encoding and text-image fusion are not supported. - """ - def __init__( - self, - model_name: str = "nyu-visionx/moco-v3-vit-b", - device: str = "cuda" if torch.cuda.is_available() else "cpu", - **kwargs: Any, - ): - self.model_name = model_name - self.device = device - self.model = ViTModel.from_pretrained(model_name, trust_remote_code=True).to( - self.device - ) - self.processor = AutoProcessor.from_pretrained("laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K") - - @staticmethod - def get_text_embeddings(texts: list[str], batch_size: int = 32): - raise ValueError("MOCO models only support image encoding.") +def mocov3_loader(**kwargs): + try: + import timm + except ImportError: + raise ImportError("Please install `pip install timm` to use MOCOv3 models.") - def get_image_embeddings( - self, - images: list[Image.Image] | DataLoader, - batch_size: int = 32, - pooling="cls", - ): - all_image_embeddings = [] - - if isinstance(images, DataLoader): - with torch.no_grad(): - for batch in tqdm(images): - inputs = self.processor(images=batch, return_tensors="pt") - inputs = {k: v.to(self.device) for k, v in inputs.items()} - image_outputs = self.model(**inputs) - features = image_outputs.last_hidden_state - if pooling == "cls": - features = features[:, 0, :] # TODO: confirm best practice - elif pooling == "mean": - features = features.mean(dim=1) - else: - raise ValueError( - "Pooling methods not implemented. Use cls or mean." - ) - all_image_embeddings.append(features.cpu()) - else: - with torch.no_grad(): - for i in tqdm(range(0, len(images), batch_size)): - batch_images = images[i : i + batch_size] - inputs = self.processor(images=batch_images, return_tensors="pt") - inputs = {k: v.to(self.device) for k, v in inputs.items()} - image_outputs = self.model(**inputs) - features = image_outputs.last_hidden_state - if pooling == "cls": - features = features[:, 0, :] - elif pooling == "mean": - features = features.mean(dim=1) - else: - raise ValueError( - "Pooling methods not implemented. Use cls or mean." - ) - all_image_embeddings.append(features.cpu()) - - all_image_embeddings = torch.cat(all_image_embeddings, dim=0) - return all_image_embeddings - - @staticmethod - def calculate_probs(text_embeddings, image_embeddings): - raise ValueError("MOCO models only support image encoding.") - - def get_fused_embeddings( - self, - texts: list[str] = None, - images: list[Image.Image] | DataLoader = None, - fusion_mode="sum", - batch_size: int = 32, - ): - if texts is None and images is None: - raise ValueError("images must be provided for MOCO models") - - text_embeddings = None - image_embeddings = None - - if texts is not None: - text_embeddings = self.get_text_embeddings(texts, batch_size) - - if images is not None: - image_embeddings = self.get_image_embeddings(images, batch_size) - - if text_embeddings is not None and image_embeddings is not None: + class MOCOv3Wrapper: + """A wrapper class for MOCOv3 models that supports image encoding. + Text encoding and text-image fusion are not supported. + """ + def __init__( + self, + model_name: str = "nyu-visionx/moco-v3-vit-b", + device: str = "cuda" if torch.cuda.is_available() else "cpu", + **kwargs: Any, + ): + self.model_name = model_name + self.device = device + model = timm.create_model('vit_base_patch16_224', pretrained=True, num_classes=0, pretrained_cfg_overlay=dict(hf_hub_id=model_name)) + + self.model = model.eval() + + # get model specific transforms (normalization, resize) + data_config = timm.data.resolve_model_data_config(self.model) + self.processor = timm.data.create_transform(**data_config, is_training=False) + + + @staticmethod + def get_text_embeddings(texts: list[str], batch_size: int = 32): raise ValueError("MOCO models only support image encoding.") - elif text_embeddings is not None: - return text_embeddings - elif image_embeddings is not None: - return image_embeddings + + def get_image_embeddings( + self, + images: list[Image.Image] | DataLoader, + batch_size: int = 32, + ): + all_image_embeddings = [] + + if isinstance(images, DataLoader): + import torchvision.transforms.functional as F + with torch.no_grad(): + for batch in tqdm(images): + inputs = torch.vstack([self.processor(F.to_pil_image(b.to("cpu"))).unsqueeze(0) for b in batch]) + output = self.model(inputs) # output is (batch_size, num_features) shaped tensor + all_image_embeddings.append(output) + else: + with torch.no_grad(): + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + output = self.model(self.processor(batch_images)) # output is (batch_size, num_features) shaped tensor + all_image_embeddings.append(output) + + all_image_embeddings = torch.cat(all_image_embeddings, dim=0) + return all_image_embeddings + + @staticmethod + def calculate_probs(text_embeddings, image_embeddings): + raise ValueError("MOCO models only support image encoding.") + + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] | DataLoader = None, + fusion_mode="sum", + batch_size: int = 32, + ): + if texts is None and images is None: + raise ValueError("images must be provided for MOCO models") + + text_embeddings = None + image_embeddings = None + + if texts is not None: + text_embeddings = self.get_text_embeddings(texts, batch_size) + + if images is not None: + image_embeddings = self.get_image_embeddings(images, batch_size) + + if text_embeddings is not None and image_embeddings is not None: + raise ValueError("MOCO models only support image encoding.") + elif text_embeddings is not None: + return text_embeddings + elif image_embeddings is not None: + return image_embeddings + + return MOCOv3Wrapper(**kwargs) mocov3_vit_base = ModelMeta( loader=partial( - MOCOv3Wrapper, + mocov3_loader, model_name="nyu-visionx/moco-v3-vit-b", ), name="nyu-visionx/moco-v3-vit-b", @@ -122,7 +112,7 @@ def get_fused_embeddings( mocov3_vit_large = ModelMeta( loader=partial( - MOCOv3Wrapper, + mocov3_loader, model_name="nyu-visionx/moco-v3-vit-l", ), name="nyu-visionx/moco-v3-vit-l", diff --git a/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/OxfordFlowersClassification.json b/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/OxfordFlowersClassification.json new file mode 100644 index 0000000000..f4ea1b602c --- /dev/null +++ b/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/OxfordFlowersClassification.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "a37b1891609c0376fa81eced756e7863e1bd873b", + "evaluation_time": 347.9395191669464, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "accuracy": 0.8880392156862745, + "f1": 0.8858313070145011, + "f1_weighted": 0.8855993382917493, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8880392156862745, + "scores_per_experiment": [ + { + "accuracy": 0.8921568627450981, + "f1": 0.8908725994288734, + "f1_weighted": 0.890707976140322 + }, + { + "accuracy": 0.888235294117647, + "f1": 0.8862588054389066, + "f1_weighted": 0.8857789823937094 + }, + { + "accuracy": 0.8833333333333333, + "f1": 0.8807271651302992, + "f1_weighted": 0.8807519823317017 + }, + { + "accuracy": 0.884313725490196, + "f1": 0.8818842242107174, + "f1_weighted": 0.8815393805409137 + }, + { + "accuracy": 0.8921568627450981, + "f1": 0.8894137408637092, + "f1_weighted": 0.8892183700520992 + } + ] + } + ] + }, + "task_name": "OxfordFlowersClassification" +} \ No newline at end of file diff --git a/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/model_meta.json b/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/model_meta.json new file mode 100644 index 0000000000..d97b884a3b --- /dev/null +++ b/results-mieb/nyu-visionx__moco-v3-vit-b/7d091cd70772c5c0ecf7f00b5f12ca609a99d69d/model_meta.json @@ -0,0 +1 @@ +{"name": "nyu-visionx/moco-v3-vit-b", "revision": "7d091cd70772c5c0ecf7f00b5f12ca609a99d69d", "release_date": "2024-06-03", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "mocov3_loader"} \ No newline at end of file From 9c0d8366969c52cf452030c746bb4e501c7887eb Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Wed, 16 Oct 2024 20:22:02 +0000 Subject: [PATCH 3/4] add large model results --- mteb/models/moco_models.py | 5 +- .../OxfordFlowersClassification.json | 48 +++++++++++++++++++ .../model_meta.json | 1 + 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/OxfordFlowersClassification.json create mode 100644 results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/model_meta.json diff --git a/mteb/models/moco_models.py b/mteb/models/moco_models.py index 4f695d3d95..35956432a8 100644 --- a/mteb/models/moco_models.py +++ b/mteb/models/moco_models.py @@ -29,7 +29,10 @@ def __init__( ): self.model_name = model_name self.device = device - model = timm.create_model('vit_base_patch16_224', pretrained=True, num_classes=0, pretrained_cfg_overlay=dict(hf_hub_id=model_name)) + name = 'vit_base_patch16_224' + if 'vit-l' in model_name: + name = 'vit_large_patch16_224' + model = timm.create_model(name, pretrained=True, num_classes=0, pretrained_cfg_overlay=dict(hf_hub_id=model_name)) self.model = model.eval() diff --git a/results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/OxfordFlowersClassification.json b/results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/OxfordFlowersClassification.json new file mode 100644 index 0000000000..a1fb3e1e31 --- /dev/null +++ b/results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/OxfordFlowersClassification.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "a37b1891609c0376fa81eced756e7863e1bd873b", + "evaluation_time": 780.7572722434998, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "accuracy": 0.8990196078431373, + "f1": 0.8976575858316652, + "f1_weighted": 0.8973426503552833, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8990196078431373, + "scores_per_experiment": [ + { + "accuracy": 0.8960784313725491, + "f1": 0.8954744443526484, + "f1_weighted": 0.8949013011291541 + }, + { + "accuracy": 0.9009803921568628, + "f1": 0.89924527322374, + "f1_weighted": 0.89919388345273 + }, + { + "accuracy": 0.8960784313725491, + "f1": 0.8937528098526334, + "f1_weighted": 0.8936883759623736 + }, + { + "accuracy": 0.8990196078431373, + "f1": 0.8985132758082764, + "f1_weighted": 0.8977617544247366 + }, + { + "accuracy": 0.9029411764705882, + "f1": 0.901302125921028, + "f1_weighted": 0.901167936807422 + } + ] + } + ] + }, + "task_name": "OxfordFlowersClassification" +} \ No newline at end of file diff --git a/results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/model_meta.json b/results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/model_meta.json new file mode 100644 index 0000000000..b9e1a399c2 --- /dev/null +++ b/results-mieb/nyu-visionx__moco-v3-vit-l/7bf75358d616f39b9716148bf4e3425f3bd35b47/model_meta.json @@ -0,0 +1 @@ +{"name": "nyu-visionx/moco-v3-vit-l", "revision": "7bf75358d616f39b9716148bf4e3425f3bd35b47", "release_date": "2024-06-03", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "mocov3_loader"} \ No newline at end of file From a691aa71c76ba2f4d175be35c2e5acf2760052cd Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Wed, 16 Oct 2024 20:40:55 +0000 Subject: [PATCH 4/4] make lint --- .../Image/Any2TextMultipleChoiceEvaluator.py | 2 +- mteb/models/moco_models.py | 42 +++++++++++++------ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/mteb/evaluation/evaluators/Image/Any2TextMultipleChoiceEvaluator.py b/mteb/evaluation/evaluators/Image/Any2TextMultipleChoiceEvaluator.py index f682225ba5..a93714e770 100644 --- a/mteb/evaluation/evaluators/Image/Any2TextMultipleChoiceEvaluator.py +++ b/mteb/evaluation/evaluators/Image/Any2TextMultipleChoiceEvaluator.py @@ -62,7 +62,7 @@ def __call__( encode_kwargs["batch_size"] = 64 label_list = list( - set([x for n in self.dataset[self.choices_column_name] for x in n]) + {x for n in self.dataset[self.choices_column_name] for x in n} ) label_embeddings = model.get_text_embeddings(label_list) label_embedding_dict = {} diff --git a/mteb/models/moco_models.py b/mteb/models/moco_models.py index 35956432a8..3fa7dfe203 100644 --- a/mteb/models/moco_models.py +++ b/mteb/models/moco_models.py @@ -16,11 +16,12 @@ def mocov3_loader(**kwargs): import timm except ImportError: raise ImportError("Please install `pip install timm` to use MOCOv3 models.") - + class MOCOv3Wrapper: """A wrapper class for MOCOv3 models that supports image encoding. Text encoding and text-image fusion are not supported. """ + def __init__( self, model_name: str = "nyu-visionx/moco-v3-vit-b", @@ -29,22 +30,28 @@ def __init__( ): self.model_name = model_name self.device = device - name = 'vit_base_patch16_224' - if 'vit-l' in model_name: - name = 'vit_large_patch16_224' - model = timm.create_model(name, pretrained=True, num_classes=0, pretrained_cfg_overlay=dict(hf_hub_id=model_name)) + name = "vit_base_patch16_224" + if "vit-l" in model_name: + name = "vit_large_patch16_224" + model = timm.create_model( + name, + pretrained=True, + num_classes=0, + pretrained_cfg_overlay={"hf_hub_id": model_name}, + ) self.model = model.eval() # get model specific transforms (normalization, resize) data_config = timm.data.resolve_model_data_config(self.model) - self.processor = timm.data.create_transform(**data_config, is_training=False) - + self.processor = timm.data.create_transform( + **data_config, is_training=False + ) @staticmethod def get_text_embeddings(texts: list[str], batch_size: int = 32): raise ValueError("MOCO models only support image encoding.") - + def get_image_embeddings( self, images: list[Image.Image] | DataLoader, @@ -54,16 +61,26 @@ def get_image_embeddings( if isinstance(images, DataLoader): import torchvision.transforms.functional as F + with torch.no_grad(): for batch in tqdm(images): - inputs = torch.vstack([self.processor(F.to_pil_image(b.to("cpu"))).unsqueeze(0) for b in batch]) - output = self.model(inputs) # output is (batch_size, num_features) shaped tensor + inputs = torch.vstack( + [ + self.processor(F.to_pil_image(b.to("cpu"))).unsqueeze(0) + for b in batch + ] + ) + output = self.model( + inputs + ) # output is (batch_size, num_features) shaped tensor all_image_embeddings.append(output) else: with torch.no_grad(): for i in tqdm(range(0, len(images), batch_size)): batch_images = images[i : i + batch_size] - output = self.model(self.processor(batch_images)) # output is (batch_size, num_features) shaped tensor + output = self.model( + self.processor(batch_images) + ) # output is (batch_size, num_features) shaped tensor all_image_embeddings.append(output) all_image_embeddings = torch.cat(all_image_embeddings, dim=0) @@ -98,9 +115,10 @@ def get_fused_embeddings( return text_embeddings elif image_embeddings is not None: return image_embeddings - + return MOCOv3Wrapper(**kwargs) + mocov3_vit_base = ModelMeta( loader=partial( mocov3_loader,