Skip to content

[mieb] new version with downsampled train split to 32 per class #1327

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
isaac-chung merged 13 commits into from
Oct 28, 2024
Merged

[mieb] new version with downsampled train split to 32 per class #1327

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
865f7a8
new version with downsampled train split to 32 per class
isaac-chung Oct 26, 2024
46632a5
force load truncated image file
isaac-chung Oct 27, 2024
0839aee
make lint
isaac-chung Oct 27, 2024
3d55f2c
add open clip models
isaac-chung Oct 27, 2024
aa6c309
Merge branch 'mieb' into open-clip-models
gowitheflow-1998 Oct 28, 2024
4e33b7f
Update __init__.py
gowitheflow-1998 Oct 28, 2024
5a9a8e5
lint
gowitheflow-1998 Oct 28, 2024
a98951d
fix model overview
gowitheflow-1998 Oct 28, 2024
29db476
Merge branch 'open-clip-models' of https://github.com/embeddings-benc…
gowitheflow-1998 Oct 28, 2024
541e731
fix ImageCLS undersample; run birdsnap
gowitheflow-1998 Oct 28, 2024
ed7ac94
Merge branch 'mieb' into birdsnap-downsample-train-split
isaac-chung Oct 28, 2024
97ac09d
make lint
isaac-chung Oct 28, 2024
b40cb44
make lint
isaac-chung Oct 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions mteb/abstasks/Image/AbsTaskImageClassification.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from typing import Any

import numpy as np
from PIL import ImageFile

from mteb.abstasks.TaskMetadata import HFSubset

Expand All @@ -16,6 +17,8 @@
)
from ..AbsTask import AbsTask, ScoresDict

ImageFile.LOAD_TRUNCATED_IMAGES = True

logger = logging.getLogger(__name__)


Expand Down Expand Up @@ -133,7 +136,7 @@ def _evaluate_subset(
"=" * 10 + f" Experiment {i+1}/{self.n_experiments} " + "=" * 10
)
# Bootstrap `self.samples_per_label` samples per label for each split
X_sampled, y_sampled, idxs = self._undersample_data(
undersampled_train, idxs = self._undersample_data(
train_split,
self.label_column_name,
self.samples_per_label,
Expand All @@ -142,8 +145,7 @@ def _evaluate_subset(

if self.method == "kNN":
evaluator = ImagekNNClassificationEvaluator(
X_sampled,
y_sampled,
undersampled_train,
eval_split,
self.image_column_name,
self.label_column_name,
Expand All @@ -153,8 +155,7 @@ def _evaluate_subset(
)
elif self.method == "kNN-pytorch":
evaluator = ImagekNNClassificationEvaluatorPytorch(
X_sampled,
y_sampled,
undersampled_train,
eval_split,
self.image_column_name,
self.label_column_name,
Expand All @@ -164,8 +165,7 @@ def _evaluate_subset(
)
elif self.method == "logReg":
evaluator = ImagelogRegClassificationEvaluator(
X_sampled,
y_sampled,
undersampled_train,
eval_split,
self.image_column_name,
self.label_column_name,
Expand Down Expand Up @@ -199,15 +199,15 @@ def _undersample_data(
label_counter = defaultdict(int)
selected_indices = []

labels = dataset_split[label_column_name]
for i in idxs:
label = dataset_split[i][label_column_name]
label = labels[i]
if label_counter[label] < samples_per_label:
selected_indices.append(i)
label_counter[label] += 1

undersampled_dataset = dataset_split.select(selected_indices)
return (
undersampled_dataset[self.image_column_name],
undersampled_dataset[self.label_column_name],
undersampled_dataset,
idxs,
)
59 changes: 22 additions & 37 deletions mteb/evaluation/evaluators/Image/ClassificationEvaluator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
from __future__ import annotations

import logging
import math
import os
from typing import Any

import numpy as np
import torch
from datasets import Dataset
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import (
accuracy_score,
Expand Down Expand Up @@ -55,8 +56,7 @@ def custom_collate_fn(batch):
class ImagekNNClassificationEvaluator(Evaluator):
def __init__(
self,
images_train,
y_train,
dataset_train,
dataset_test,
image_column_name,
label_column_name,
Expand All @@ -69,17 +69,13 @@ def __init__(
super().__init__(**kwargs)

if limit is not None:
images_train = images_train[:limit]
y_train = y_train[:limit]
dataset_test = dataset_test[:limit]
dataset_train = dataset_train.select(list(range(limit)))

self.images_train = images_train
self.y_train = y_train
self.dataset_train = ImageDataset(
Dataset.from_dict({"image": images_train, "label": y_train}),
image_column_name=image_column_name,
transform=transform,
dataset_train, image_column_name=image_column_name, transform=transform
)
self.y_train = dataset_train[label_column_name]

self.dataset_test = ImageDataset(
dataset_test, image_column_name=image_column_name, transform=transform
)
Expand All @@ -102,7 +98,7 @@ def __call__(self, model, test_cache=None):
batch_size=self.encode_kwargs["batch_size"],
shuffle=False,
collate_fn=custom_collate_fn,
num_workers=16,
num_workers=min(math.floor(os.cpu_count() / 2), 16),
)
X_train = model.get_image_embeddings(
dataloader_train, batch_size=self.encode_kwargs["batch_size"]
Expand All @@ -111,7 +107,7 @@ def __call__(self, model, test_cache=None):
self.dataset_test,
batch_size=self.encode_kwargs["batch_size"],
shuffle=False,
num_workers=16,
num_workers=min(math.floor(os.cpu_count() / 2), 16),
)
if test_cache is None:
X_test = model.get_image_embeddings(
Expand Down Expand Up @@ -145,8 +141,7 @@ def __call__(self, model, test_cache=None):
class ImagekNNClassificationEvaluatorPytorch(Evaluator):
def __init__(
self,
images_train,
y_train,
dataset_train,
dataset_test,
image_column_name,
label_column_name,
Expand All @@ -158,17 +153,13 @@ def __init__(
):
super().__init__(**kwargs)
if limit is not None:
images_train = images_train[:limit]
y_train = y_train[:limit]
dataset_test = dataset_test[:limit]
dataset_train = dataset_train.select(list(range(limit)))

self.images_train = images_train
self.dataset_train = ImageDataset(
Dataset.from_dict({"image": images_train, "label": y_train}),
image_column_name=image_column_name,
transform=transform,
dataset_train, image_column_name=image_column_name, transform=transform
)
self.y_train = y_train
self.y_train = dataset_train[label_column_name]

self.dataset_test = ImageDataset(
dataset_test, image_column_name=image_column_name, transform=transform
)
Expand All @@ -192,7 +183,7 @@ def __call__(self, model: Encoder, test_cache=None):
batch_size=self.encode_kwargs["batch_size"],
shuffle=False,
collate_fn=custom_collate_fn,
num_workers=16,
num_workers=min(math.floor(os.cpu_count() / 2), 16),
)
X_train = model.get_image_embeddings(
dataloader_train, batch_size=self.encode_kwargs["batch_size"]
Expand All @@ -202,7 +193,7 @@ def __call__(self, model: Encoder, test_cache=None):
self.dataset_test,
batch_size=self.encode_kwargs["batch_size"],
shuffle=False,
num_workers=16,
num_workers=min(math.floor(os.cpu_count() / 2), 16),
)
if test_cache is None:
X_test = model.get_image_embeddings(
Expand Down Expand Up @@ -311,8 +302,7 @@ def _dot_score(a: Tensor, b: Tensor):
class ImagelogRegClassificationEvaluator(Evaluator):
def __init__(
self,
images_train,
y_train,
dataset_train,
dataset_test,
image_column_name,
label_column_name,
Expand All @@ -329,17 +319,12 @@ def __init__(
self.encode_kwargs["batch_size"] = 32

if limit is not None:
images_train = images_train[:limit]
y_train = y_train[:limit]
dataset_test = dataset_test[:limit]
dataset_train = dataset_train.select(list(range(limit)))

self.images_train = images_train
self.y_train = y_train
self.dataset_train = ImageDataset(
Dataset.from_dict({"image": images_train, "label": y_train}),
image_column_name=image_column_name,
transform=transform,
dataset_train, image_column_name=image_column_name, transform=transform
)
self.y_train = dataset_train[label_column_name]
self.dataset_test = ImageDataset(
dataset_test, image_column_name=image_column_name, transform=transform
)
Expand All @@ -361,7 +346,7 @@ def __call__(self, model, test_cache=None):
batch_size=self.encode_kwargs["batch_size"],
shuffle=False,
collate_fn=custom_collate_fn,
num_workers=16,
num_workers=min(math.floor(os.cpu_count() / 2), 16),
)
X_train = model.get_image_embeddings(
dataloader_train, batch_size=self.encode_kwargs["batch_size"]
Expand All @@ -371,7 +356,7 @@ def __call__(self, model, test_cache=None):
batch_size=self.encode_kwargs["batch_size"],
shuffle=False,
collate_fn=custom_collate_fn,
num_workers=16,
num_workers=min(math.floor(os.cpu_count() / 2), 16),
)
if test_cache is None:
X_test = model.get_image_embeddings(
Expand Down
Loading