ci: Removed 3.8 dependency

.github/workflows/lint.yml

@@ -15,7 +15,7 @@ jobs:
 
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.9"
           cache: "pip"
 
       - name: Install dependencies

.github/workflows/mmteb.yml

@@ -16,7 +16,7 @@ jobs:
 
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.9"
           cache: "pip"
 
       - name: Install dependencies
@@ -38,7 +38,7 @@ jobs:
 
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.9"
           cache: "pip"
 
       - name: Install dependencies

.github/workflows/test.yml

@@ -16,11 +16,11 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest] #, macos-latest, windows-latest]
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
         include:
           # Add Windows with Python 3.8 only to avoid tests taking too long
           - os: windows-latest
-            python-version: "3.8"
+            python-version: "3.9"
 
     steps:
       - uses: actions/checkout@v3

mteb/abstasks/AbsTask.py

@@ -3,7 +3,8 @@
 import logging
 import random
 from abc import ABC, abstractmethod
-from typing import Any, Dict, Sequence, TypedDict
+from collections.abc import Sequence
+from typing import Any, TypedDict
 
 import datasets
 import numpy as np
@@ -19,7 +20,7 @@
 
 logger = logging.getLogger(__name__)
 
-ScoresDict = Dict[str, Any]
+ScoresDict = dict[str, Any]
 # ^ e.g {'main_score': 0.5, 'hf_subset': 'en-de', 'languages': ['eng-Latn', 'deu-Latn']}
 
 

mteb/abstasks/AbsTaskClusteringFast.py

@@ -4,7 +4,7 @@
 import logging
 import random
 from collections import Counter, defaultdict
-from typing import Any, Dict
+from typing import Any
 
 import numpy as np
 import sklearn
@@ -21,7 +21,7 @@
 logger = logging.getLogger(__name__)
 
 
-MultilingualDataset = Dict[HFSubset, DatasetDict]
+MultilingualDataset = dict[HFSubset, DatasetDict]
 
 
 def evaluate_clustering_bootstrapped(

mteb/abstasks/AbsTaskInstructionRetrieval.py

@@ -248,12 +248,12 @@ class AbsTaskInstructionRetrieval(AbsTask):
         instruction: A relevant document will provide the projected or actual date of completion of the project, its estimated or actual total cost, or the estimated or ongoing electrical output of the finished project. Discussions of the social, political, or ecological impact of the project are not relevant.
 
     Child-classes must implement the following properties:
-    self.corpus = Dict[corpus_id, Dict[str, str]] #id => dict with document datas like title and text
-    self.queries = Dict[query_id, str] #id => query
-    self.relevant_docs = Dict[query_id, Dict[corpus_id, int]]
-    self.og_instructions = Dict[str, str] query => original instruction
-    self.changed_instructions = Dict[str, str] query => changed instruction
-    self.top_ranked = Dict[query_id, List[corpus_id]] #id => list of top ranked document ids
+    self.corpus = dict[corpus_id, dict[str, str]] #id => dict with document datas like title and text
+    self.queries = dict[query_id, str] #id => query
+    self.relevant_docs = dict[query_id, dict[corpus_id, int]]
+    self.og_instructions = dict[str, str] query => original instruction
+    self.changed_instructions = dict[str, str] query => changed instruction
+    self.top_ranked = dict[query_id, list[corpus_id]] #id => list of top ranked document ids
 
     See https://arxiv.org/abs/2403.15246 for more details
     """

mteb/abstasks/AbsTaskRetrieval.py

@@ -219,8 +219,8 @@ class AbsTaskRetrieval(AbsTask):
         Semantically, it should contain dict[split_name, dict[sample_id, dict[str, str]]]
         E.g. {"test": {"document_one": {"_id": "d1", "title": "title", "text": "text"}}}
 
-    self.queries: dict[str, dict[str, Union[str, List[str]]]]
-        Semantically, it should contain dict[split_name, dict[sample_id, str]] or dict[split_name, dict[sample_id, List[str]]] for conversations
+    self.queries: dict[str, dict[str, Union[str, list[str]]]]
+        Semantically, it should contain dict[split_name, dict[sample_id, str]] or dict[split_name, dict[sample_id, list[str]]] for conversations
         E.g. {"test": {"q1": "query"}}
         or {"test": {"q1": ["turn1", "turn2", "turn3"]}}
 

mteb/abstasks/TaskMetadata.py

@@ -1,11 +1,12 @@
 from __future__ import annotations
 
 import logging
+from collections.abc import Mapping
 from datetime import date
-from typing import Any, Dict, List, Mapping, Union
+from typing import Annotated, Any, Union
 
 from pydantic import AnyUrl, BaseModel, BeforeValidator, TypeAdapter, field_validator
-from typing_extensions import Annotated, Literal
+from typing_extensions import Literal
 
 from ..languages import (
     ISO_LANGUAGE_SCRIPT,
@@ -114,7 +115,7 @@
 SPLIT_NAME = str
 HFSubset = str
 LANGUAGES = Union[
-    List[ISO_LANGUAGE_SCRIPT], Mapping[HFSubset, List[ISO_LANGUAGE_SCRIPT]]
+    list[ISO_LANGUAGE_SCRIPT], Mapping[HFSubset, list[ISO_LANGUAGE_SCRIPT]]
 ]
 
 PROGRAMMING_LANGS = [
@@ -162,7 +163,7 @@
 )
 
 METRIC_NAME = str
-METRIC_VALUE = Union[int, float, Dict[str, Any]]
+METRIC_VALUE = Union[int, float, dict[str, Any]]
 
 logger = logging.getLogger(__name__)
 

mteb/abstasks/stratification.py

@@ -113,7 +113,7 @@ def _get_most_desired_combination(samples_with_combination):
 
     Parameters
     ----------
-    samples_with_combination : Dict[Combination, List[int]], :code:`(n_combinations)`
+    samples_with_combination : dict[Combination, list[int]], :code:`(n_combinations)`
             map from each label combination present in y to list of sample indexes that have this combination assigned
 
     Returns:
@@ -155,7 +155,7 @@ class IterativeStratification(_BaseKFold):
     order : int, >= 1
         the order of label relationship to take into account when balancing sample distribution across labels
 
-    sample_distribution_per_fold : None or List[float], :code:`(n_splits)`
+    sample_distribution_per_fold : None or list[float], :code:`(n_splits)`
         desired percentage of samples in each of the folds, if None and equal distribution of samples per fold
         is assumed i.e. 1/n_splits for each fold. The value is held in :code:`self.percentage_per_fold`.
 
@@ -195,7 +195,7 @@ def __init__(
     def _prepare_stratification(self, y):
         """Prepares variables for performing stratification
 
-        For the purpose of clarity, the type Combination denotes List[int], :code:`(self.order)` and represents a
+        For the purpose of clarity, the type Combination denotes list[int], :code:`(self.order)` and represents a
         label combination of the order we want to preserve among folds in stratification. The total number of
         combinations present in :code:`(y)` will be denoted as :code:`(n_combinations)`.
 
@@ -208,7 +208,7 @@ def _prepare_stratification(self, y):
         self.desired_samples_per_fold: np.array[Float], :code:`(n_splits)`
             number of samples desired per fold
 
-        self.desired_samples_per_combination_per_fold: Dict[Combination, np.array[Float]], :code:`(n_combinations, n_splits)`
+        self.desired_samples_per_combination_per_fold: dict[Combination, np.array[Float]], :code:`(n_combinations, n_splits)`
             number of samples evidencing each combination desired per each fold
 
         Parameters
@@ -218,22 +218,22 @@ def _prepare_stratification(self, y):
 
         Returns:
         -------
-        rows : List[List[int]], :code:`(n_samples, n_labels)`
+        rows : list[list[int]], :code:`(n_samples, n_labels)`
             list of label indices assigned to each sample
 
-        rows_used : Dict[int, bool], :code:`(n_samples)`
+        rows_used : dict[int, bool], :code:`(n_samples)`
             boolean map from a given sample index to boolean value whether it has been already assigned to a fold or not
 
-        all_combinations :  List[Combination], :code:`(n_combinations)`
+        all_combinations :  list[Combination], :code:`(n_combinations)`
             list of all label combinations of order self.order present in y
 
-        per_row_combinations : List[Combination], :code:`(n_samples)`
+        per_row_combinations : list[Combination], :code:`(n_samples)`
             list of all label combinations of order self.order present in y per row
 
-        samples_with_combination : Dict[Combination, List[int]], :code:`(n_combinations)`
+        samples_with_combination : dict[Combination, list[int]], :code:`(n_combinations)`
             map from each label combination present in y to list of sample indexes that have this combination assigned
 
-        folds: List[List[int]] (n_splits)
+        folds: list[list[int]] (n_splits)
             list of lists to be populated with samples
 
         """
@@ -353,7 +353,7 @@ def _iter_test_indices(self, X, y=None, groups=None):
 
         Yields:
         ------
-        fold : List[int]
+        fold : list[int]
             indexes of test samples for a given fold, yielded for each of the folds
         """
         (

mteb/benchmarks/benchmarks.py

@@ -1,10 +1,10 @@
 from __future__ import annotations
 
+from collections.abc import Sequence
 from dataclasses import dataclass
-from typing import Sequence
+from typing import Annotated
 
 from pydantic import AnyUrl, BeforeValidator, TypeAdapter
-from typing_extensions import Annotated
 
 from mteb.abstasks.AbsTask import AbsTask
 from mteb.overview import get_tasks

mteb/encoder_interface.py

@@ -1,12 +1,13 @@
 from __future__ import annotations
 
+from collections.abc import Sequence
 from enum import Enum
-from typing import Any, Dict, List, Protocol, Sequence, Union, runtime_checkable
+from typing import Any, Protocol, Union, runtime_checkable
 
 import numpy as np
 import torch
 
-Corpus = Union[List[Dict[str, str]], Dict[str, List[str]]]
+Corpus = Union[list[dict[str, str]], dict[str, list[str]]]
 
 
 class PromptType(str, Enum):

mteb/evaluation/MTEB.py

@@ -4,12 +4,13 @@
 import logging
 import os
 import traceback
+from collections.abc import Iterable
 from copy import copy
 from datetime import datetime
 from itertools import chain
 from pathlib import Path
 from time import time
-from typing import Any, Iterable
+from typing import Any
 
 import datasets
 from sentence_transformers import SentenceTransformer

mteb/evaluation/evaluators/RerankingEvaluator.py

@@ -260,7 +260,7 @@ def _encode_candidates_individual(
             is_relevant = [True] * len(positive) + [False] * len(negative)
 
             if isinstance(query, str):
-                # .encoding interface requires List[str] as input
+                # .encoding interface requires list[str] as input
                 query = [query]
             query_emb = np.asarray(encode_queries_func(query, **self.encode_kwargs))
             docs_emb = np.asarray(encode_corpus_func(docs, **self.encode_kwargs))
@@ -345,7 +345,7 @@ def _encode_candidates_miracl_individual(
             docs = list(instance["candidates"])
 
             if isinstance(query, str):
-                # .encoding interface requires List[str] as input
+                # .encoding interface requires list[str] as input
                 query_emb = np.asarray(
                     encode_queries_func([query], **self.encode_kwargs)
                 )
@@ -545,8 +545,8 @@ def ap_score(is_relevant, pred_scores):
         """Computes AP score
 
         Args:
-            is_relevant (`List[bool]` of length `num_pos+num_neg`): True if the document is relevant
-            pred_scores (`List[float]` of length `num_pos+num_neg`): Predicted similarity scores
+            is_relevant (`list[bool]` of length `num_pos+num_neg`): True if the document is relevant
+            pred_scores (`list[float]` of length `num_pos+num_neg`): Predicted similarity scores
 
         Returns:
             ap_score (`float`): AP score

mteb/evaluation/evaluators/model_encode.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import logging
-from typing import Sequence
+from collections.abc import Sequence
 
 import numpy as np
 import torch

mteb/evaluation/evaluators/utils.py

@@ -281,13 +281,16 @@ def rank_score(x: dict[str, float]) -> float:
 def download(url: str, fname: str):
     resp = requests.get(url, stream=True)
     total = int(resp.headers.get("content-length", 0))
-    with open(fname, "wb") as file, tqdm.tqdm(
-        desc=fname,
-        total=total,
-        unit="iB",
-        unit_scale=True,
-        unit_divisor=1024,
-    ) as bar:
+    with (
+        open(fname, "wb") as file,
+        tqdm.tqdm(
+            desc=fname,
+            total=total,
+            unit="iB",
+            unit_scale=True,
+            unit_divisor=1024,
+        ) as bar,
+    ):
         for data in resp.iter_content(chunk_size=1024):
             size = file.write(data)
             bar.update(size)

mteb/load_results/load_results.py

@@ -5,8 +5,8 @@
 import os
 import subprocess
 from collections import defaultdict
+from collections.abc import Sequence
 from pathlib import Path
-from typing import Dict, List, Sequence
 
 from mteb.abstasks.AbsTask import AbsTask
 from mteb.load_results.mteb_results import MTEBResults
@@ -16,7 +16,7 @@
 MODEL_NAME = str
 REVISION = str
 
-RESULTS = Dict[MODEL_NAME, Dict[REVISION, List[MTEBResults]]]
+RESULTS = dict[MODEL_NAME, dict[REVISION, list[MTEBResults]]]
 
 
 def download_of_results(

mteb/model_meta.py

@@ -2,11 +2,10 @@
 
 from datetime import date
 from functools import partial
-from typing import Any, Callable, Literal
+from typing import Annotated, Any, Callable, Literal
 
 from pydantic import BaseModel, BeforeValidator, TypeAdapter
 from sentence_transformers import SentenceTransformer
-from typing_extensions import Annotated
 
 from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode
 

mteb/task_aggregation.py

@@ -2,7 +2,6 @@
 
 import logging
 from collections import defaultdict
-from typing import Dict
 
 import numpy as np
 
@@ -12,7 +11,7 @@
 
 logger = logging.getLogger(__name__)
 
-AGGREGATION = Dict[MODEL_NAME, Dict[REVISION, Dict[str, float]]]
+AGGREGATION = dict[MODEL_NAME, dict[REVISION, dict[str, float]]]
 
 
 def mean(results: RESULTS) -> AGGREGATION:

mteb/task_selection.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Any, Callable, List
+from typing import Any, Callable
 
 import pandas as pd
 from scipy.stats import pearsonr, spearmanr
@@ -14,7 +14,7 @@
 
 MODEL_NAME = str
 REVISION = str
-METRIC = Callable[[List[float], List[float]], float]
+METRIC = Callable[[list[float], list[float]], float]
 
 
 def spearman(x: list[float], y: list[float]) -> float:

mteb/tasks/Clustering/nob/snl_clustering.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
 import random
+from collections.abc import Iterable
 from itertools import islice
-from typing import Iterable, TypeVar
+from typing import TypeVar
 
 import datasets