diff --git a/mteb/abstasks/AbsTaskAnyClassification.py b/mteb/abstasks/AbsTaskAnyClassification.py index ceefe38eb4..450a42a4e1 100644 --- a/mteb/abstasks/AbsTaskAnyClassification.py +++ b/mteb/abstasks/AbsTaskAnyClassification.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from collections import Counter, defaultdict +from collections import defaultdict from typing import Any import numpy as np @@ -20,6 +20,11 @@ ) from ..evaluation.evaluators.ClassificationEvaluator import ClassificationEvaluator +from ._statistics_calculation import ( + calculate_image_statistics, + calculate_label_statistics, + calculate_text_statistics, +) from .AbsTask import AbsTask ImageFile.LOAD_TRUNCATED_IMAGES = True @@ -31,7 +36,6 @@ class ClassificationDescriptiveStatistics(DescriptiveStatistics): Attributes: num_samples: number of samples in the dataset. - number_of_characters: Total number of symbols in the dataset. number_texts_intersect_with_train: Number of texts in the train split text_statistics: Statistics for text @@ -40,7 +44,6 @@ class ClassificationDescriptiveStatistics(DescriptiveStatistics): """ num_samples: int - number_of_characters: int | None number_texts_intersect_with_train: int | None text_statistics: TextStatistics | None @@ -226,76 +229,24 @@ def _calculate_metrics_from_split( if split != self.train_split: train_text = self.dataset[self.train_split][self.input_column_name] - total_text_len = 0 - text_len = None - img_widths, img_heights = None, None + image_statistics = None + text_statistics = None num_texts_in_train = None if "image" in self.metadata.modalities: - img_widths, img_heights = [], [] - for img in inputs: - width, height = img.size # type: ignore - img_heights.append(height) - img_widths.append(width) + image_statistics = calculate_image_statistics(inputs) if "text" in self.metadata.modalities: - text_len = [len(t) for t in inputs] - total_text_len = sum(text_len) + text_statistics = calculate_text_statistics(inputs) num_texts_in_train = ( len(set(inputs) & set(train_text)) if split != self.train_split else None ) - if isinstance(label[0], int): - label_len = [1] * len(label) - total_label_len = len(label) - total_labels = label - else: - # multilabel classification - label_len = [len(l) for l in label] - total_label_len = sum(label_len) - total_labels = [] - for l in label: - total_labels.extend(l if len(l) > 0 else [None]) - - label_count = Counter(total_labels) - - text_statistics, image_statistics = None, None - if text_len: - text_statistics = TextStatistics( - min_text_length=min(text_len), - average_text_length=total_text_len / len(inputs), - max_text_length=max(text_len), - unique_texts=len(set(inputs)), - ) - - if img_widths: - image_statistics = ImageStatistics( - min_image_width=min(img_widths), - average_image_width=sum(img_widths) / len(img_widths), - max_image_width=max(img_widths), - min_image_height=min(img_heights), - average_image_height=sum(img_heights) / len(img_heights), - max_image_height=max(img_heights), - ) - - label_statistics = LabelStatistics( - min_labels_per_text=min(label_len), - average_label_per_text=total_label_len / len(label), - max_labels_per_text=max(label_len), - unique_labels=len(label_count), - labels={ - str(label): { - "count": value, - } - for label, value in label_count.items() - }, - ) + label_statistics = calculate_label_statistics(label) return ClassificationDescriptiveStatistics( num_samples=len(inputs), - # text - number_of_characters=total_text_len, number_texts_intersect_with_train=num_texts_in_train if num_texts_in_train else None, diff --git a/mteb/abstasks/AbsTaskAnyClustering.py b/mteb/abstasks/AbsTaskAnyClustering.py index b1d1c6a954..39c9f758d1 100644 --- a/mteb/abstasks/AbsTaskAnyClustering.py +++ b/mteb/abstasks/AbsTaskAnyClustering.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -from collections import Counter from typing import Any import numpy as np @@ -18,6 +17,11 @@ ) from ..evaluation.evaluators import ClusteringEvaluator +from ._statistics_calculation import ( + calculate_image_statistics, + calculate_label_statistics, + calculate_text_statistics, +) from .AbsTask import AbsTask logger = logging.getLogger(__name__) @@ -134,63 +138,15 @@ def _calculate_metrics_from_split( labels = [item for sublist in labels for item in sublist] total_text_len = 0 - text_len = None - img_widths, img_heights = None, None - + text_statistics, image_statistics = None, None if "image" in self.metadata.modalities: - img_widths, img_heights = [], [] - for img in inputs: - width, height = img.size # type: ignore - img_heights.append(height) - img_widths.append(width) + image_statistics = calculate_image_statistics(inputs) + if "text" in self.metadata.modalities: - text_len = [len(t) for t in inputs] - total_text_len = sum(text_len) + text_statistics = calculate_text_statistics(inputs) + + label_statistics = calculate_label_statistics(labels) - text_statistics, image_statistics = None, None - if text_len: - text_statistics = TextStatistics( - min_text_length=min(text_len), - average_text_length=total_text_len / len(inputs), - max_text_length=max(text_len), - unique_texts=len(set(inputs)), - ) - if img_widths: - image_statistics = ImageStatistics( - min_image_width=min(img_widths), - average_image_width=sum(img_widths) / len(img_widths), - max_image_width=max(img_widths), - min_image_height=min(img_heights), - average_image_height=sum(img_heights) / len(img_heights), - max_image_height=max(img_heights), - ) - - # labels - if isinstance(labels[0], int): - label_len = [1] * len(labels) - total_label_len = len(labels) - total_labels = labels - else: - # multilabel case - label_len = [len(l) for l in labels] - total_label_len = sum(label_len) - total_labels = [] - for l in labels: - total_labels.extend(l if len(l) > 0 else [None]) - - label_count = Counter(total_labels) - label_statistics = LabelStatistics( - min_labels_per_text=min(label_len), - average_label_per_text=total_label_len / len(labels), - max_labels_per_text=max(label_len), - unique_labels=len(label_count), - labels={ - str(label): { - "count": value, - } - for label, value in label_count.items() - }, - ) return ClusteringDescriptiveStatistics( num_samples=len(inputs), number_of_characters=total_text_len, diff --git a/mteb/abstasks/AbsTaskAnySTS.py b/mteb/abstasks/AbsTaskAnySTS.py index 09286bc208..c27de57136 100644 --- a/mteb/abstasks/AbsTaskAnySTS.py +++ b/mteb/abstasks/AbsTaskAnySTS.py @@ -15,6 +15,11 @@ ) from ..evaluation.evaluators import AnySTSEvaluator +from ._statistics_calculation import ( + calculate_image_statistics, + calculate_score_statistics, + calculate_text_statistics, +) from .AbsTask import AbsTask logger = logging.getLogger(__name__) @@ -108,71 +113,32 @@ def _calculate_metrics_from_split( score = self.dataset[split]["score"] if "text" in self.metadata.modalities: - text1_statistics = TextStatistics( - min_text_length=min(len(s) for s in sentence1), - average_text_length=sum(len(s) for s in sentence1) / len(sentence1), - max_text_length=max(len(s) for s in sentence1), - unique_texts=len(set(sentence1)), - ) - text2_statistics = TextStatistics( - min_text_length=min(len(s) for s in sentence2), - max_text_length=max(len(s) for s in sentence2), - average_text_length=sum(len(s) for s in sentence2) / len(sentence2), - unique_texts=len(set(sentence2)), - ) - sentence1_len = [len(s) for s in sentence1] - sentence2_len = [len(s) for s in sentence2] - number_of_characters = sum(sentence1_len) + sum(sentence2_len) + text1_statistics = calculate_text_statistics(sentence1) + text2_statistics = calculate_text_statistics(sentence2) + unique_pairs = len(set(zip(sentence1, sentence2))) else: text1_statistics = None text2_statistics = None - number_of_characters = None unique_pairs = None if "image" in self.metadata.modalities: - img_widths1, img_heights1 = [], [] - for img in sentence1: - width, height = img.size - img_heights1.append(height) - img_widths1.append(width) - - image1_statistics = ImageStatistics( - min_image_width=min(img_widths1), - average_image_width=sum(img_widths1) / len(img_widths1), - max_image_width=max(img_widths1), - min_image_height=min(img_heights1), - average_image_height=sum(img_heights1) / len(img_heights1), - max_image_height=max(img_widths1), - ) - - img_widths2, img_heights2 = [], [] - for img in sentence2: - width, height = img.size - img_heights2.append(height) - img_widths2.append(width) - - image2_statistics = ImageStatistics( - min_image_width=min(img_widths2), - average_image_width=sum(img_widths2) / len(img_widths2), - max_image_width=max(img_widths2), - min_image_height=min(img_heights2), - average_image_height=sum(img_heights2) / len(img_heights2), - max_image_height=max(img_widths2), - ) + image1_statistics = calculate_image_statistics(sentence1) + image2_statistics = calculate_image_statistics(sentence2) else: image1_statistics = None image2_statistics = None - labels_statistics = ScoreStatistics( - min_score=min(score), - avg_score=sum(score) / len(score), - max_score=max(score), - ) + labels_statistics = calculate_score_statistics(score) return AnySTSDescriptiveStatistics( num_samples=len(sentence1), - number_of_characters=number_of_characters, + number_of_characters=( + text1_statistics["total_text_length"] + + text2_statistics["total_text_length"] + if text1_statistics + else None + ), unique_pairs=unique_pairs, text1_statistics=text1_statistics, text2_statistics=text2_statistics, diff --git a/mteb/abstasks/AbsTaskAnyZeroShotClassification.py b/mteb/abstasks/AbsTaskAnyZeroShotClassification.py index df89974fe3..9c9585091d 100644 --- a/mteb/abstasks/AbsTaskAnyZeroShotClassification.py +++ b/mteb/abstasks/AbsTaskAnyZeroShotClassification.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -from collections import Counter from typing import Any from datasets import Dataset @@ -16,6 +15,11 @@ from ..evaluation.evaluators import ZeroShotClassificationEvaluator from ..models.encoder_interface import Encoder +from ._statistics_calculation import ( + calculate_image_statistics, + calculate_label_statistics, + calculate_text_statistics, +) from .AbsTask import AbsTask logger = logging.getLogger(__name__) @@ -32,9 +36,7 @@ class ZeroShotClassificationDescriptiveStatistics(DescriptiveStatistics): image_statistics: Statistics for images label_statistics: Statistics for dataset labels - min_label_text_length: Minimum length of candidate label text - average_label_text_length: Average length of candidate label text - max_label_text_length: Maximum length of candidate label text + candidates_labels_text_statistics: Statistics for candidate labels text """ num_samples: int @@ -43,10 +45,7 @@ class ZeroShotClassificationDescriptiveStatistics(DescriptiveStatistics): text_statistics: TextStatistics | None image_statistics: ImageStatistics | None label_statistics: LabelStatistics - - min_label_text_length: int - average_label_text_length: float - max_label_text_length: int + candidates_labels_text_statistics: TextStatistics class AbsTaskAnyZeroShotClassification(AbsTask): @@ -83,47 +82,17 @@ def _calculate_metrics_from_split( labels = self.dataset[split][self.label_column_name] num_samples = len(inputs) - label_count = Counter(labels) - # build image statistics image_statistics = None - if "image" in self.metadata.modalities: - img_widths, img_heights = [], [] - for img in inputs: - w, h = img.size # type: ignore - img_widths.append(w) - img_heights.append(h) - - image_statistics = ImageStatistics( - min_image_width=min(img_widths), - average_image_width=sum(img_widths) / len(img_widths), - max_image_width=max(img_widths), - min_image_height=min(img_heights), - average_image_height=sum(img_heights) / len(img_heights), - max_image_height=max(img_heights), - ) - text_statistics = None + + if "image" in self.metadata.modalities: + image_statistics = calculate_image_statistics(inputs) if self.metadata.modalities == ["text"]: - # build text statistics - text_lengths = [len(str(text)) for text in inputs] - text_statistics = TextStatistics( - min_text_length=min(text_lengths), - average_text_length=sum(text_lengths) / len(text_lengths), - max_text_length=max(text_lengths), - ) - - # single‐label per sample => use LabelStatistics - label_statistics = LabelStatistics( - min_labels_per_text=1, - average_label_per_text=1.0, - max_labels_per_text=1, - unique_labels=len(label_count), - labels={str(lbl): {"count": cnt} for lbl, cnt in label_count.items()}, - ) + text_statistics = calculate_text_statistics(inputs) - # candidate‐label text lengths - candidate_lens = [len(c) for c in self.get_candidate_labels()] + label_statistics = calculate_label_statistics(labels) + candidate_lens = calculate_text_statistics(self.get_candidate_labels()) return ZeroShotClassificationDescriptiveStatistics( num_samples=num_samples, @@ -131,9 +100,7 @@ def _calculate_metrics_from_split( text_statistics=text_statistics, image_statistics=image_statistics, label_statistics=label_statistics, - min_label_text_length=min(candidate_lens), - average_label_text_length=sum(candidate_lens) / len(candidate_lens), - max_label_text_length=max(candidate_lens), + candidates_labels_text_statistics=candidate_lens, ) def _evaluate_subset( diff --git a/mteb/abstasks/AbsTaskBitextMining.py b/mteb/abstasks/AbsTaskBitextMining.py index a33cb2d2ed..080422640c 100644 --- a/mteb/abstasks/AbsTaskBitextMining.py +++ b/mteb/abstasks/AbsTaskBitextMining.py @@ -7,9 +7,10 @@ from mteb.models.encoder_interface import Encoder from mteb.types import HFSubset, ScoresDict -from mteb.types.statistics import DescriptiveStatistics +from mteb.types.statistics import DescriptiveStatistics, TextStatistics from ..evaluation.evaluators import BitextMiningEvaluator +from ._statistics_calculation import calculate_text_statistics from .AbsTask import AbsTask logger = logging.getLogger(__name__) @@ -23,29 +24,16 @@ class BitextDescriptiveStatistics(DescriptiveStatistics): number_of_characters: Total number of symbols in the dataset. unique_pairs: Number of duplicate pairs - min_sentence1_length: Minimum length of sentence1 - average_sentence1_length: Average length of sentence1 - max_sentence1_length: Maximum length of sentence1 - unique_sentence1: Number of duplicates in sentence1 - - min_sentence2_length: Minimum length of sentence2 - average_sentence2_length: Average length of sentence2 - max_sentence2_length: Maximum length of sentence2 + sentence1_statistics: Statistics for sentence1 + sentence2_statistics: Statistics for sentence2 """ num_samples: int number_of_characters: int unique_pairs: int - min_sentence1_length: int - average_sentence1_length: float - max_sentence1_length: int - unique_sentence1: int - - min_sentence2_length: int - average_sentence2_length: float - max_sentence2_length: int - unique_sentence2: int + sentence1_statistics: TextStatistics + sentence2_statistics: TextStatistics class AbsTaskBitextMining(AbsTask): @@ -175,26 +163,20 @@ def _calculate_metrics_from_split( sent_1, sent_2 = pairs_cols[0] sentence1 = self.dataset[split][sent_1] sentence2 = self.dataset[split][sent_2] - s1_len = [len(s1) for s1 in sentence1] - s2_len = [len(s2) for s2 in sentence2] - total_s1_len = sum(s1_len) - total_s2_len = sum(s2_len) + text1_statistics = calculate_text_statistics(sentence1) + text2_statistics = calculate_text_statistics(sentence2) unique_pairs = len(set(zip(sentence1, sentence2))) - unique_sentence1 = len(set(sentence1)) - unique_sentence2 = len(set(sentence2)) + return BitextDescriptiveStatistics( num_samples=len(sentence1), - number_of_characters=total_s1_len + total_s2_len, + number_of_characters=( + text1_statistics["total_text_length"] + + text2_statistics["total_text_length"] + ), unique_pairs=unique_pairs, - min_sentence1_length=min(s1_len), - average_sentence1_length=sum(s1_len) / len(sentence1), - max_sentence1_length=max(s1_len), - unique_sentence1=unique_sentence1, - min_sentence2_length=min(s2_len), - average_sentence2_length=total_s2_len / len(sentence2), - max_sentence2_length=max(s2_len), - unique_sentence2=unique_sentence2, + sentence1_statistics=text1_statistics, + sentence2_statistics=text2_statistics, ) def _push_dataset_to_hub(self, repo_name: str) -> None: diff --git a/mteb/abstasks/AbsTaskClusteringFast.py b/mteb/abstasks/AbsTaskClusteringFast.py index 8d21b1d9e6..8e2af77747 100644 --- a/mteb/abstasks/AbsTaskClusteringFast.py +++ b/mteb/abstasks/AbsTaskClusteringFast.py @@ -3,7 +3,7 @@ import itertools import logging import random -from collections import Counter, defaultdict +from collections import defaultdict from typing import Any import numpy as np @@ -15,8 +15,12 @@ from mteb.models.encoder_interface import Encoder from mteb.types import HFSubset -from mteb.types.statistics import DescriptiveStatistics +from mteb.types.statistics import DescriptiveStatistics, LabelStatistics, TextStatistics +from ._statistics_calculation import ( + calculate_label_statistics, + calculate_text_statistics, +) from .AbsTask import AbsTask logger = logging.getLogger(__name__) @@ -79,37 +83,19 @@ def evaluate_clustering_bootstrapped( class ClusteringFastDescriptiveStatistics(DescriptiveStatistics): - """Descriptive statistics for Clustering + """Descriptive statistics for ClusteringFast Attributes: num_samples: number of samples in the dataset. - number_of_characters: Total number of symbols in the dataset. - - min_text_length: Minimum length of text - average_text_length: Average length of text - max_text_length: Maximum length of text - unique_texts: Number of unique texts - - min_labels_per_text: Minimum number of labels per text - average_labels_per_text: Average number of labels per text - max_labels_per_text: Maximum number of labels per text - unique_labels: Number of unique labels - labels: dict of label frequencies + + text_statistics: Statistics for the text + labels_statistics: Statistics for the labels """ num_samples: int - number_of_characters: int - - min_text_length: int - average_text_length: float - max_text_length: int - unique_texts: int - min_labels_per_text: int - average_labels_per_text: float - max_labels_per_text: int - unique_labels: int - labels: dict[str, dict[str, int]] + text_statistics: TextStatistics + labels_statistics: LabelStatistics class AbsTaskClusteringFast(AbsTask): @@ -233,32 +219,10 @@ def _calculate_metrics_from_split( sentences = self.dataset[split]["sentences"] labels = self.dataset[split]["labels"] - text_len = [len(t) for t in sentences] - total_text_len = sum(text_len) - total_labels = [] - for label in labels: - if isinstance(label, list): - total_labels.extend(label) - else: - total_labels.append(label) - label_counter = Counter(total_labels) return ClusteringFastDescriptiveStatistics( num_samples=len(sentences), - number_of_characters=total_text_len, - min_text_length=min(text_len), - average_text_length=total_text_len / len(sentences), - max_text_length=max(text_len), - unique_texts=len(set(text_len)), - min_labels_per_text=min(label_counter.values()), - average_labels_per_text=len(total_labels) / len(sentences), - max_labels_per_text=max(label_counter.values()), - unique_labels=len(label_counter), - labels={ - str(label): { - "count": value, - } - for label, value in label_counter.items() - }, + text_statistics=calculate_text_statistics(sentences), + labels_statistics=calculate_label_statistics(labels), ) def _push_dataset_to_hub(self, repo_name: str) -> None: diff --git a/mteb/abstasks/AbsTaskPairClassification.py b/mteb/abstasks/AbsTaskPairClassification.py index 8f8b85f39c..9cd039b9e7 100644 --- a/mteb/abstasks/AbsTaskPairClassification.py +++ b/mteb/abstasks/AbsTaskPairClassification.py @@ -1,15 +1,19 @@ from __future__ import annotations import logging -from collections import Counter, defaultdict +from collections import defaultdict from datasets import Dataset from mteb.types import ScoresDict -from mteb.types.statistics import DescriptiveStatistics +from mteb.types.statistics import DescriptiveStatistics, LabelStatistics, TextStatistics from ..evaluation.evaluators import PairClassificationEvaluator from ..models.encoder_interface import Encoder +from ._statistics_calculation import ( + calculate_label_statistics, + calculate_text_statistics, +) from .AbsTask import AbsTask logger = logging.getLogger(__name__) @@ -23,36 +27,18 @@ class PairClassificationDescriptiveStatistics(DescriptiveStatistics): number_of_characters: Total number of symbols in the dataset. unique_pairs: Number of unique pairs - min_sentence1_length: Minimum length of sentence1 - avg_sentence1_length: Average length of sentence1 - max_sentence1_length: Maximum length of sentence1 - unique_sentence1: Number of unique sentence - - min_sentence2_length: Minimum length of sentence2 - avg_sentence2_length: Average length of sentence2 - max_sentence2_length: Maximum length of sentence2 - unique_sentence2: Number of unique sentence - - unique_labels: Number of unique labels - labels: dict of label frequencies + text1_statistics: Statistics for sentence1 + text2_statistics: Statistics for sentence2 + labels_statistics: Statistics for labels """ num_samples: int number_of_characters: int unique_pairs: int - min_sentence1_length: int - avg_sentence1_length: float - max_sentence1_length: int - unique_sentence1: int - - min_sentence2_length: int - avg_sentence2_length: float - max_sentence2_length: int - unique_sentence2: int - - unique_labels: int - labels: dict[str, dict[str, int]] + text1_statistics: TextStatistics + text2_statistics: TextStatistics + labels_statistics: LabelStatistics class AbsTaskPairClassification(AbsTask): @@ -129,27 +115,18 @@ def _calculate_metrics_from_split( dataset["labels"][0] if len(dataset["labels"]) == 1 else dataset["labels"] ) - sentence1_len = [len(sentence) for sentence in sentence1] - total_sentence1_len = sum(sentence1_len) - sentence2_len = [len(sentence) for sentence in sentence2] - total_sentence2_len = sum(sentence2_len) - label_count = Counter(labels) + text1_statistics = calculate_text_statistics(sentence1) + text2_statistics = calculate_text_statistics(sentence2) return PairClassificationDescriptiveStatistics( num_samples=len(sentence1), - number_of_characters=total_sentence1_len + total_sentence2_len, + number_of_characters=( + text1_statistics["total_text_length"] + + text2_statistics["total_text_length"] + ), unique_pairs=len(set(zip(sentence1, sentence2))), - min_sentence1_length=min(sentence1_len), - avg_sentence1_length=total_sentence1_len / len(sentence1), - max_sentence1_length=max(sentence1_len), - unique_sentence1=len(set(sentence1)), - min_sentence2_length=min(sentence2_len), - avg_sentence2_length=total_sentence2_len / len(sentence2), - max_sentence2_length=max(sentence2_len), - unique_sentence2=len(set(sentence2)), - unique_labels=len(set(labels)), - labels={ - str(label): {"count": count} for label, count in label_count.items() - }, + text1_statistics=text1_statistics, + text2_statistics=text2_statistics, + labels_statistics=calculate_label_statistics(labels), ) def _push_dataset_to_hub(self, repo_name: str) -> None: diff --git a/mteb/abstasks/AbsTaskRetrieval.py b/mteb/abstasks/AbsTaskRetrieval.py index 8a5edf7119..92921100f1 100644 --- a/mteb/abstasks/AbsTaskRetrieval.py +++ b/mteb/abstasks/AbsTaskRetrieval.py @@ -11,10 +11,21 @@ from mteb.models.encoder_interface import Encoder from mteb.types import HFSubset, ScoresDict -from mteb.types.statistics import DescriptiveStatistics - +from mteb.types.statistics import ( + DescriptiveStatistics, + RelevantDocsStatistics, + TextStatistics, + TopRankedStatistics, +) + +from ..create_dataloaders import corpus_to_dict from ..evaluation.evaluators import RetrievalEvaluator from ..evaluation.evaluators.retrieval_metrics import make_score_dict +from ._statistics_calculation import ( + calculate_relevant_docs_statistics, + calculate_text_statistics, + calculate_top_ranked_statistics, +) from .AbsTask import AbsTask from .dataset_loaders import RetrievalDatasetLoader, RetrievalSplitData @@ -26,73 +37,28 @@ class RetrievalDescriptiveStatistics(DescriptiveStatistics): Attributes: num_samples: Number of queries and documents - num_relevant_docs: Number of relevant documents - - num_documents: Number of documents - min_document_length: Minimum length of documents - average_document_length: Average length of documents - max_document_length: Maximum length of documents - unique_documents: Number of unique documents - - num_queries: number of queries in the dataset - min_query_length: Minimum length of queries - average_query_length: Average length of queries - max_query_length: Maximum length of queries - unique_queries: Number of unique queries - none_queries: Number of none queries - - number_of_characters: Total number of symbols in the dataset - min_relevant_docs_per_query: Minimum number of relevant documents per query - average_relevant_docs_per_query: Average number of relevant documents per query - max_relevant_docs_per_query: Maximum number of relevant documents per query - unique_relevant_docs: Number of unique relevant documents - - num_instructions: Number of instructions - min_instruction_length: Minimum length of instructions - average_instruction_length: Average length of instructions - max_instruction_length: Maximum length of instructions - unique_instructions: Number of unique instructions - - num_top_ranked: Number of top ranked documents - min_top_ranked_per_query: Minimum number of top ranked documents per query - average_top_ranked_per_query: Average number of top ranked documents per query - max_top_ranked_per_query: Maximum number of relevant documents per query + number_of_characters: Total number of characters in queries and documents + + documents_statistics: Statistics for documents + queries_statistics: Statistics for queries + relevant_docs_statistics: Statistics for relevant documents + instructions_statistics: Statistics for instructions (if available) + top_ranked_statistics: Statistics for top ranked documents (if available) """ num_samples: int number_of_characters: int - num_documents: int - min_document_length: int - average_document_length: float - max_document_length: int - unique_documents: int - - num_queries: int - min_query_length: int - average_query_length: float - max_query_length: int - unique_queries: int - none_queries: int - - num_relevant_docs: int - min_relevant_docs_per_query: int - average_relevant_docs_per_query: float - max_relevant_docs_per_query: float - unique_relevant_docs: int + documents_statistics: TextStatistics + queries_statistics: TextStatistics + + relevant_docs_statistics: RelevantDocsStatistics # these are for datasets with instructions - num_instructions: int | None - min_instruction_length: int | None - average_instruction_length: float | None - max_instruction_length: float | None - unique_instructions: int | None + instructions_statistics: TextStatistics | None # this is for datasets that do reranking - num_top_ranked: int | None - min_top_ranked_per_query: int | None - average_top_ranked_per_query: float | None - max_top_ranked_per_query: int | None + top_ranked_statistics: TopRankedStatistics | None class AbsTaskRetrieval(AbsTask): @@ -468,86 +434,47 @@ def _calculate_metrics_from_split( instructions = split_data["instructions"] top_ranked = split_data["top_ranked"] - query_len = calculate_queries_length(queries) - doc_len = calculate_corpus_length(corpus) - num_documents = len(doc_len) if corpus is not None else 0 - num_queries = len(query_len) - num_relevant_docs = sum(len(relevant_docs[qid]) for qid in relevant_docs) - none_queries = sum(q is None or len(q) == 0 for q in queries.values()) - - # create a list of number of relevant docs per query - qrels_lengths = [ - len(relevant_docs[qid]) for qid in relevant_docs if qid in queries - ] - unique_qrels = len({doc for qid in relevant_docs for doc in relevant_docs[qid]}) - # number of qrels that are not 0 - num_qrels_non_zero = sum( - sum(1 for doc_id in docs if docs[doc_id] != 0) - for docs in relevant_docs.values() + corpus = list(corpus_to_dict(list(corpus.values()))["text"]) + queries_texts = [q for q in queries.values() if isinstance(q, str)] + num_documents = len(corpus) + num_queries = len(queries_texts) + + relevant_docs_statistics = calculate_relevant_docs_statistics( + relevant_docs, list(queries.keys()) ) - qrels_per_doc = num_qrels_non_zero / len(relevant_docs) if num_queries else 0 if instructions is not None and len(instructions) > 0: - instructions_len = [ - len(instruction) for instruction in instructions.values() - ] - num_instructions = len(instructions) - average_instruction_length = sum(instructions_len) - min_instruction_length = min(instructions_len) - max_instruction_length = max(instructions_len) - unique_instructions = len(set(instructions)) + instruction_statistics = calculate_text_statistics( + list(instructions.values()) + ) else: - num_instructions = None - average_instruction_length = None - min_instruction_length = None - max_instruction_length = None - unique_instructions = None + instruction_statistics = None if top_ranked is not None and num_queries and len(top_ranked) > 0: - top_ranked_per_query = [len(docs) for docs in top_ranked.values()] - num_top_ranked = len(top_ranked_per_query) - min_top_ranked_per_query = min(top_ranked_per_query) - average_top_ranked_per_query = sum(top_ranked_per_query) / num_queries - max_top_ranked_per_query = max(top_ranked_per_query) + top_ranked_statistics = calculate_top_ranked_statistics( + top_ranked, num_queries + ) else: - num_top_ranked = None - min_top_ranked_per_query = None - average_top_ranked_per_query = None - max_top_ranked_per_query = None + top_ranked_statistics = None + + corpus_statistics = calculate_text_statistics(corpus) + queries_statistics = calculate_text_statistics(list(queries.values())) + + number_of_characters = ( + corpus_statistics["total_text_length"] + + queries_statistics["total_text_length"] + ) + if instruction_statistics is not None: + number_of_characters += instruction_statistics["total_text_length"] return RetrievalDescriptiveStatistics( num_samples=num_documents + num_queries, - number_of_characters=sum(query_len) + sum(doc_len), - # documents - num_documents=num_documents, - min_document_length=min(doc_len), - average_document_length=sum(doc_len) / num_documents, - max_document_length=max(doc_len), - unique_documents=len(set(corpus)), - # queries - num_queries=num_queries, - min_query_length=min(query_len), - average_query_length=sum(query_len) / num_queries, - max_query_length=max(query_len), - unique_queries=len(set(queries)), - none_queries=none_queries, - # relevant docs - num_relevant_docs=num_relevant_docs, - min_relevant_docs_per_query=min(qrels_lengths), - average_relevant_docs_per_query=qrels_per_doc, - max_relevant_docs_per_query=max(qrels_lengths), - unique_relevant_docs=unique_qrels, - # instructions - num_instructions=num_instructions, - min_instruction_length=min_instruction_length, - average_instruction_length=average_instruction_length, - max_instruction_length=max_instruction_length, - unique_instructions=unique_instructions, - # top ranked - num_top_ranked=num_top_ranked, - min_top_ranked_per_query=min_top_ranked_per_query, - average_top_ranked_per_query=average_top_ranked_per_query, - max_top_ranked_per_query=max_top_ranked_per_query, + number_of_characters=number_of_characters, + documents_statistics=corpus_statistics, + queries_statistics=queries_statistics, + relevant_docs_statistics=relevant_docs_statistics, + instructions_statistics=instruction_statistics, + top_ranked_statistics=top_ranked_statistics, ) def _push_dataset_to_hub(self, repo_name: str) -> None: diff --git a/mteb/abstasks/AbsTaskSummarization.py b/mteb/abstasks/AbsTaskSummarization.py index 4468ecaeaf..8a6eb17502 100644 --- a/mteb/abstasks/AbsTaskSummarization.py +++ b/mteb/abstasks/AbsTaskSummarization.py @@ -8,9 +8,13 @@ from mteb.models.encoder_interface import Encoder from mteb.types import ScoresDict -from mteb.types.statistics import DescriptiveStatistics +from mteb.types.statistics import DescriptiveStatistics, ScoreStatistics, TextStatistics from ..evaluation.evaluators import SummarizationEvaluator +from ._statistics_calculation import ( + calculate_score_statistics, + calculate_text_statistics, +) from .AbsTask import AbsTask logger = logging.getLogger(__name__) @@ -23,47 +27,19 @@ class SummarizationDescriptiveStatistics(DescriptiveStatistics): num_samples: number of samples in the dataset. number_of_characters: Total number of symbols in the dataset. - min_text_length: Minimum length of text - avg_text_length: Average length of text - max_text_length: Maximum length of text - unique_texts: Number of unique texts - - min_human_summaries_length: Minimum length of human summaries - avg_human_summaries_length: Average length of human summaries - max_human_summaries_length: Maximum length of human summaries - unique_human_summaries: Number of unique human summaries - - min_machine_summaries_length: Minimum length of machine summaries - avg_machine_summaries_length: Average length of machine summaries - max_machine_summaries_length: Maximum length of machine summaries - unique_machine_summaries: Number of unique machine summaries - - min_relevance: Minimum relevance score - avg_relevance: Average relevance score - max_relevance: Maximum relevance score + text_statistics: Statistics for the text + human_summaries_statistics: Statistics for human summaries + machine_summaries_statistics: Statistics for machine summaries + score_statistics: Statistics for the relevance scoresk """ num_samples: int number_of_characters: int - min_text_length: int - avg_text_length: float - max_text_length: int - unique_texts: int - - min_human_summaries_length: int - avg_human_summaries_length: float - max_human_summaries_length: int - unique_human_summaries: int - - min_machine_summaries_length: int - avg_machine_summaries_length: float - max_machine_summaries_length: int - unique_machine_summaries: int - - min_relevance: float - avg_relevance: float - max_relevance: float + text_statistics: TextStatistics + human_summaries_statistics: TextStatistics + machine_summaries_statistics: TextStatistics + score_statistics: ScoreStatistics class AbsTaskSummarization(AbsTask): @@ -73,7 +49,7 @@ class AbsTaskSummarization(AbsTask): text: str human_summaries: list[str] machine_summaries: list[str] - relevance: list[float] (the score of the machine generated summaries) + relevance: list[list[float]] (the score of the machine generated summaries) """ min_score: int @@ -150,31 +126,21 @@ def _calculate_metrics_from_split( for s in machine_summaries: all_machine_summaries.extend(s) - text_len = [len(t) for t in text] - total_text_len = sum(text_len) - human_summaries_len = [len(s) for s in human_summaries] - total_human_summaries_len = sum(human_summaries_len) - machine_summaries_len = [len(s) for s in machine_summaries] - total_machine_summaries_len = sum(machine_summaries_len) - total_relevance = sum(sum(x) / len(x) for x in relevance) + text_statistics = calculate_text_statistics(text) + human_summaries_statistics = calculate_text_statistics(all_human_summaries) + machine_summaries_statistics = calculate_text_statistics(all_machine_summaries) + + relevance = [item for sublist in relevance for item in sublist] + return SummarizationDescriptiveStatistics( num_samples=len(text), - number_of_characters=total_text_len - + total_human_summaries_len - + total_machine_summaries_len, - min_text_length=min(text_len), - avg_text_length=total_text_len / len(text), - max_text_length=max(text_len), - unique_texts=len(set(text)), - min_human_summaries_length=min(human_summaries_len), - avg_human_summaries_length=total_human_summaries_len / len(text), - max_human_summaries_length=max(human_summaries_len), - unique_human_summaries=len(set(all_human_summaries)), - min_machine_summaries_length=min(machine_summaries_len), - avg_machine_summaries_length=total_machine_summaries_len / len(text), - max_machine_summaries_length=max(machine_summaries_len), - unique_machine_summaries=len(set(all_machine_summaries)), - min_relevance=min(relevance), - avg_relevance=total_relevance / len(relevance), - max_relevance=max(relevance), + number_of_characters=( + text_statistics["total_text_length"] + + human_summaries_statistics["total_text_length"] + + machine_summaries_statistics["total_text_length"] + ), + text_statistics=text_statistics, + human_summaries_statistics=human_summaries_statistics, + machine_summaries_statistics=machine_summaries_statistics, + score_statistics=calculate_score_statistics(relevance), ) diff --git a/mteb/abstasks/_statistics_calculation.py b/mteb/abstasks/_statistics_calculation.py new file mode 100644 index 0000000000..30aa1f2e91 --- /dev/null +++ b/mteb/abstasks/_statistics_calculation.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +from collections import Counter +from typing import Any + +from mteb.types.statistics import ( + ImageStatistics, + LabelStatistics, + RelevantDocsStatistics, + ScoreStatistics, + TextStatistics, + TopRankedStatistics, +) + + +def calculate_text_statistics(texts: list[str]) -> TextStatistics: + """Calculate descriptive statistics for a list of texts. + + Args: + texts: List of texts to analyze. + + Returns: + TextStatistics: A dictionary containing the descriptive statistics. + """ + lengths = [len(text) for text in texts] + unique_texts = len(set(texts)) + + return TextStatistics( + total_text_length=sum(lengths), + min_text_length=min(lengths), + average_text_length=sum(lengths) / len(lengths), + max_text_length=max(lengths), + unique_texts=unique_texts, + ) + + +def calculate_image_statistics(images: list[Any]) -> ImageStatistics: + """Calculate descriptive statistics for a list of images. + + Args: + images: List of images to analyze. Each image should have a `size` attribute that returns a tuple (width, height). + + Returns: + ImageStatistics: A dictionary containing the descriptive statistics. + """ + img_widths, img_heights = [], [] + for img in images: + width, height = img.size # type: ignore + img_heights.append(height) + img_widths.append(width) + + return ImageStatistics( + min_image_width=min(img_widths), + average_image_width=sum(img_widths) / len(img_widths), + max_image_width=max(img_widths), + min_image_height=min(img_heights), + average_image_height=sum(img_heights) / len(img_heights), + max_image_height=max(img_heights), + # some image types (PngImageFile) may be unhashable + unique_images=len({id(img) for img in images}), + ) + + +def calculate_label_statistics(labels: list[int | list[int]]) -> LabelStatistics: + """Calculate descriptive statistics for a list of labels. + + Args: + labels: List of labels, where each label can be an integer or a list of integers (for multilabel classification). + + Returns: + LabelStatistics: A dictionary containing the descriptive statistics. + """ + if not isinstance(labels[0], list): + label_len = [1] * len(labels) + total_label_len = len(labels) + total_labels = labels + elif isinstance(labels[0], list): + # multilabel classification + label_len = [len(l) for l in labels] + total_label_len = sum(label_len) + total_labels = [] + for l in labels: + total_labels.extend(l if len(l) > 0 else [None]) + else: + raise ValueError( + "Labels must be a list of integers or a list of lists of integers." + ) + + label_count = Counter(total_labels) + return LabelStatistics( + min_labels_per_text=min(label_len), + average_label_per_text=total_label_len / len(labels), + max_labels_per_text=max(label_len), + unique_labels=len(label_count), + labels={ + str(label): { + "count": value, + } + for label, value in label_count.items() + }, + ) + + +def calculate_score_statistics(scores: list[int | float]) -> ScoreStatistics: + """Calculate descriptive statistics for a list of scores. + + Args: + scores: List of scores to analyze. + + Returns: + ScoreStatistics: A dictionary containing the descriptive statistics. + """ + return ScoreStatistics( + min_score=min(scores), + avg_score=sum(scores) / len(scores), + max_score=max(scores), + ) + + +def calculate_top_ranked_statistics( + top_ranked: dict[str, list[str]], num_queries: int +) -> TopRankedStatistics: + """Calculate statistics for top-ranked items. + + Args: + top_ranked: List of lists, where each inner list contains IDs of top-ranked items. + num_queries: Total number of queries. + + Returns: + dict: A dictionary with the count of top-ranked items per ID. + """ + return TopRankedStatistics( + num_top_ranked=sum( + len(docs) for docs in top_ranked.values() if docs is not None + ), + min_top_ranked_per_query=min( + len(docs) for docs in top_ranked.values() if docs is not None + ), + average_top_ranked_per_query=( + sum(len(docs) for docs in top_ranked.values() if docs is not None) + / num_queries + ), + max_top_ranked_per_query=max( + len(docs) for docs in top_ranked.values() if docs is not None + ), + ) + + +def calculate_relevant_docs_statistics( + relevant_docs: dict[str, dict[str, float]], queries_ids: list[str] +) -> RelevantDocsStatistics: + qrels_lengths = [ + len(relevant_docs[qid]) for qid in relevant_docs if qid in queries_ids + ] + unique_qrels = len({doc for qid in relevant_docs for doc in relevant_docs[qid]}) + # number of qrels that are not 0 + num_qrels_non_zero = sum( + sum(1 for doc_id in docs if docs[doc_id] != 0) + for docs in relevant_docs.values() + ) + qrels_per_doc = num_qrels_non_zero / len(relevant_docs) if len(queries_ids) else 0 + + return RelevantDocsStatistics( + num_relevant_docs=num_qrels_non_zero, + min_relevant_docs_per_query=min(qrels_lengths), + average_relevant_docs_per_query=qrels_per_doc, + max_relevant_docs_per_query=max(qrels_lengths), + unique_relevant_docs=unique_qrels, + ) diff --git a/mteb/types/statistics.py b/mteb/types/statistics.py index f57362eda3..e0a9fa610c 100644 --- a/mteb/types/statistics.py +++ b/mteb/types/statistics.py @@ -13,12 +13,14 @@ class TextStatistics(TypedDict): """Class for descriptive statistics for texts. Attributes: + total_text_length: Total length of all texts min_text_length: Minimum length of text average_text_length: Average length of text max_text_length: Maximum length of text unique_texts: Number of unique texts """ + total_text_length: int min_text_length: int average_text_length: float max_text_length: int @@ -36,6 +38,8 @@ class ImageStatistics(TypedDict): min_image_height: Minimum height of images average_image_height: Average height of images max_image_height: Maximum height of images + + unique_images: Number of unique images """ min_image_width: float @@ -46,6 +50,8 @@ class ImageStatistics(TypedDict): average_image_height: float max_image_height: float + unique_images: int + class LabelStatistics(TypedDict): """Class for descriptive statistics for texts. @@ -71,14 +77,45 @@ class ScoreStatistics(TypedDict): """Class for descriptive statistics for texts. Attributes: - min_labels_per_text: Minimum number of labels per text - average_label_per_text: Average number of labels per text - max_labels_per_text: Maximum number of labels per text - - unique_labels: Number of unique labels - labels: dict of label frequencies + min_score: Minimum score + avg_score: Average score + max_score: Maximum score """ min_score: int avg_score: float max_score: int + + +class TopRankedStatistics(TypedDict): + """Statistics for top ranked documents in a retrieval task. + + Attributes: + num_top_ranked: Total number of top ranked documents across all queries. + min_top_ranked_per_query: Minimum number of top ranked documents for any query. + average_top_ranked_per_query: Average number of top ranked documents per query. + max_top_ranked_per_query: Maximum number of top ranked documents for any query. + """ + + num_top_ranked: int + min_top_ranked_per_query: int + average_top_ranked_per_query: float + max_top_ranked_per_query: int + + +class RelevantDocsStatistics(TypedDict): + """Statistics for relevant documents in a retrieval task. + + Attributes: + num_relevant_docs: Total number of relevant documents across all queries. + min_relevant_docs_per_query: Minimum number of relevant documents for any query. + average_relevant_docs_per_query: Average number of relevant documents per query. + max_relevant_docs_per_query: Maximum number of relevant documents for any query. + unique_relevant_docs: Number of unique relevant documents across all queries. + """ + + num_relevant_docs: int + min_relevant_docs_per_query: int + average_relevant_docs_per_query: float + max_relevant_docs_per_query: float + unique_relevant_docs: int diff --git a/tests/test_benchmark/mock_tasks.py b/tests/test_benchmark/mock_tasks.py index ca988154a4..6bd919cdd0 100644 --- a/tests/test_benchmark/mock_tasks.py +++ b/tests/test_benchmark/mock_tasks.py @@ -2,7 +2,6 @@ from __future__ import annotations -import numpy as np from datasets import Dataset, DatasetDict from PIL import Image @@ -86,9 +85,9 @@ class MockClassificationTask(AbsTaskAnyClassification): expected_stats = { "test": { "num_samples": 2, - "number_of_characters": 52, "number_texts_intersect_with_train": 1, "text_statistics": { + "total_text_length": 52, "min_text_length": 23, "average_text_length": 26.0, "max_text_length": 29, @@ -105,9 +104,9 @@ class MockClassificationTask(AbsTaskAnyClassification): }, "train": { "num_samples": 2, - "number_of_characters": 53, "number_texts_intersect_with_train": None, "text_statistics": { + "total_text_length": 53, "min_text_length": 23, "average_text_length": 26.5, "max_text_length": 30, @@ -160,9 +159,9 @@ class MockMultilingualClassificationTask(AbsTaskAnyClassification): expected_stats = { "test": { "num_samples": 4, - "number_of_characters": 104, "number_texts_intersect_with_train": 1, "text_statistics": { + "total_text_length": 104, "min_text_length": 23, "average_text_length": 26.0, "max_text_length": 29, @@ -179,9 +178,9 @@ class MockMultilingualClassificationTask(AbsTaskAnyClassification): "hf_subset_descriptive_stats": { "eng": { "num_samples": 2, - "number_of_characters": 52, "number_texts_intersect_with_train": 1, "text_statistics": { + "total_text_length": 52, "min_text_length": 23, "average_text_length": 26.0, "max_text_length": 29, @@ -198,9 +197,9 @@ class MockMultilingualClassificationTask(AbsTaskAnyClassification): }, "fra": { "num_samples": 2, - "number_of_characters": 52, "number_texts_intersect_with_train": 1, "text_statistics": { + "total_text_length": 52, "min_text_length": 23, "average_text_length": 26.0, "max_text_length": 29, @@ -219,9 +218,9 @@ class MockMultilingualClassificationTask(AbsTaskAnyClassification): }, "train": { "num_samples": 4, - "number_of_characters": 106, "number_texts_intersect_with_train": None, "text_statistics": { + "total_text_length": 106, "min_text_length": 23, "average_text_length": 26.5, "max_text_length": 30, @@ -238,9 +237,9 @@ class MockMultilingualClassificationTask(AbsTaskAnyClassification): "hf_subset_descriptive_stats": { "eng": { "num_samples": 2, - "number_of_characters": 53, "number_texts_intersect_with_train": None, "text_statistics": { + "total_text_length": 53, "min_text_length": 23, "average_text_length": 26.5, "max_text_length": 30, @@ -257,9 +256,9 @@ class MockMultilingualClassificationTask(AbsTaskAnyClassification): }, "fra": { "num_samples": 2, - "number_of_characters": 53, "number_texts_intersect_with_train": None, "text_statistics": { + "total_text_length": 53, "min_text_length": 23, "average_text_length": 26.5, "max_text_length": 30, @@ -320,14 +319,20 @@ class MockBitextMiningTask(AbsTaskBitextMining): "num_samples": 2, "number_of_characters": 113, "unique_pairs": 2, - "min_sentence1_length": 23, - "average_sentence1_length": 26.0, - "max_sentence1_length": 29, - "unique_sentence1": 2, - "min_sentence2_length": 24, - "average_sentence2_length": 30.5, - "max_sentence2_length": 37, - "unique_sentence2": 2, + "sentence1_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "sentence2_statistics": { + "total_text_length": 61, + "min_text_length": 24, + "average_text_length": 30.5, + "max_text_length": 37, + "unique_texts": 2, + }, } } @@ -364,40 +369,58 @@ class MockMultilingualBitextMiningTask(AbsTaskBitextMining): "num_samples": 4, "number_of_characters": 226, "unique_pairs": 2, - "min_sentence1_length": 23, - "average_sentence1_length": 26.0, - "max_sentence1_length": 29, - "unique_sentence1": 2, - "min_sentence2_length": 24, - "average_sentence2_length": 30.5, - "max_sentence2_length": 37, - "unique_sentence2": 2, + "sentence1_statistics": { + "total_text_length": 104, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "sentence2_statistics": { + "total_text_length": 122, + "min_text_length": 24, + "average_text_length": 30.5, + "max_text_length": 37, + "unique_texts": 2, + }, "hf_subset_descriptive_stats": { "eng": { "num_samples": 2, "number_of_characters": 113, "unique_pairs": 2, - "min_sentence1_length": 23, - "average_sentence1_length": 26.0, - "max_sentence1_length": 29, - "unique_sentence1": 2, - "min_sentence2_length": 24, - "average_sentence2_length": 30.5, - "max_sentence2_length": 37, - "unique_sentence2": 2, + "sentence1_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "sentence2_statistics": { + "total_text_length": 61, + "min_text_length": 24, + "average_text_length": 30.5, + "max_text_length": 37, + "unique_texts": 2, + }, }, "fra": { "num_samples": 2, "number_of_characters": 113, "unique_pairs": 2, - "min_sentence1_length": 23, - "average_sentence1_length": 26.0, - "max_sentence1_length": 29, - "unique_sentence1": 2, - "min_sentence2_length": 24, - "average_sentence2_length": 30.5, - "max_sentence2_length": 37, - "unique_sentence2": 2, + "sentence1_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "sentence2_statistics": { + "total_text_length": 61, + "min_text_length": 24, + "average_text_length": 30.5, + "max_text_length": 37, + "unique_texts": 2, + }, }, }, } @@ -441,40 +464,58 @@ class MockMultilingualParallelBitextMiningTask(AbsTaskBitextMining): "num_samples": 4, "number_of_characters": 226, "unique_pairs": 4, - "min_sentence1_length": 23, - "average_sentence1_length": 28.25, - "max_sentence1_length": 37, - "unique_sentence1": 4, - "min_sentence2_length": 23, - "average_sentence2_length": 28.25, - "max_sentence2_length": 37, - "unique_sentence2": 4, + "sentence1_statistics": { + "total_text_length": 113, + "min_text_length": 23, + "average_text_length": 28.25, + "max_text_length": 37, + "unique_texts": 4, + }, + "sentence2_statistics": { + "total_text_length": 113, + "min_text_length": 23, + "average_text_length": 28.25, + "max_text_length": 37, + "unique_texts": 4, + }, "hf_subset_descriptive_stats": { "eng_Latn-fra_Latn": { "num_samples": 2, "number_of_characters": 113, "unique_pairs": 2, - "min_sentence1_length": 23, - "average_sentence1_length": 26.0, - "max_sentence1_length": 29, - "unique_sentence1": 2, - "min_sentence2_length": 24, - "average_sentence2_length": 30.5, - "max_sentence2_length": 37, - "unique_sentence2": 2, + "sentence1_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "sentence2_statistics": { + "total_text_length": 61, + "min_text_length": 24, + "average_text_length": 30.5, + "max_text_length": 37, + "unique_texts": 2, + }, }, "fra_Latn-eng_Latn": { "num_samples": 2, "number_of_characters": 113, "unique_pairs": 2, - "min_sentence1_length": 24, - "average_sentence1_length": 30.5, - "max_sentence1_length": 37, - "unique_sentence1": 2, - "min_sentence2_length": 23, - "average_sentence2_length": 26.0, - "max_sentence2_length": 29, - "unique_sentence2": 2, + "sentence1_statistics": { + "total_text_length": 61, + "min_text_length": 24, + "average_text_length": 30.5, + "max_text_length": 37, + "unique_texts": 2, + }, + "sentence2_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, }, }, } @@ -515,8 +556,9 @@ class MockClusteringTask(AbsTaskAnyClustering): expected_stats = { "test": { "num_samples": 3, - "number_of_characters": 81, + "number_of_characters": 0, "text_statistics": { + "total_text_length": 81, "min_text_length": 23, "average_text_length": 27.0, "max_text_length": 29, @@ -530,7 +572,7 @@ class MockClusteringTask(AbsTaskAnyClustering): "unique_labels": 3, "labels": {"0": {"count": 1}, "1": {"count": 1}, "2": {"count": 1}}, }, - }, + } } metadata = TaskMetadata( @@ -567,75 +609,68 @@ class MockMultilingualClusteringTask(AbsTaskAnyClustering): expected_stats = { "test": { "num_samples": 6, - "number_of_characters": 162, + "number_of_characters": 0, + "text_statistics": { + "total_text_length": 162, + "min_text_length": 23, + "average_text_length": 27.0, + "max_text_length": 29, + "unique_texts": 3, + }, "image_statistics": None, "label_statistics": { + "min_labels_per_text": 1, "average_label_per_text": 1.0, - "labels": { - "0": { - "count": 2, - }, - "1": { - "count": 2, - }, - "2": { - "count": 2, - }, - }, "max_labels_per_text": 1, - "min_labels_per_text": 1, "unique_labels": 3, - }, - "text_statistics": { - "average_text_length": 27.0, - "max_text_length": 29, - "min_text_length": 23, - "unique_texts": 3, + "labels": {"0": {"count": 2}, "1": {"count": 2}, "2": {"count": 2}}, }, "hf_subset_descriptive_stats": { "eng": { + "num_samples": 3, + "number_of_characters": 0, + "text_statistics": { + "total_text_length": 81, + "min_text_length": 23, + "average_text_length": 27.0, + "max_text_length": 29, + "unique_texts": 3, + }, "image_statistics": None, "label_statistics": { + "min_labels_per_text": 1, "average_label_per_text": 1.0, + "max_labels_per_text": 1, + "unique_labels": 3, "labels": { "0": {"count": 1}, "1": {"count": 1}, "2": {"count": 1}, }, - "max_labels_per_text": 1, - "min_labels_per_text": 1, - "unique_labels": 3, }, + }, + "fra": { "num_samples": 3, - "number_of_characters": 81, + "number_of_characters": 0, "text_statistics": { + "total_text_length": 81, + "min_text_length": 23, "average_text_length": 27.0, "max_text_length": 29, - "min_text_length": 23, "unique_texts": 3, }, - }, - "fra": { "image_statistics": None, "label_statistics": { + "min_labels_per_text": 1, "average_label_per_text": 1.0, + "max_labels_per_text": 1, + "unique_labels": 3, "labels": { "0": {"count": 1}, "1": {"count": 1}, "2": {"count": 1}, }, - "max_labels_per_text": 1, - "min_labels_per_text": 1, - "unique_labels": 3, - }, - "text_statistics": { - "average_text_length": 27.0, - "max_text_length": 29, - "min_text_length": 23, - "unique_texts": 3, }, - "num_samples": 3, - "number_of_characters": 81, }, }, } @@ -682,16 +717,20 @@ class MockClusteringFastTask(AbsTaskClusteringFast): expected_stats = { "test": { "num_samples": 3, - "number_of_characters": 81, - "min_text_length": 23, - "average_text_length": 27.0, - "max_text_length": 29, - "unique_texts": 2, - "min_labels_per_text": 1, - "average_labels_per_text": 1.0, - "max_labels_per_text": 1, - "unique_labels": 3, - "labels": {"0": {"count": 1}, "1": {"count": 1}, "2": {"count": 1}}, + "text_statistics": { + "total_text_length": 81, + "min_text_length": 23, + "average_text_length": 27.0, + "max_text_length": 29, + "unique_texts": 3, + }, + "labels_statistics": { + "min_labels_per_text": 1, + "average_label_per_text": 1.0, + "max_labels_per_text": 1, + "unique_labels": 3, + "labels": {"0": {"count": 1}, "1": {"count": 1}, "2": {"count": 1}}, + }, } } @@ -729,42 +768,62 @@ class MockMultilingualClusteringFastTask(AbsTaskClusteringFast): expected_stats = { "test": { "num_samples": 6, - "number_of_characters": 162, - "min_text_length": 23, - "average_text_length": 27.0, - "max_text_length": 29, - "unique_texts": 2, - "min_labels_per_text": 2, - "average_labels_per_text": 1.0, - "max_labels_per_text": 2, - "unique_labels": 3, - "labels": {"0": {"count": 2}, "1": {"count": 2}, "2": {"count": 2}}, + "text_statistics": { + "total_text_length": 162, + "min_text_length": 23, + "average_text_length": 27.0, + "max_text_length": 29, + "unique_texts": 3, + }, + "labels_statistics": { + "min_labels_per_text": 1, + "average_label_per_text": 1.0, + "max_labels_per_text": 1, + "unique_labels": 3, + "labels": {"0": {"count": 2}, "1": {"count": 2}, "2": {"count": 2}}, + }, "hf_subset_descriptive_stats": { "eng": { "num_samples": 3, - "number_of_characters": 81, - "min_text_length": 23, - "average_text_length": 27.0, - "max_text_length": 29, - "unique_texts": 2, - "min_labels_per_text": 1, - "average_labels_per_text": 1.0, - "max_labels_per_text": 1, - "unique_labels": 3, - "labels": {"0": {"count": 1}, "1": {"count": 1}, "2": {"count": 1}}, + "text_statistics": { + "total_text_length": 81, + "min_text_length": 23, + "average_text_length": 27.0, + "max_text_length": 29, + "unique_texts": 3, + }, + "labels_statistics": { + "min_labels_per_text": 1, + "average_label_per_text": 1.0, + "max_labels_per_text": 1, + "unique_labels": 3, + "labels": { + "0": {"count": 1}, + "1": {"count": 1}, + "2": {"count": 1}, + }, + }, }, "fra": { "num_samples": 3, - "number_of_characters": 81, - "min_text_length": 23, - "average_text_length": 27.0, - "max_text_length": 29, - "unique_texts": 2, - "min_labels_per_text": 1, - "average_labels_per_text": 1.0, - "max_labels_per_text": 1, - "unique_labels": 3, - "labels": {"0": {"count": 1}, "1": {"count": 1}, "2": {"count": 1}}, + "text_statistics": { + "total_text_length": 81, + "min_text_length": 23, + "average_text_length": 27.0, + "max_text_length": 29, + "unique_texts": 3, + }, + "labels_statistics": { + "min_labels_per_text": 1, + "average_label_per_text": 1.0, + "max_labels_per_text": 1, + "unique_labels": 3, + "labels": { + "0": {"count": 1}, + "1": {"count": 1}, + "2": {"count": 1}, + }, + }, }, }, } @@ -809,16 +868,27 @@ class MockPairClassificationTask(AbsTaskPairClassification): "num_samples": 2, "number_of_characters": 113, "unique_pairs": 2, - "min_sentence1_length": 23, - "avg_sentence1_length": 26.0, - "max_sentence1_length": 29, - "unique_sentence1": 2, - "min_sentence2_length": 24, - "avg_sentence2_length": 30.5, - "max_sentence2_length": 37, - "unique_sentence2": 2, - "unique_labels": 2, - "labels": {"1": {"count": 1}, "0": {"count": 1}}, + "text1_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "text2_statistics": { + "total_text_length": 61, + "min_text_length": 24, + "average_text_length": 30.5, + "max_text_length": 37, + "unique_texts": 2, + }, + "labels_statistics": { + "min_labels_per_text": 1, + "average_label_per_text": 1.0, + "max_labels_per_text": 1, + "unique_labels": 2, + "labels": {"1": {"count": 1}, "0": {"count": 1}}, + }, } } @@ -859,46 +929,79 @@ class MockMultilingualPairClassificationTask(AbsTaskPairClassification): "num_samples": 4, "number_of_characters": 226, "unique_pairs": 2, - "min_sentence1_length": 23, - "avg_sentence1_length": 26.0, - "max_sentence1_length": 29, - "unique_sentence1": 2, - "min_sentence2_length": 24, - "avg_sentence2_length": 30.5, - "max_sentence2_length": 37, - "unique_sentence2": 2, - "unique_labels": 2, - "labels": {"1": {"count": 2}, "0": {"count": 2}}, + "text1_statistics": { + "total_text_length": 104, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "text2_statistics": { + "total_text_length": 122, + "min_text_length": 24, + "average_text_length": 30.5, + "max_text_length": 37, + "unique_texts": 2, + }, + "labels_statistics": { + "min_labels_per_text": 1, + "average_label_per_text": 1.0, + "max_labels_per_text": 1, + "unique_labels": 2, + "labels": {"1": {"count": 2}, "0": {"count": 2}}, + }, "hf_subset_descriptive_stats": { "eng": { "num_samples": 2, "number_of_characters": 113, "unique_pairs": 2, - "min_sentence1_length": 23, - "avg_sentence1_length": 26.0, - "max_sentence1_length": 29, - "unique_sentence1": 2, - "min_sentence2_length": 24, - "avg_sentence2_length": 30.5, - "max_sentence2_length": 37, - "unique_sentence2": 2, - "unique_labels": 2, - "labels": {"1": {"count": 1}, "0": {"count": 1}}, + "text1_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "text2_statistics": { + "total_text_length": 61, + "min_text_length": 24, + "average_text_length": 30.5, + "max_text_length": 37, + "unique_texts": 2, + }, + "labels_statistics": { + "min_labels_per_text": 1, + "average_label_per_text": 1.0, + "max_labels_per_text": 1, + "unique_labels": 2, + "labels": {"1": {"count": 1}, "0": {"count": 1}}, + }, }, "fra": { "num_samples": 2, "number_of_characters": 113, "unique_pairs": 2, - "min_sentence1_length": 23, - "avg_sentence1_length": 26.0, - "max_sentence1_length": 29, - "unique_sentence1": 2, - "min_sentence2_length": 24, - "avg_sentence2_length": 30.5, - "max_sentence2_length": 37, - "unique_sentence2": 2, - "unique_labels": 2, - "labels": {"1": {"count": 1}, "0": {"count": 1}}, + "text1_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "text2_statistics": { + "total_text_length": 61, + "min_text_length": 24, + "average_text_length": 30.5, + "max_text_length": 37, + "unique_texts": 2, + }, + "labels_statistics": { + "min_labels_per_text": 1, + "average_label_per_text": 1.0, + "max_labels_per_text": 1, + "unique_labels": 2, + "labels": {"1": {"count": 1}, "0": {"count": 1}}, + }, }, }, } @@ -946,15 +1049,17 @@ class MockSTSTask(AbsTaskAnySTS): "number_of_characters": 113, "unique_pairs": 2, "text1_statistics": { + "total_text_length": 52, "min_text_length": 23, "average_text_length": 26.0, "max_text_length": 29, "unique_texts": 2, }, "text2_statistics": { + "total_text_length": 61, "min_text_length": 24, - "max_text_length": 37, "average_text_length": 30.5, + "max_text_length": 37, "unique_texts": 2, }, "image1_statistics": None, @@ -1002,15 +1107,17 @@ class MockMultilingualSTSTask(AbsTaskAnySTS): "number_of_characters": 226, "unique_pairs": 2, "text1_statistics": { + "total_text_length": 104, "min_text_length": 23, "average_text_length": 26.0, "max_text_length": 29, "unique_texts": 2, }, "text2_statistics": { + "total_text_length": 122, "min_text_length": 24, - "max_text_length": 37, "average_text_length": 30.5, + "max_text_length": 37, "unique_texts": 2, }, "image1_statistics": None, @@ -1022,15 +1129,17 @@ class MockMultilingualSTSTask(AbsTaskAnySTS): "number_of_characters": 113, "unique_pairs": 2, "text1_statistics": { + "total_text_length": 52, "min_text_length": 23, "average_text_length": 26.0, "max_text_length": 29, "unique_texts": 2, }, "text2_statistics": { + "total_text_length": 61, "min_text_length": 24, - "max_text_length": 37, "average_text_length": 30.5, + "max_text_length": 37, "unique_texts": 2, }, "image1_statistics": None, @@ -1046,15 +1155,17 @@ class MockMultilingualSTSTask(AbsTaskAnySTS): "number_of_characters": 113, "unique_pairs": 2, "text1_statistics": { + "total_text_length": 52, "min_text_length": 23, "average_text_length": 26.0, "max_text_length": 29, "unique_texts": 2, }, "text2_statistics": { + "total_text_length": 61, "min_text_length": 24, - "max_text_length": 37, "average_text_length": 30.5, + "max_text_length": 37, "unique_texts": 2, }, "image1_statistics": None, @@ -1108,22 +1219,29 @@ class MockSummarizationTask(AbsTaskSummarization): expected_stats = { "test": { "num_samples": 2, - "number_of_characters": 60, - "min_text_length": 23, - "avg_text_length": 26.0, - "max_text_length": 29, - "unique_texts": 2, - "min_human_summaries_length": 2, - "avg_human_summaries_length": 2.0, - "max_human_summaries_length": 2, - "unique_human_summaries": 2, - "min_machine_summaries_length": 2, - "avg_machine_summaries_length": 2.0, - "max_machine_summaries_length": 2, - "unique_machine_summaries": 2, - "min_relevance": [0, 1], - "avg_relevance": 0.5, - "max_relevance": [1, 0], + "number_of_characters": 244, + "text_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "human_summaries_statistics": { + "total_text_length": 80, + "min_text_length": 17, + "average_text_length": 20.0, + "max_text_length": 23, + "unique_texts": 2, + }, + "machine_summaries_statistics": { + "total_text_length": 112, + "min_text_length": 25, + "average_text_length": 28.0, + "max_text_length": 31, + "unique_texts": 2, + }, + "score_statistics": {"min_score": 0, "avg_score": 0.5, "max_score": 1}, } } @@ -1168,60 +1286,89 @@ class MockMultilingualSummarizationTask(AbsTaskSummarization): expected_stats = { "test": { "num_samples": 4, - "number_of_characters": 120, - "min_text_length": 23, - "avg_text_length": 26.0, - "max_text_length": 29, - "unique_texts": 2, - "min_human_summaries_length": 2, - "avg_human_summaries_length": 2.0, - "max_human_summaries_length": 2, - "unique_human_summaries": 2, - "min_machine_summaries_length": 2, - "avg_machine_summaries_length": 2.0, - "max_machine_summaries_length": 2, - "unique_machine_summaries": 2, - "min_relevance": [0, 1], - "avg_relevance": 0.5, - "max_relevance": [1, 0], + "number_of_characters": 488, + "text_statistics": { + "total_text_length": 104, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "human_summaries_statistics": { + "total_text_length": 160, + "min_text_length": 17, + "average_text_length": 20.0, + "max_text_length": 23, + "unique_texts": 2, + }, + "machine_summaries_statistics": { + "total_text_length": 224, + "min_text_length": 25, + "average_text_length": 28.0, + "max_text_length": 31, + "unique_texts": 2, + }, + "score_statistics": {"min_score": 0, "avg_score": 0.5, "max_score": 1}, "hf_subset_descriptive_stats": { "eng": { "num_samples": 2, - "number_of_characters": 60, - "min_text_length": 23, - "avg_text_length": 26.0, - "max_text_length": 29, - "unique_texts": 2, - "min_human_summaries_length": 2, - "avg_human_summaries_length": 2.0, - "max_human_summaries_length": 2, - "unique_human_summaries": 2, - "min_machine_summaries_length": 2, - "avg_machine_summaries_length": 2.0, - "max_machine_summaries_length": 2, - "unique_machine_summaries": 2, - "min_relevance": [0, 1], - "avg_relevance": 0.5, - "max_relevance": [1, 0], + "number_of_characters": 244, + "text_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "human_summaries_statistics": { + "total_text_length": 80, + "min_text_length": 17, + "average_text_length": 20.0, + "max_text_length": 23, + "unique_texts": 2, + }, + "machine_summaries_statistics": { + "total_text_length": 112, + "min_text_length": 25, + "average_text_length": 28.0, + "max_text_length": 31, + "unique_texts": 2, + }, + "score_statistics": { + "min_score": 0, + "avg_score": 0.5, + "max_score": 1, + }, }, "fra": { "num_samples": 2, - "number_of_characters": 60, - "min_text_length": 23, - "avg_text_length": 26.0, - "max_text_length": 29, - "unique_texts": 2, - "min_human_summaries_length": 2, - "avg_human_summaries_length": 2.0, - "max_human_summaries_length": 2, - "unique_human_summaries": 2, - "min_machine_summaries_length": 2, - "avg_machine_summaries_length": 2.0, - "max_machine_summaries_length": 2, - "unique_machine_summaries": 2, - "min_relevance": [0, 1], - "avg_relevance": 0.5, - "max_relevance": [1, 0], + "number_of_characters": 244, + "text_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "human_summaries_statistics": { + "total_text_length": 80, + "min_text_length": 17, + "average_text_length": 20.0, + "max_text_length": 23, + "unique_texts": 2, + }, + "machine_summaries_statistics": { + "total_text_length": 112, + "min_text_length": 25, + "average_text_length": 28.0, + "max_text_length": 31, + "unique_texts": 2, + }, + "score_statistics": { + "min_score": 0, + "avg_score": 0.5, + "max_score": 1, + }, }, }, } @@ -1273,31 +1420,34 @@ class MockRerankingTask(AbsTaskRetrieval): "test": { "num_samples": 4, "number_of_characters": 112, - "num_documents": 2, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 2, - "num_queries": 2, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 2, - "none_queries": 0, - "num_relevant_docs": 4, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 2, - "num_instructions": None, - "min_instruction_length": None, - "average_instruction_length": None, - "max_instruction_length": None, - "unique_instructions": None, - "num_top_ranked": 2, - "min_top_ranked_per_query": 2, - "average_top_ranked_per_query": 2.0, - "max_top_ranked_per_query": 2, + "documents_statistics": { + "total_text_length": 60, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + "instructions_statistics": None, + "top_ranked_statistics": { + "num_top_ranked": 4, + "min_top_ranked_per_query": 2, + "average_top_ranked_per_query": 2.0, + "max_top_ranked_per_query": 2, + }, } } @@ -1321,89 +1471,98 @@ class MockMultilingualRerankingTask(AbsTaskRetrieval): "test": { "num_samples": 8, "number_of_characters": 224, - "num_documents": 4, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 4, - "num_queries": 4, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 4, - "none_queries": 0, - "num_relevant_docs": 8, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 4, - "num_instructions": None, - "min_instruction_length": None, - "average_instruction_length": None, - "max_instruction_length": None, - "unique_instructions": None, - "num_top_ranked": 4, - "min_top_ranked_per_query": 2, - "average_top_ranked_per_query": 2.0, - "max_top_ranked_per_query": 2, + "documents_statistics": { + "total_text_length": 120, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 104, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 4, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 4, + }, + "instructions_statistics": None, + "top_ranked_statistics": { + "num_top_ranked": 8, + "min_top_ranked_per_query": 2, + "average_top_ranked_per_query": 2.0, + "max_top_ranked_per_query": 2, + }, "hf_subset_descriptive_stats": { "eng": { "num_samples": 4, "number_of_characters": 112, - "num_documents": 2, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 2, - "num_queries": 2, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 2, - "none_queries": 0, - "num_relevant_docs": 4, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 2, - "num_instructions": None, - "min_instruction_length": None, - "average_instruction_length": None, - "max_instruction_length": None, - "unique_instructions": None, - "num_top_ranked": 2, - "min_top_ranked_per_query": 2, - "average_top_ranked_per_query": 2.0, - "max_top_ranked_per_query": 2, + "documents_statistics": { + "total_text_length": 60, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + "instructions_statistics": None, + "top_ranked_statistics": { + "num_top_ranked": 4, + "min_top_ranked_per_query": 2, + "average_top_ranked_per_query": 2.0, + "max_top_ranked_per_query": 2, + }, }, "fra": { "num_samples": 4, "number_of_characters": 112, - "num_documents": 2, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 2, - "num_queries": 2, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 2, - "none_queries": 0, - "num_relevant_docs": 4, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 2, - "num_instructions": None, - "min_instruction_length": None, - "average_instruction_length": None, - "max_instruction_length": None, - "unique_instructions": None, - "num_top_ranked": 2, - "min_top_ranked_per_query": 2, - "average_top_ranked_per_query": 2.0, - "max_top_ranked_per_query": 2, + "documents_statistics": { + "total_text_length": 60, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + "instructions_statistics": None, + "top_ranked_statistics": { + "num_top_ranked": 4, + "min_top_ranked_per_query": 2, + "average_top_ranked_per_query": 2.0, + "max_top_ranked_per_query": 2, + }, }, }, } @@ -1432,60 +1591,56 @@ class MockRetrievalTask(AbsTaskRetrieval): "val": { "num_samples": 4, "number_of_characters": 112, - "num_documents": 2, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 2, - "num_queries": 2, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 2, - "none_queries": 0, - "num_relevant_docs": 4, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 2, - "num_instructions": None, - "min_instruction_length": None, - "average_instruction_length": None, - "max_instruction_length": None, - "unique_instructions": None, - "num_top_ranked": None, - "min_top_ranked_per_query": None, - "average_top_ranked_per_query": None, - "max_top_ranked_per_query": None, + "documents_statistics": { + "total_text_length": 60, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + "instructions_statistics": None, + "top_ranked_statistics": None, }, "test": { "num_samples": 4, "number_of_characters": 112, - "num_documents": 2, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 2, - "num_queries": 2, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 2, - "none_queries": 0, - "num_relevant_docs": 4, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 2, - "num_instructions": None, - "min_instruction_length": None, - "average_instruction_length": None, - "max_instruction_length": None, - "unique_instructions": None, - "num_top_ranked": None, - "min_top_ranked_per_query": None, - "average_top_ranked_per_query": None, - "max_top_ranked_per_query": None, + "documents_statistics": { + "total_text_length": 60, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + "instructions_statistics": None, + "top_ranked_statistics": None, }, } @@ -1512,60 +1667,56 @@ class MockRetrievalDialogTask(AbsTaskRetrieval): "val": { "num_samples": 4, "number_of_characters": 112, - "num_documents": 2, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 2, - "num_queries": 2, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 2, - "none_queries": 0, - "num_relevant_docs": 4, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 2, - "num_instructions": None, - "min_instruction_length": None, - "average_instruction_length": None, - "max_instruction_length": None, - "unique_instructions": None, - "num_top_ranked": None, - "min_top_ranked_per_query": None, - "average_top_ranked_per_query": None, - "max_top_ranked_per_query": None, + "documents_statistics": { + "total_text_length": 60, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + "instructions_statistics": None, + "top_ranked_statistics": None, }, "test": { "num_samples": 4, "number_of_characters": 112, - "num_documents": 2, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 2, - "num_queries": 2, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 2, - "none_queries": 0, - "num_relevant_docs": 4, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 2, - "num_instructions": None, - "min_instruction_length": None, - "average_instruction_length": None, - "max_instruction_length": None, - "unique_instructions": None, - "num_top_ranked": None, - "min_top_ranked_per_query": None, - "average_top_ranked_per_query": None, - "max_top_ranked_per_query": None, + "documents_statistics": { + "total_text_length": 60, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + "instructions_statistics": None, + "top_ranked_statistics": None, }, } @@ -1604,178 +1755,166 @@ class MockMultilingualRetrievalTask(AbsTaskRetrieval): "val": { "num_samples": 8, "number_of_characters": 224, - "num_documents": 4, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 4, - "num_queries": 4, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 4, - "none_queries": 0, - "num_relevant_docs": 8, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 4, - "num_instructions": None, - "min_instruction_length": None, - "average_instruction_length": None, - "max_instruction_length": None, - "unique_instructions": None, - "num_top_ranked": None, - "min_top_ranked_per_query": None, - "average_top_ranked_per_query": None, - "max_top_ranked_per_query": None, + "documents_statistics": { + "total_text_length": 120, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 104, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 4, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 4, + }, + "instructions_statistics": None, + "top_ranked_statistics": None, "hf_subset_descriptive_stats": { "eng": { "num_samples": 4, "number_of_characters": 112, - "num_documents": 2, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 2, - "num_queries": 2, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 2, - "none_queries": 0, - "num_relevant_docs": 4, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 2, - "num_instructions": None, - "min_instruction_length": None, - "average_instruction_length": None, - "max_instruction_length": None, - "unique_instructions": None, - "num_top_ranked": None, - "min_top_ranked_per_query": None, - "average_top_ranked_per_query": None, - "max_top_ranked_per_query": None, + "documents_statistics": { + "total_text_length": 60, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + "instructions_statistics": None, + "top_ranked_statistics": None, }, "fra": { "num_samples": 4, "number_of_characters": 112, - "num_documents": 2, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 2, - "num_queries": 2, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 2, - "none_queries": 0, - "num_relevant_docs": 4, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 2, - "num_instructions": None, - "min_instruction_length": None, - "average_instruction_length": None, - "max_instruction_length": None, - "unique_instructions": None, - "num_top_ranked": None, - "min_top_ranked_per_query": None, - "average_top_ranked_per_query": None, - "max_top_ranked_per_query": None, + "documents_statistics": { + "total_text_length": 60, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + "instructions_statistics": None, + "top_ranked_statistics": None, }, }, }, "test": { "num_samples": 8, "number_of_characters": 224, - "num_documents": 4, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 4, - "num_queries": 4, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 4, - "none_queries": 0, - "num_relevant_docs": 8, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 4, - "num_instructions": None, - "min_instruction_length": None, - "average_instruction_length": None, - "max_instruction_length": None, - "unique_instructions": None, - "num_top_ranked": None, - "min_top_ranked_per_query": None, - "average_top_ranked_per_query": None, - "max_top_ranked_per_query": None, + "documents_statistics": { + "total_text_length": 120, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 104, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 4, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 4, + }, + "instructions_statistics": None, + "top_ranked_statistics": None, "hf_subset_descriptive_stats": { "eng": { "num_samples": 4, "number_of_characters": 112, - "num_documents": 2, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 2, - "num_queries": 2, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 2, - "none_queries": 0, - "num_relevant_docs": 4, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 2, - "num_instructions": None, - "min_instruction_length": None, - "average_instruction_length": None, - "max_instruction_length": None, - "unique_instructions": None, - "num_top_ranked": None, - "min_top_ranked_per_query": None, - "average_top_ranked_per_query": None, - "max_top_ranked_per_query": None, + "documents_statistics": { + "total_text_length": 60, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + "instructions_statistics": None, + "top_ranked_statistics": None, }, "fra": { "num_samples": 4, "number_of_characters": 112, - "num_documents": 2, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 2, - "num_queries": 2, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 2, - "none_queries": 0, - "num_relevant_docs": 4, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 2, - "num_instructions": None, - "min_instruction_length": None, - "average_instruction_length": None, - "max_instruction_length": None, - "unique_instructions": None, - "num_top_ranked": None, - "min_top_ranked_per_query": None, - "average_top_ranked_per_query": None, - "max_top_ranked_per_query": None, + "documents_statistics": { + "total_text_length": 60, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + "instructions_statistics": None, + "top_ranked_statistics": None, }, }, }, @@ -1804,9 +1943,9 @@ class MockMultilabelClassification(AbsTaskMultilabelClassification): expected_stats = { "test": { "num_samples": 6, - "number_of_characters": 156, "number_texts_intersect_with_train": 1, "text_statistics": { + "total_text_length": 156, "min_text_length": 23, "average_text_length": 26.0, "max_text_length": 29, @@ -1823,9 +1962,9 @@ class MockMultilabelClassification(AbsTaskMultilabelClassification): }, "train": { "num_samples": 6, - "number_of_characters": 159, "number_texts_intersect_with_train": None, "text_statistics": { + "total_text_length": 159, "min_text_length": 23, "average_text_length": 26.5, "max_text_length": 30, @@ -1877,9 +2016,9 @@ class MockMultilingualMultilabelClassification(AbsTaskMultilabelClassification): expected_stats = { "test": { "num_samples": 12, - "number_of_characters": 312, "number_texts_intersect_with_train": 1, "text_statistics": { + "total_text_length": 312, "min_text_length": 23, "average_text_length": 26.0, "max_text_length": 29, @@ -1896,9 +2035,9 @@ class MockMultilingualMultilabelClassification(AbsTaskMultilabelClassification): "hf_subset_descriptive_stats": { "eng": { "num_samples": 6, - "number_of_characters": 156, "number_texts_intersect_with_train": 1, "text_statistics": { + "total_text_length": 156, "min_text_length": 23, "average_text_length": 26.0, "max_text_length": 29, @@ -1915,9 +2054,9 @@ class MockMultilingualMultilabelClassification(AbsTaskMultilabelClassification): }, "fra": { "num_samples": 6, - "number_of_characters": 156, "number_texts_intersect_with_train": 1, "text_statistics": { + "total_text_length": 156, "min_text_length": 23, "average_text_length": 26.0, "max_text_length": 29, @@ -1936,9 +2075,9 @@ class MockMultilingualMultilabelClassification(AbsTaskMultilabelClassification): }, "train": { "num_samples": 12, - "number_of_characters": 318, "number_texts_intersect_with_train": None, "text_statistics": { + "total_text_length": 318, "min_text_length": 23, "average_text_length": 26.5, "max_text_length": 30, @@ -1955,9 +2094,9 @@ class MockMultilingualMultilabelClassification(AbsTaskMultilabelClassification): "hf_subset_descriptive_stats": { "eng": { "num_samples": 6, - "number_of_characters": 159, "number_texts_intersect_with_train": None, "text_statistics": { + "total_text_length": 159, "min_text_length": 23, "average_text_length": 26.5, "max_text_length": 30, @@ -1974,9 +2113,9 @@ class MockMultilingualMultilabelClassification(AbsTaskMultilabelClassification): }, "fra": { "num_samples": 6, - "number_of_characters": 159, "number_texts_intersect_with_train": None, "text_statistics": { + "total_text_length": 159, "min_text_length": 23, "average_text_length": 26.5, "max_text_length": 30, @@ -2036,32 +2175,36 @@ class MockInstructionRetrieval(AbsTaskRetrieval): expected_stats = { "test": { "num_samples": 4, - "number_of_characters": 112, - "num_documents": 2, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 2, - "num_queries": 2, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 2, - "none_queries": 0, - "num_relevant_docs": 4, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 2, - "num_instructions": 2, - "min_instruction_length": 26, - "average_instruction_length": 58, - "max_instruction_length": 32, - "unique_instructions": 2, - "num_top_ranked": None, - "min_top_ranked_per_query": None, - "average_top_ranked_per_query": None, - "max_top_ranked_per_query": None, + "number_of_characters": 170, + "documents_statistics": { + "total_text_length": 60, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + "instructions_statistics": { + "total_text_length": 58, + "min_text_length": 26, + "average_text_length": 29.0, + "max_text_length": 32, + "unique_texts": 2, + }, + "top_ranked_statistics": None, } } @@ -2084,32 +2227,41 @@ class MockInstructionReranking(AbsTaskRetrieval): expected_stats = { "test": { "num_samples": 4, - "number_of_characters": 112, - "num_documents": 2, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 2, - "num_queries": 2, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 2, - "none_queries": 0, - "num_relevant_docs": 4, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 2, - "num_instructions": 2, - "min_instruction_length": 26, - "average_instruction_length": 58, - "max_instruction_length": 32, - "unique_instructions": 2, - "num_top_ranked": 2, - "min_top_ranked_per_query": 2, - "average_top_ranked_per_query": 2.0, - "max_top_ranked_per_query": 2, + "number_of_characters": 170, + "documents_statistics": { + "total_text_length": 60, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + "instructions_statistics": { + "total_text_length": 58, + "min_text_length": 26, + "average_text_length": 29.0, + "max_text_length": 32, + "unique_texts": 2, + }, + "top_ranked_statistics": { + "num_top_ranked": 4, + "min_top_ranked_per_query": 2, + "average_top_ranked_per_query": 2.0, + "max_top_ranked_per_query": 2, + }, } } @@ -2131,90 +2283,102 @@ class MockMultilingualInstructionRetrieval(AbsTaskRetrieval): expected_stats = { "test": { "num_samples": 8, - "number_of_characters": 224, - "num_documents": 4, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 4, - "num_queries": 4, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 4, - "none_queries": 0, - "num_relevant_docs": 8, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 4, - "num_instructions": 4, - "min_instruction_length": 26, - "average_instruction_length": 116, - "max_instruction_length": 32, - "unique_instructions": 4, - "num_top_ranked": None, - "min_top_ranked_per_query": None, - "average_top_ranked_per_query": None, - "max_top_ranked_per_query": None, + "number_of_characters": 340, + "documents_statistics": { + "total_text_length": 120, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 104, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 4, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 4, + }, + "instructions_statistics": { + "total_text_length": 116, + "min_text_length": 26, + "average_text_length": 29.0, + "max_text_length": 32, + "unique_texts": 2, + }, + "top_ranked_statistics": None, "hf_subset_descriptive_stats": { "eng": { "num_samples": 4, - "number_of_characters": 112, - "num_documents": 2, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 2, - "num_queries": 2, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 2, - "none_queries": 0, - "num_relevant_docs": 4, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 2, - "num_instructions": 2, - "min_instruction_length": 26, - "average_instruction_length": 58, - "max_instruction_length": 32, - "unique_instructions": 2, - "num_top_ranked": None, - "min_top_ranked_per_query": None, - "average_top_ranked_per_query": None, - "max_top_ranked_per_query": None, + "number_of_characters": 170, + "documents_statistics": { + "total_text_length": 60, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + "instructions_statistics": { + "total_text_length": 58, + "min_text_length": 26, + "average_text_length": 29.0, + "max_text_length": 32, + "unique_texts": 2, + }, + "top_ranked_statistics": None, }, "fra": { "num_samples": 4, - "number_of_characters": 112, - "num_documents": 2, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 2, - "num_queries": 2, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 2, - "none_queries": 0, - "num_relevant_docs": 4, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 2, - "num_instructions": 2, - "min_instruction_length": 26, - "average_instruction_length": 58, - "max_instruction_length": 32, - "unique_instructions": 2, - "num_top_ranked": None, - "min_top_ranked_per_query": None, - "average_top_ranked_per_query": None, - "max_top_ranked_per_query": None, + "number_of_characters": 170, + "documents_statistics": { + "total_text_length": 60, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + "instructions_statistics": { + "total_text_length": 58, + "min_text_length": 26, + "average_text_length": 29.0, + "max_text_length": 32, + "unique_texts": 2, + }, + "top_ranked_statistics": None, }, }, } @@ -2241,90 +2405,117 @@ class MockMultilingualInstructionReranking(AbsTaskRetrieval): expected_stats = { "test": { "num_samples": 8, - "number_of_characters": 224, - "num_documents": 4, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 4, - "num_queries": 4, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 4, - "none_queries": 0, - "num_relevant_docs": 8, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 4, - "num_instructions": 4, - "min_instruction_length": 26, - "average_instruction_length": 116, - "max_instruction_length": 32, - "unique_instructions": 4, - "num_top_ranked": 4, - "min_top_ranked_per_query": 2, - "average_top_ranked_per_query": 2.0, - "max_top_ranked_per_query": 2, + "number_of_characters": 340, + "documents_statistics": { + "total_text_length": 120, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 104, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 4, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 4, + }, + "instructions_statistics": { + "total_text_length": 116, + "min_text_length": 26, + "average_text_length": 29.0, + "max_text_length": 32, + "unique_texts": 2, + }, + "top_ranked_statistics": { + "num_top_ranked": 8, + "min_top_ranked_per_query": 2, + "average_top_ranked_per_query": 2.0, + "max_top_ranked_per_query": 2, + }, "hf_subset_descriptive_stats": { "eng": { "num_samples": 4, - "number_of_characters": 112, - "num_documents": 2, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 2, - "num_queries": 2, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 2, - "none_queries": 0, - "num_relevant_docs": 4, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 2, - "num_instructions": 2, - "min_instruction_length": 26, - "average_instruction_length": 58, - "max_instruction_length": 32, - "unique_instructions": 2, - "num_top_ranked": 2, - "min_top_ranked_per_query": 2, - "average_top_ranked_per_query": 2.0, - "max_top_ranked_per_query": 2, + "number_of_characters": 170, + "documents_statistics": { + "total_text_length": 60, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + "instructions_statistics": { + "total_text_length": 58, + "min_text_length": 26, + "average_text_length": 29.0, + "max_text_length": 32, + "unique_texts": 2, + }, + "top_ranked_statistics": { + "num_top_ranked": 4, + "min_top_ranked_per_query": 2, + "average_top_ranked_per_query": 2.0, + "max_top_ranked_per_query": 2, + }, }, "fra": { "num_samples": 4, - "number_of_characters": 112, - "num_documents": 2, - "min_document_length": 27, - "average_document_length": 30.0, - "max_document_length": 33, - "unique_documents": 2, - "num_queries": 2, - "min_query_length": 23, - "average_query_length": 26.0, - "max_query_length": 29, - "unique_queries": 2, - "none_queries": 0, - "num_relevant_docs": 4, - "min_relevant_docs_per_query": 2, - "average_relevant_docs_per_query": 1.0, - "max_relevant_docs_per_query": 2, - "unique_relevant_docs": 2, - "num_instructions": 2, - "min_instruction_length": 26, - "average_instruction_length": 58, - "max_instruction_length": 32, - "unique_instructions": 2, - "num_top_ranked": 2, - "min_top_ranked_per_query": 2, - "average_top_ranked_per_query": 2.0, - "max_top_ranked_per_query": 2, + "number_of_characters": 170, + "documents_statistics": { + "total_text_length": 60, + "min_text_length": 27, + "average_text_length": 30.0, + "max_text_length": 33, + "unique_texts": 2, + }, + "queries_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, + "relevant_docs_statistics": { + "num_relevant_docs": 2, + "min_relevant_docs_per_query": 2, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 2, + "unique_relevant_docs": 2, + }, + "instructions_statistics": { + "total_text_length": 58, + "min_text_length": 26, + "average_text_length": 29.0, + "max_text_length": 32, + "unique_texts": 2, + }, + "top_ranked_statistics": { + "num_top_ranked": 4, + "min_top_ranked_per_query": 2, + "average_top_ranked_per_query": 2.0, + "max_top_ranked_per_query": 2, + }, }, }, } @@ -2393,7 +2584,7 @@ class MockMultiChoiceTask(AbsTaskAny2AnyMultiChoice): metadata.category = "it2i" def load_data(self, **kwargs): - images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] # noqa: NPY002 + images = [self.np_rng.integers(0, 255, (100, 100, 3)) for _ in range(2)] images = [ Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images ] @@ -2468,7 +2659,7 @@ class MockMultilingualMultiChoiceTask(AbsTaskAny2AnyMultiChoice): metadata.category = "it2i" def load_data(self, **kwargs): - images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] # noqa: NPY002 + images = [self.np_rng.integers(0, 255, (100, 100, 3)) for _ in range(2)] images = [ Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images ] @@ -2553,7 +2744,7 @@ class MockAny2AnyRetrievalI2TTask(AbsTaskAny2AnyRetrieval): metadata.category = "i2t" def load_data(self, **kwargs): - images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] # noqa: NPY002 + images = [self.np_rng.integers(0, 255, (100, 100, 3)) for _ in range(2)] images = [ Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images ] @@ -2623,7 +2814,7 @@ class MockAny2AnyRetrievalT2ITask(AbsTaskAny2AnyRetrieval): metadata.category = "t2i" def load_data(self, **kwargs): - images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] # noqa: NPY002 + images = [self.np_rng.integers(0, 255, (100, 100, 3)) for _ in range(2)] images = [ Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images ] @@ -2663,7 +2854,6 @@ class MockImageClassificationTask(AbsTaskAnyClassification): expected_stats = { "test": { "num_samples": 2, - "number_of_characters": 0, "number_texts_intersect_with_train": None, "text_statistics": None, "image_statistics": { @@ -2673,6 +2863,7 @@ class MockImageClassificationTask(AbsTaskAnyClassification): "min_image_height": 100, "average_image_height": 100.0, "max_image_height": 100, + "unique_images": 2, }, "label_statistics": { "min_labels_per_text": 1, @@ -2684,7 +2875,6 @@ class MockImageClassificationTask(AbsTaskAnyClassification): }, "train": { "num_samples": 10, - "number_of_characters": 0, "number_texts_intersect_with_train": None, "text_statistics": None, "image_statistics": { @@ -2694,6 +2884,7 @@ class MockImageClassificationTask(AbsTaskAnyClassification): "min_image_height": 100, "average_image_height": 100.0, "max_image_height": 100, + "unique_images": 10, }, "label_statistics": { "min_labels_per_text": 1, @@ -2718,7 +2909,7 @@ class MockImageClassificationTask(AbsTaskAnyClassification): input_column_name = "image" def load_data(self, **kwargs): - images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] # noqa: NPY002 + images = [self.np_rng.integers(0, 255, (100, 100, 3)) for _ in range(2)] images = [ Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images ] @@ -2749,7 +2940,6 @@ class MockMultilingualImageClassificationTask(AbsTaskAnyClassification): expected_stats = { "test": { "num_samples": 4, - "number_of_characters": 0, "number_texts_intersect_with_train": None, "text_statistics": None, "image_statistics": { @@ -2759,6 +2949,7 @@ class MockMultilingualImageClassificationTask(AbsTaskAnyClassification): "min_image_height": 100, "average_image_height": 100.0, "max_image_height": 100, + "unique_images": 4, }, "label_statistics": { "min_labels_per_text": 1, @@ -2770,7 +2961,6 @@ class MockMultilingualImageClassificationTask(AbsTaskAnyClassification): "hf_subset_descriptive_stats": { "eng": { "num_samples": 2, - "number_of_characters": 0, "number_texts_intersect_with_train": None, "text_statistics": None, "image_statistics": { @@ -2780,6 +2970,7 @@ class MockMultilingualImageClassificationTask(AbsTaskAnyClassification): "min_image_height": 100, "average_image_height": 100.0, "max_image_height": 100, + "unique_images": 2, }, "label_statistics": { "min_labels_per_text": 1, @@ -2791,7 +2982,6 @@ class MockMultilingualImageClassificationTask(AbsTaskAnyClassification): }, "fra": { "num_samples": 2, - "number_of_characters": 0, "number_texts_intersect_with_train": None, "text_statistics": None, "image_statistics": { @@ -2801,6 +2991,7 @@ class MockMultilingualImageClassificationTask(AbsTaskAnyClassification): "min_image_height": 100, "average_image_height": 100.0, "max_image_height": 100, + "unique_images": 2, }, "label_statistics": { "min_labels_per_text": 1, @@ -2814,7 +3005,6 @@ class MockMultilingualImageClassificationTask(AbsTaskAnyClassification): }, "train": { "num_samples": 20, - "number_of_characters": 0, "number_texts_intersect_with_train": None, "text_statistics": None, "image_statistics": { @@ -2824,6 +3014,7 @@ class MockMultilingualImageClassificationTask(AbsTaskAnyClassification): "min_image_height": 100, "average_image_height": 100.0, "max_image_height": 100, + "unique_images": 20, }, "label_statistics": { "min_labels_per_text": 1, @@ -2835,7 +3026,6 @@ class MockMultilingualImageClassificationTask(AbsTaskAnyClassification): "hf_subset_descriptive_stats": { "eng": { "num_samples": 10, - "number_of_characters": 0, "number_texts_intersect_with_train": None, "text_statistics": None, "image_statistics": { @@ -2845,6 +3035,7 @@ class MockMultilingualImageClassificationTask(AbsTaskAnyClassification): "min_image_height": 100, "average_image_height": 100.0, "max_image_height": 100, + "unique_images": 10, }, "label_statistics": { "min_labels_per_text": 1, @@ -2856,7 +3047,6 @@ class MockMultilingualImageClassificationTask(AbsTaskAnyClassification): }, "fra": { "num_samples": 10, - "number_of_characters": 0, "number_texts_intersect_with_train": None, "text_statistics": None, "image_statistics": { @@ -2866,6 +3056,7 @@ class MockMultilingualImageClassificationTask(AbsTaskAnyClassification): "min_image_height": 100, "average_image_height": 100.0, "max_image_height": 100, + "unique_images": 10, }, "label_statistics": { "min_labels_per_text": 1, @@ -2891,7 +3082,7 @@ class MockMultilingualImageClassificationTask(AbsTaskAnyClassification): input_column_name = "image" def load_data(self, **kwargs): - images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] # noqa: NPY002 + images = [self.np_rng.integers(0, 255, (100, 100, 3)) for _ in range(2)] images = [ Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images ] @@ -2927,19 +3118,20 @@ class MockImageClusteringTask(AbsTaskAnyClustering): "number_of_characters": 0, "text_statistics": None, "image_statistics": { - "average_image_height": 100.0, + "min_image_width": 100, "average_image_width": 100.0, - "max_image_height": 100, "max_image_width": 100, "min_image_height": 100, - "min_image_width": 100, + "average_image_height": 100.0, + "max_image_height": 100, + "unique_images": 2, }, "label_statistics": { "min_labels_per_text": 1, "average_label_per_text": 1.0, "max_labels_per_text": 1, "unique_labels": 2, - "labels": {"0": {"count": 1}, "1": {"count": 1}}, + "labels": {"1": {"count": 1}, "0": {"count": 1}}, }, } } @@ -2955,7 +3147,7 @@ class MockImageClusteringTask(AbsTaskAnyClustering): label_column_name = "label" def load_data(self, **kwargs): - images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] # noqa: NPY002 + images = [self.np_rng.integers(0, 255, (100, 100, 3)) for _ in range(2)] images = [ Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images ] @@ -3009,7 +3201,7 @@ class MockImageMultilabelClassificationTask(AbsTaskImageMultilabelClassification samples_per_label = 3 def load_data(self, **kwargs): - images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] # noqa: NPY002 + images = [self.np_rng.integers(0, 255, (100, 100, 3)) for _ in range(2)] images = [ Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images ] @@ -3109,7 +3301,7 @@ class MockMultilingualImageMultilabelClassificationTask( metadata.eval_langs = multilingual_eval_langs def load_data(self, **kwargs): - images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] # noqa: NPY002 + images = [self.np_rng.integers(0, 255, (100, 100, 3)) for _ in range(2)] images = [ Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images ] @@ -3162,7 +3354,7 @@ class MockImageTextPairClassificationTask(AbsTaskImageTextPairClassification): metadata.category = "i2t" def load_data(self, **kwargs): - images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] # noqa: NPY002 + images = [self.np_rng.integers(0, 255, (100, 100, 3)) for _ in range(2)] images = [ Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images ] @@ -3221,7 +3413,7 @@ class MockMultilingualImageTextPairClassificationTask( metadata.eval_langs = multilingual_eval_langs def load_data(self, **kwargs): - images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] # noqa: NPY002 + images = [self.np_rng.integers(0, 255, (100, 100, 3)) for _ in range(2)] images = [ Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images ] @@ -3259,6 +3451,7 @@ class MockVisualSTSTask(AbsTaskAnySTS): "min_image_height": 100, "average_image_height": 100.0, "max_image_height": 100, + "unique_images": 2, }, "image2_statistics": { "min_image_width": 100, @@ -3267,6 +3460,7 @@ class MockVisualSTSTask(AbsTaskAnySTS): "min_image_height": 100, "average_image_height": 100.0, "max_image_height": 100, + "unique_images": 2, }, "label_statistics": {"min_score": 0.5, "avg_score": 0.5, "max_score": 0.5}, } @@ -3282,7 +3476,7 @@ class MockVisualSTSTask(AbsTaskAnySTS): metadata.category = "i2i" def load_data(self, **kwargs): - images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] # noqa: NPY002 + images = [self.np_rng.integers(0, 255, (100, 100, 3)) for _ in range(2)] images = [ Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images @@ -3306,30 +3500,32 @@ def load_data(self, **kwargs): class MockZeroShotClassificationTask(AbsTaskAnyZeroShotClassification): expected_stats = { "test": { + "num_samples": 2, + "number_of_characters": None, + "text_statistics": None, "image_statistics": { - "average_image_height": 100.0, + "min_image_width": 100, "average_image_width": 100.0, - "max_image_height": 100, "max_image_width": 100, "min_image_height": 100, - "min_image_width": 100, + "average_image_height": 100.0, + "max_image_height": 100, + "unique_images": 2, }, "label_statistics": { + "min_labels_per_text": 1, "average_label_per_text": 1.0, - "labels": { - "label1": {"count": 1}, - "label2": {"count": 1}, - }, "max_labels_per_text": 1, - "min_labels_per_text": 1, "unique_labels": 2, + "labels": {"label1": {"count": 1}, "label2": {"count": 1}}, + }, + "candidates_labels_text_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, }, - "max_label_text_length": 29, - "min_label_text_length": 23, - "num_samples": 2, - "average_label_text_length": 26.0, - "number_of_characters": None, - "text_statistics": None, } } @@ -3343,7 +3539,7 @@ class MockZeroShotClassificationTask(AbsTaskAnyZeroShotClassification): metadata.category = "i2t" def load_data(self, **kwargs): - images = [np.random.randint(0, 255, (100, 100, 3)) for _ in range(2)] # noqa: NPY002 + images = [self.np_rng.integers(0, 255, (100, 100, 3)) for _ in range(2)] images = [ Image.fromarray(image.astype("uint8")).convert("RGBA") for image in images @@ -3369,33 +3565,36 @@ def get_candidate_labels(self) -> list[str]: class MockTextZeroShotClassificationTask(AbsTaskAnyZeroShotClassification): expected_stats = { "test": { + "num_samples": 2, + "number_of_characters": None, + "text_statistics": { + "total_text_length": 52, + "min_text_length": 23, + "average_text_length": 26.0, + "max_text_length": 29, + "unique_texts": 2, + }, "image_statistics": None, "label_statistics": { + "min_labels_per_text": 1, "average_label_per_text": 1.0, - "labels": { - "label1": {"count": 1}, - "label2": {"count": 1}, - }, "max_labels_per_text": 1, - "min_labels_per_text": 1, "unique_labels": 2, + "labels": {"label1": {"count": 1}, "label2": {"count": 1}}, }, - "max_label_text_length": 29, - "min_label_text_length": 23, - "num_samples": 2, - "average_label_text_length": 26.0, - "number_of_characters": None, - "text_statistics": { + "candidates_labels_text_statistics": { + "total_text_length": 52, + "min_text_length": 23, "average_text_length": 26.0, "max_text_length": 29, - "min_text_length": 23, + "unique_texts": 2, }, } } metadata = TaskMetadata( type="ZeroShotClassification", - name="MockZeroShotClassification", + name="MockTextZeroShotClassification", main_score="accuracy", **general_args, # type: ignore )