From 88177f5f8ab1c3041ceae4b93b8462da479a9165 Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Tue, 14 Jan 2025 13:22:29 +0300 Subject: [PATCH 1/4] fix loading pre 11 --- mteb/load_results/task_results.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mteb/load_results/task_results.py b/mteb/load_results/task_results.py index e1b9b9d69d..0c04d52694 100644 --- a/mteb/load_results/task_results.py +++ b/mteb/load_results/task_results.py @@ -387,11 +387,13 @@ def _convert_from_before_v1_11_0(cls, data: dict) -> TaskResult: main_score = task.metadata.main_score for split, split_score in scores.items(): for hf_subset, hf_subset_scores in split_score.items(): - if task.metadata.type == "STS": + if task.metadata.type in ("STS", "PairClassification", "Reranking"): for name, prev_name in [ ("cosine", "cos_sim"), ("manhattan", "manhattan"), ("euclidean", "euclidean"), + ("dot", "dot"), + ("max", "max"), ]: prev_name_scores = hf_subset_scores.pop( prev_name, {"spearman": "NaN"} From 6942ba1f43e6a4a09bf904ec28909c40e8386208 Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Tue, 14 Jan 2025 13:27:00 +0300 Subject: [PATCH 2/4] add similarity --- mteb/load_results/task_results.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mteb/load_results/task_results.py b/mteb/load_results/task_results.py index 0c04d52694..842361cf60 100644 --- a/mteb/load_results/task_results.py +++ b/mteb/load_results/task_results.py @@ -394,6 +394,7 @@ def _convert_from_before_v1_11_0(cls, data: dict) -> TaskResult: ("euclidean", "euclidean"), ("dot", "dot"), ("max", "max"), + ("similarity", "similarity") ]: prev_name_scores = hf_subset_scores.pop( prev_name, {"spearman": "NaN"} From 3101a6c67f26ed1de0b26945c480bf8b780f95e0 Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Tue, 14 Jan 2025 13:30:37 +0300 Subject: [PATCH 3/4] lint --- mteb/load_results/task_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mteb/load_results/task_results.py b/mteb/load_results/task_results.py index 842361cf60..c6e4281b79 100644 --- a/mteb/load_results/task_results.py +++ b/mteb/load_results/task_results.py @@ -394,7 +394,7 @@ def _convert_from_before_v1_11_0(cls, data: dict) -> TaskResult: ("euclidean", "euclidean"), ("dot", "dot"), ("max", "max"), - ("similarity", "similarity") + ("similarity", "similarity"), ]: prev_name_scores = hf_subset_scores.pop( prev_name, {"spearman": "NaN"} From 7cc2fe0165dc6d33553f1bac9d7a58fdda0b055c Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Tue, 14 Jan 2025 13:32:57 +0300 Subject: [PATCH 4/4] run all task types --- mteb/load_results/task_results.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/mteb/load_results/task_results.py b/mteb/load_results/task_results.py index c6e4281b79..72cae5a93d 100644 --- a/mteb/load_results/task_results.py +++ b/mteb/load_results/task_results.py @@ -387,18 +387,16 @@ def _convert_from_before_v1_11_0(cls, data: dict) -> TaskResult: main_score = task.metadata.main_score for split, split_score in scores.items(): for hf_subset, hf_subset_scores in split_score.items(): - if task.metadata.type in ("STS", "PairClassification", "Reranking"): - for name, prev_name in [ - ("cosine", "cos_sim"), - ("manhattan", "manhattan"), - ("euclidean", "euclidean"), - ("dot", "dot"), - ("max", "max"), - ("similarity", "similarity"), - ]: - prev_name_scores = hf_subset_scores.pop( - prev_name, {"spearman": "NaN"} - ) + for name, prev_name in [ + ("cosine", "cos_sim"), + ("manhattan", "manhattan"), + ("euclidean", "euclidean"), + ("dot", "dot"), + ("max", "max"), + ("similarity", "similarity"), + ]: + prev_name_scores = hf_subset_scores.pop(prev_name, None) + if prev_name_scores is not None: for k, v in prev_name_scores.items(): hf_subset_scores[f"{name}_{k}"] = v