diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/CEDRClassification.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/CEDRClassification.json new file mode 100644 index 0000000000..bb1c561d40 --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/CEDRClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "c0ba03d058e3e1b2f3fd20518875a4563dd12db4", + "task_name": "CEDRClassification", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "accuracy": 0.440489, + "f1": 0.331878, + "lrap": 0.665266, + "scores_per_experiment": [ + { + "accuracy": 0.45271, + "f1": 0.289083, + "lrap": 0.649947 + }, + { + "accuracy": 0.504251, + "f1": 0.405432, + "lrap": 0.706323 + }, + { + "accuracy": 0.456429, + "f1": 0.300366, + "lrap": 0.644049 + }, + { + "accuracy": 0.478215, + "f1": 0.362387, + "lrap": 0.673964 + }, + { + "accuracy": 0.46068, + "f1": 0.354197, + "lrap": 0.681243 + }, + { + "accuracy": 0.391605, + "f1": 0.300982, + "lrap": 0.653507 + }, + { + "accuracy": 0.353348, + "f1": 0.3574, + "lrap": 0.669075 + }, + { + "accuracy": 0.489904, + "f1": 0.317934, + "lrap": 0.66525 + }, + { + "accuracy": 0.399044, + "f1": 0.268548, + "lrap": 0.649841 + }, + { + "accuracy": 0.418704, + "f1": 0.362453, + "lrap": 0.659458 + } + ], + "main_score": 0.440489, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 12.207267761230469, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/GeoreviewClassification.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/GeoreviewClassification.json new file mode 100644 index 0000000000..dbbf95b4de --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/GeoreviewClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "3765c0d1de6b7d264bc459433c45e5a75513839c", + "task_name": "GeoreviewClassification", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "accuracy": 0.433936, + "f1": 0.431696, + "f1_weighted": 0.431706, + "scores_per_experiment": [ + { + "accuracy": 0.444336, + "f1": 0.44221, + "f1_weighted": 0.442259 + }, + { + "accuracy": 0.444824, + "f1": 0.441732, + "f1_weighted": 0.441737 + }, + { + "accuracy": 0.444336, + "f1": 0.435421, + "f1_weighted": 0.43545 + }, + { + "accuracy": 0.414062, + "f1": 0.417061, + "f1_weighted": 0.417095 + }, + { + "accuracy": 0.421387, + "f1": 0.427308, + "f1_weighted": 0.427333 + }, + { + "accuracy": 0.420898, + "f1": 0.417962, + "f1_weighted": 0.417969 + }, + { + "accuracy": 0.467285, + "f1": 0.464286, + "f1_weighted": 0.464287 + }, + { + "accuracy": 0.472168, + "f1": 0.459022, + "f1_weighted": 0.458967 + }, + { + "accuracy": 0.374023, + "f1": 0.374138, + "f1_weighted": 0.374137 + }, + { + "accuracy": 0.436035, + "f1": 0.437821, + "f1_weighted": 0.437832 + } + ], + "main_score": 0.433936, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 9.894148588180542, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/GeoreviewClusteringP2P.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/GeoreviewClusteringP2P.json new file mode 100644 index 0000000000..4b8903915c --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/GeoreviewClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "97a313c8fc85b47f13f33e7e9a95c1ad888c7fec", + "task_name": "GeoreviewClusteringP2P", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.682167, + 0.688579, + 0.71259, + 0.702199, + 0.683211, + 0.681277, + 0.692437, + 0.688077, + 0.700128, + 0.695916 + ] + }, + "v_measure": 0.692658, + "v_measure_std": 0.009598, + "main_score": 0.692658, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 11.321293592453003, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/HeadlineClassification.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/HeadlineClassification.json new file mode 100644 index 0000000000..5869f4d5a0 --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/HeadlineClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb", + "task_name": "HeadlineClassification", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "accuracy": 0.75874, + "f1": 0.758527, + "f1_weighted": 0.758529, + "scores_per_experiment": [ + { + "accuracy": 0.704102, + "f1": 0.708731, + "f1_weighted": 0.708757 + }, + { + "accuracy": 0.769043, + "f1": 0.768265, + "f1_weighted": 0.768256 + }, + { + "accuracy": 0.765137, + "f1": 0.764255, + "f1_weighted": 0.764245 + }, + { + "accuracy": 0.755859, + "f1": 0.756639, + "f1_weighted": 0.756634 + }, + { + "accuracy": 0.782715, + "f1": 0.783646, + "f1_weighted": 0.783642 + }, + { + "accuracy": 0.780273, + "f1": 0.778675, + "f1_weighted": 0.778675 + }, + { + "accuracy": 0.745605, + "f1": 0.744286, + "f1_weighted": 0.744271 + }, + { + "accuracy": 0.753906, + "f1": 0.752349, + "f1_weighted": 0.752357 + }, + { + "accuracy": 0.73877, + "f1": 0.737767, + "f1_weighted": 0.7378 + }, + { + "accuracy": 0.791992, + "f1": 0.790661, + "f1_weighted": 0.790656 + } + ], + "main_score": 0.75874, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 5.842599630355835, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/InappropriatenessClassification.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/InappropriatenessClassification.json new file mode 100644 index 0000000000..0c1806d1ed --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/InappropriatenessClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285", + "task_name": "InappropriatenessClassification", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "accuracy": 0.620605, + "f1": 0.615843, + "f1_weighted": 0.615843, + "ap": 0.575877, + "ap_weighted": 0.575877, + "scores_per_experiment": [ + { + "accuracy": 0.647949, + "f1": 0.647808, + "f1_weighted": 0.647808, + "ap": 0.596777, + "ap_weighted": 0.596777 + }, + { + "accuracy": 0.640625, + "f1": 0.638473, + "f1_weighted": 0.638473, + "ap": 0.587444, + "ap_weighted": 0.587444 + }, + { + "accuracy": 0.620605, + "f1": 0.612567, + "f1_weighted": 0.612567, + "ap": 0.580735, + "ap_weighted": 0.580735 + }, + { + "accuracy": 0.603516, + "f1": 0.596073, + "f1_weighted": 0.596073, + "ap": 0.560185, + "ap_weighted": 0.560185 + }, + { + "accuracy": 0.615723, + "f1": 0.599433, + "f1_weighted": 0.599433, + "ap": 0.580305, + "ap_weighted": 0.580305 + }, + { + "accuracy": 0.556152, + "f1": 0.549493, + "f1_weighted": 0.549493, + "ap": 0.530613, + "ap_weighted": 0.530613 + }, + { + "accuracy": 0.633789, + "f1": 0.633257, + "f1_weighted": 0.633257, + "ap": 0.583527, + "ap_weighted": 0.583527 + }, + { + "accuracy": 0.647949, + "f1": 0.641862, + "f1_weighted": 0.641862, + "ap": 0.591337, + "ap_weighted": 0.591337 + }, + { + "accuracy": 0.607422, + "f1": 0.607272, + "f1_weighted": 0.607272, + "ap": 0.564817, + "ap_weighted": 0.564817 + }, + { + "accuracy": 0.632324, + "f1": 0.632191, + "f1_weighted": 0.632191, + "ap": 0.583029, + "ap_weighted": 0.583029 + } + ], + "main_score": 0.620605, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 5.455994129180908, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/KinopoiskClassification.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/KinopoiskClassification.json new file mode 100644 index 0000000000..9dd53f4f20 --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/KinopoiskClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "5911f26666ac11af46cb9c6849d0dc80a378af24", + "task_name": "KinopoiskClassification", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "accuracy": 0.551267, + "f1": 0.547573, + "f1_weighted": 0.547573, + "scores_per_experiment": [ + { + "accuracy": 0.478, + "f1": 0.480871, + "f1_weighted": 0.480871 + }, + { + "accuracy": 0.558667, + "f1": 0.552872, + "f1_weighted": 0.552872 + }, + { + "accuracy": 0.566, + "f1": 0.55294, + "f1_weighted": 0.55294 + }, + { + "accuracy": 0.528667, + "f1": 0.53128, + "f1_weighted": 0.53128 + }, + { + "accuracy": 0.568, + "f1": 0.555625, + "f1_weighted": 0.555625 + }, + { + "accuracy": 0.543333, + "f1": 0.544085, + "f1_weighted": 0.544085 + }, + { + "accuracy": 0.587333, + "f1": 0.590086, + "f1_weighted": 0.590086 + }, + { + "accuracy": 0.555333, + "f1": 0.550708, + "f1_weighted": 0.550708 + }, + { + "accuracy": 0.576, + "f1": 0.565013, + "f1_weighted": 0.565013 + }, + { + "accuracy": 0.551333, + "f1": 0.552251, + "f1_weighted": 0.552251 + } + ], + "main_score": 0.551267, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 8.271155834197998, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/MIRACLReranking.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/MIRACLReranking.json new file mode 100644 index 0000000000..89f2aa8d5f --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "task_name": "MIRACLReranking", + "mteb_version": "1.38.0", + "scores": { + "dev": [ + { + "NDCG@1(MIRACL)": 0.4563, + "NDCG@3(MIRACL)": 0.48112, + "NDCG@5(MIRACL)": 0.50845, + "NDCG@10(MIRACL)": 0.54602, + "NDCG@20(MIRACL)": 0.57528, + "NDCG@100(MIRACL)": 0.59328, + "NDCG@1000(MIRACL)": 0.59328, + "MAP@1(MIRACL)": 0.28736, + "MAP@3(MIRACL)": 0.39914, + "MAP@5(MIRACL)": 0.43665, + "MAP@10(MIRACL)": 0.46712, + "MAP@20(MIRACL)": 0.48274, + "MAP@100(MIRACL)": 0.48938, + "MAP@1000(MIRACL)": 0.48938, + "Recall@1(MIRACL)": 0.28736, + "Recall@3(MIRACL)": 0.47259, + "Recall@5(MIRACL)": 0.55881, + "Recall@10(MIRACL)": 0.65676, + "Recall@20(MIRACL)": 0.7376, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "P@1(MIRACL)": 0.4563, + "P@3(MIRACL)": 0.29698, + "P@5(MIRACL)": 0.22759, + "P@10(MIRACL)": 0.14507, + "P@20(MIRACL)": 0.08609, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "nAUC_NDCG@1_max(MIRACL)": 0.252496, + "nAUC_NDCG@1_std(MIRACL)": 0.055815, + "nAUC_NDCG@1_diff1(MIRACL)": 0.223731, + "nAUC_NDCG@3_max(MIRACL)": 0.222025, + "nAUC_NDCG@3_std(MIRACL)": 0.061064, + "nAUC_NDCG@3_diff1(MIRACL)": 0.172335, + "nAUC_NDCG@5_max(MIRACL)": 0.23193, + "nAUC_NDCG@5_std(MIRACL)": 0.076277, + "nAUC_NDCG@5_diff1(MIRACL)": 0.151627, + "nAUC_NDCG@10_max(MIRACL)": 0.275092, + "nAUC_NDCG@10_std(MIRACL)": 0.100087, + "nAUC_NDCG@10_diff1(MIRACL)": 0.114062, + "nAUC_NDCG@20_max(MIRACL)": 0.307689, + "nAUC_NDCG@20_std(MIRACL)": 0.114879, + "nAUC_NDCG@20_diff1(MIRACL)": 0.085228, + "nAUC_NDCG@100_max(MIRACL)": 0.335023, + "nAUC_NDCG@100_std(MIRACL)": 0.121739, + "nAUC_NDCG@100_diff1(MIRACL)": 0.065365, + "nAUC_NDCG@1000_max(MIRACL)": 0.335023, + "nAUC_NDCG@1000_std(MIRACL)": 0.121739, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.065365, + "nAUC_MAP@1_max(MIRACL)": 0.100968, + "nAUC_MAP@1_std(MIRACL)": -0.00287, + "nAUC_MAP@1_diff1(MIRACL)": 0.324479, + "nAUC_MAP@3_max(MIRACL)": 0.152418, + "nAUC_MAP@3_std(MIRACL)": 0.023167, + "nAUC_MAP@3_diff1(MIRACL)": 0.245924, + "nAUC_MAP@5_max(MIRACL)": 0.176914, + "nAUC_MAP@5_std(MIRACL)": 0.044299, + "nAUC_MAP@5_diff1(MIRACL)": 0.213462, + "nAUC_MAP@10_max(MIRACL)": 0.208747, + "nAUC_MAP@10_std(MIRACL)": 0.061633, + "nAUC_MAP@10_diff1(MIRACL)": 0.184143, + "nAUC_MAP@20_max(MIRACL)": 0.22494, + "nAUC_MAP@20_std(MIRACL)": 0.069591, + "nAUC_MAP@20_diff1(MIRACL)": 0.170931, + "nAUC_MAP@100_max(MIRACL)": 0.234058, + "nAUC_MAP@100_std(MIRACL)": 0.072245, + "nAUC_MAP@100_diff1(MIRACL)": 0.164634, + "nAUC_MAP@1000_max(MIRACL)": 0.234058, + "nAUC_MAP@1000_std(MIRACL)": 0.072245, + "nAUC_MAP@1000_diff1(MIRACL)": 0.164634, + "nAUC_Recall@1_max(MIRACL)": 0.100968, + "nAUC_Recall@1_std(MIRACL)": -0.00287, + "nAUC_Recall@1_diff1(MIRACL)": 0.324479, + "nAUC_Recall@3_max(MIRACL)": 0.155281, + "nAUC_Recall@3_std(MIRACL)": 0.041568, + "nAUC_Recall@3_diff1(MIRACL)": 0.172732, + "nAUC_Recall@5_max(MIRACL)": 0.199125, + "nAUC_Recall@5_std(MIRACL)": 0.082023, + "nAUC_Recall@5_diff1(MIRACL)": 0.092976, + "nAUC_Recall@10_max(MIRACL)": 0.320077, + "nAUC_Recall@10_std(MIRACL)": 0.151511, + "nAUC_Recall@10_diff1(MIRACL)": -0.02034, + "nAUC_Recall@20_max(MIRACL)": 0.447762, + "nAUC_Recall@20_std(MIRACL)": 0.212596, + "nAUC_Recall@20_diff1(MIRACL)": -0.136716, + "nAUC_Recall@100_max(MIRACL)": 0.63116, + "nAUC_Recall@100_std(MIRACL)": 0.274989, + "nAUC_Recall@100_diff1(MIRACL)": -0.276729, + "nAUC_Recall@1000_max(MIRACL)": 0.63116, + "nAUC_Recall@1000_std(MIRACL)": 0.274989, + "nAUC_Recall@1000_diff1(MIRACL)": -0.276729, + "nAUC_P@1_max(MIRACL)": 0.252496, + "nAUC_P@1_std(MIRACL)": 0.055815, + "nAUC_P@1_diff1(MIRACL)": 0.223731, + "nAUC_P@3_max(MIRACL)": 0.281878, + "nAUC_P@3_std(MIRACL)": 0.115045, + "nAUC_P@3_diff1(MIRACL)": -0.072454, + "nAUC_P@5_max(MIRACL)": 0.286259, + "nAUC_P@5_std(MIRACL)": 0.144043, + "nAUC_P@5_diff1(MIRACL)": -0.154274, + "nAUC_P@10_max(MIRACL)": 0.307565, + "nAUC_P@10_std(MIRACL)": 0.153767, + "nAUC_P@10_diff1(MIRACL)": -0.224367, + "nAUC_P@20_max(MIRACL)": 0.302763, + "nAUC_P@20_std(MIRACL)": 0.152486, + "nAUC_P@20_diff1(MIRACL)": -0.256253, + "nAUC_P@100_max(MIRACL)": 0.309276, + "nAUC_P@100_std(MIRACL)": 0.130972, + "nAUC_P@100_diff1(MIRACL)": -0.271368, + "nAUC_P@1000_max(MIRACL)": 0.309276, + "nAUC_P@1000_std(MIRACL)": 0.130972, + "nAUC_P@1000_diff1(MIRACL)": -0.271368, + "main_score": 0.54602, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 170.65750789642334, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/MIRACLRetrieval.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/MIRACLRetrieval.json new file mode 100644 index 0000000000..30c86f4b6b --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/MIRACLRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "main", + "task_name": "MIRACLRetrieval", + "mteb_version": "1.38.0", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.41933, + "ndcg_at_3": 0.42452, + "ndcg_at_5": 0.4509, + "ndcg_at_10": 0.4988, + "ndcg_at_20": 0.54166, + "ndcg_at_100": 0.58688, + "ndcg_at_1000": 0.59907, + "map_at_1": 0.21347, + "map_at_3": 0.31953, + "map_at_5": 0.35647, + "map_at_10": 0.3907, + "map_at_20": 0.41046, + "map_at_100": 0.42318, + "map_at_1000": 0.42424, + "recall_at_1": 0.21347, + "recall_at_3": 0.397, + "recall_at_5": 0.4932, + "recall_at_10": 0.62281, + "recall_at_20": 0.74544, + "recall_at_100": 0.91103, + "recall_at_1000": 0.97679, + "precision_at_1": 0.41933, + "precision_at_3": 0.29473, + "precision_at_5": 0.23147, + "precision_at_10": 0.15567, + "precision_at_20": 0.09708, + "precision_at_100": 0.02539, + "precision_at_1000": 0.00278, + "mrr_at_1": 0.419329, + "mrr_at_3": 0.528088, + "mrr_at_5": 0.548855, + "mrr_at_10": 0.562019, + "mrr_at_20": 0.566834, + "mrr_at_100": 0.568521, + "mrr_at_1000": 0.568591, + "nauc_ndcg_at_1_max": 0.177948, + "nauc_ndcg_at_1_std": -0.042306, + "nauc_ndcg_at_1_diff1": 0.267486, + "nauc_ndcg_at_3_max": 0.182226, + "nauc_ndcg_at_3_std": -0.029181, + "nauc_ndcg_at_3_diff1": 0.217772, + "nauc_ndcg_at_5_max": 0.180534, + "nauc_ndcg_at_5_std": -0.031657, + "nauc_ndcg_at_5_diff1": 0.220323, + "nauc_ndcg_at_10_max": 0.185994, + "nauc_ndcg_at_10_std": -0.028842, + "nauc_ndcg_at_10_diff1": 0.222386, + "nauc_ndcg_at_20_max": 0.209662, + "nauc_ndcg_at_20_std": -0.003359, + "nauc_ndcg_at_20_diff1": 0.213912, + "nauc_ndcg_at_100_max": 0.227141, + "nauc_ndcg_at_100_std": 0.023982, + "nauc_ndcg_at_100_diff1": 0.215839, + "nauc_ndcg_at_1000_max": 0.220508, + "nauc_ndcg_at_1000_std": 0.01432, + "nauc_ndcg_at_1000_diff1": 0.21623, + "nauc_map_at_1_max": 0.116366, + "nauc_map_at_1_std": -0.097332, + "nauc_map_at_1_diff1": 0.295102, + "nauc_map_at_3_max": 0.146222, + "nauc_map_at_3_std": -0.073703, + "nauc_map_at_3_diff1": 0.250541, + "nauc_map_at_5_max": 0.160664, + "nauc_map_at_5_std": -0.062581, + "nauc_map_at_5_diff1": 0.243973, + "nauc_map_at_10_max": 0.167065, + "nauc_map_at_10_std": -0.056384, + "nauc_map_at_10_diff1": 0.236401, + "nauc_map_at_20_max": 0.178965, + "nauc_map_at_20_std": -0.044945, + "nauc_map_at_20_diff1": 0.232663, + "nauc_map_at_100_max": 0.184995, + "nauc_map_at_100_std": -0.036532, + "nauc_map_at_100_diff1": 0.231416, + "nauc_map_at_1000_max": 0.18476, + "nauc_map_at_1000_std": -0.036696, + "nauc_map_at_1000_diff1": 0.231328, + "nauc_recall_at_1_max": 0.116366, + "nauc_recall_at_1_std": -0.097332, + "nauc_recall_at_1_diff1": 0.295102, + "nauc_recall_at_3_max": 0.134522, + "nauc_recall_at_3_std": -0.041476, + "nauc_recall_at_3_diff1": 0.193454, + "nauc_recall_at_5_max": 0.133845, + "nauc_recall_at_5_std": -0.025108, + "nauc_recall_at_5_diff1": 0.165855, + "nauc_recall_at_10_max": 0.147533, + "nauc_recall_at_10_std": 0.000499, + "nauc_recall_at_10_diff1": 0.151894, + "nauc_recall_at_20_max": 0.199773, + "nauc_recall_at_20_std": 0.066814, + "nauc_recall_at_20_diff1": 0.11577, + "nauc_recall_at_100_max": 0.394092, + "nauc_recall_at_100_std": 0.364298, + "nauc_recall_at_100_diff1": 0.131192, + "nauc_recall_at_1000_max": 0.572166, + "nauc_recall_at_1000_std": 0.613842, + "nauc_recall_at_1000_diff1": 0.084923, + "nauc_precision_at_1_max": 0.177948, + "nauc_precision_at_1_std": -0.042306, + "nauc_precision_at_1_diff1": 0.267486, + "nauc_precision_at_3_max": 0.20049, + "nauc_precision_at_3_std": 0.050218, + "nauc_precision_at_3_diff1": 0.087496, + "nauc_precision_at_5_max": 0.196892, + "nauc_precision_at_5_std": 0.075318, + "nauc_precision_at_5_diff1": 0.033712, + "nauc_precision_at_10_max": 0.161537, + "nauc_precision_at_10_std": 0.087404, + "nauc_precision_at_10_diff1": -0.029216, + "nauc_precision_at_20_max": 0.176444, + "nauc_precision_at_20_std": 0.152236, + "nauc_precision_at_20_diff1": -0.079925, + "nauc_precision_at_100_max": 0.142452, + "nauc_precision_at_100_std": 0.190841, + "nauc_precision_at_100_diff1": -0.114071, + "nauc_precision_at_1000_max": 0.101232, + "nauc_precision_at_1000_std": 0.156075, + "nauc_precision_at_1000_diff1": -0.118874, + "nauc_mrr_at_1_max": 0.177948, + "nauc_mrr_at_1_std": -0.042306, + "nauc_mrr_at_1_diff1": 0.267486, + "nauc_mrr_at_3_max": 0.207428, + "nauc_mrr_at_3_std": -0.007461, + "nauc_mrr_at_3_diff1": 0.240696, + "nauc_mrr_at_5_max": 0.203596, + "nauc_mrr_at_5_std": -0.005902, + "nauc_mrr_at_5_diff1": 0.234806, + "nauc_mrr_at_10_max": 0.207556, + "nauc_mrr_at_10_std": -0.006089, + "nauc_mrr_at_10_diff1": 0.237502, + "nauc_mrr_at_20_max": 0.209453, + "nauc_mrr_at_20_std": -0.004075, + "nauc_mrr_at_20_diff1": 0.238064, + "nauc_mrr_at_100_max": 0.208823, + "nauc_mrr_at_100_std": -0.004624, + "nauc_mrr_at_100_diff1": 0.239007, + "nauc_mrr_at_1000_max": 0.208687, + "nauc_mrr_at_1000_std": -0.004804, + "nauc_mrr_at_1000_diff1": 0.238976, + "main_score": 0.4988, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 6475.160351514816, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/MassiveIntentClassification.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/MassiveIntentClassification.json new file mode 100644 index 0000000000..273d52c4c1 --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/MassiveIntentClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6", + "task_name": "MassiveIntentClassification", + "mteb_version": "1.38.0", + "scores": { + "validation": [ + { + "accuracy": 0.698377, + "f1": 0.675255, + "f1_weighted": 0.697586, + "scores_per_experiment": [ + { + "accuracy": 0.686178, + "f1": 0.676285, + "f1_weighted": 0.685489 + }, + { + "accuracy": 0.718151, + "f1": 0.681388, + "f1_weighted": 0.719644 + }, + { + "accuracy": 0.684702, + "f1": 0.664003, + "f1_weighted": 0.682062 + }, + { + "accuracy": 0.715199, + "f1": 0.682997, + "f1_weighted": 0.71026 + }, + { + "accuracy": 0.684211, + "f1": 0.664066, + "f1_weighted": 0.68751 + }, + { + "accuracy": 0.698967, + "f1": 0.683572, + "f1_weighted": 0.694036 + }, + { + "accuracy": 0.706837, + "f1": 0.679012, + "f1_weighted": 0.709147 + }, + { + "accuracy": 0.683227, + "f1": 0.670833, + "f1_weighted": 0.680932 + }, + { + "accuracy": 0.692081, + "f1": 0.660593, + "f1_weighted": 0.694496 + }, + { + "accuracy": 0.714215, + "f1": 0.689806, + "f1_weighted": 0.712287 + } + ], + "main_score": 0.698377, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + } + ], + "test": [ + { + "accuracy": 0.691022, + "f1": 0.669508, + "f1_weighted": 0.692414, + "scores_per_experiment": [ + { + "accuracy": 0.69267, + "f1": 0.684839, + "f1_weighted": 0.692185 + }, + { + "accuracy": 0.704102, + "f1": 0.670026, + "f1_weighted": 0.705869 + }, + { + "accuracy": 0.672831, + "f1": 0.65215, + "f1_weighted": 0.674471 + }, + { + "accuracy": 0.713853, + "f1": 0.692156, + "f1_weighted": 0.714319 + }, + { + "accuracy": 0.691997, + "f1": 0.666703, + "f1_weighted": 0.693663 + }, + { + "accuracy": 0.68191, + "f1": 0.666752, + "f1_weighted": 0.683187 + }, + { + "accuracy": 0.691661, + "f1": 0.665992, + "f1_weighted": 0.695683 + }, + { + "accuracy": 0.676866, + "f1": 0.653613, + "f1_weighted": 0.677133 + }, + { + "accuracy": 0.681574, + "f1": 0.661579, + "f1_weighted": 0.684247 + }, + { + "accuracy": 0.702757, + "f1": 0.681269, + "f1_weighted": 0.703384 + } + ], + "main_score": 0.691022, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 33.84981822967529, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/MassiveScenarioClassification.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/MassiveScenarioClassification.json new file mode 100644 index 0000000000..83c9d29d16 --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/MassiveScenarioClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "task_name": "MassiveScenarioClassification", + "mteb_version": "1.38.0", + "scores": { + "validation": [ + { + "accuracy": 0.734825, + "f1": 0.732802, + "f1_weighted": 0.73456, + "scores_per_experiment": [ + { + "accuracy": 0.761928, + "f1": 0.758865, + "f1_weighted": 0.760864 + }, + { + "accuracy": 0.727004, + "f1": 0.729916, + "f1_weighted": 0.722183 + }, + { + "accuracy": 0.707821, + "f1": 0.717349, + "f1_weighted": 0.711637 + }, + { + "accuracy": 0.715691, + "f1": 0.712942, + "f1_weighted": 0.71155 + }, + { + "accuracy": 0.759469, + "f1": 0.757063, + "f1_weighted": 0.75997 + }, + { + "accuracy": 0.739793, + "f1": 0.730918, + "f1_weighted": 0.736798 + }, + { + "accuracy": 0.725037, + "f1": 0.721838, + "f1_weighted": 0.730605 + }, + { + "accuracy": 0.724545, + "f1": 0.724378, + "f1_weighted": 0.72619 + }, + { + "accuracy": 0.740777, + "f1": 0.739765, + "f1_weighted": 0.739386 + }, + { + "accuracy": 0.746188, + "f1": 0.73498, + "f1_weighted": 0.746414 + } + ], + "main_score": 0.734825, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + } + ], + "test": [ + { + "accuracy": 0.735676, + "f1": 0.734875, + "f1_weighted": 0.734768, + "scores_per_experiment": [ + { + "accuracy": 0.755212, + "f1": 0.757682, + "f1_weighted": 0.75297 + }, + { + "accuracy": 0.726631, + "f1": 0.731317, + "f1_weighted": 0.720899 + }, + { + "accuracy": 0.720578, + "f1": 0.725127, + "f1_weighted": 0.720941 + }, + { + "accuracy": 0.729657, + "f1": 0.727603, + "f1_weighted": 0.726079 + }, + { + "accuracy": 0.747814, + "f1": 0.739353, + "f1_weighted": 0.748593 + }, + { + "accuracy": 0.735709, + "f1": 0.72886, + "f1_weighted": 0.730663 + }, + { + "accuracy": 0.723268, + "f1": 0.717774, + "f1_weighted": 0.728578 + }, + { + "accuracy": 0.721251, + "f1": 0.727053, + "f1_weighted": 0.722838 + }, + { + "accuracy": 0.741089, + "f1": 0.741211, + "f1_weighted": 0.739332 + }, + { + "accuracy": 0.755548, + "f1": 0.752772, + "f1_weighted": 0.756787 + } + ], + "main_score": 0.735676, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 11.881885528564453, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RUParaPhraserSTS.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RUParaPhraserSTS.json new file mode 100644 index 0000000000..2bfd8665a8 --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RUParaPhraserSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "43265056790b8f7c59e0139acb4be0a8dad2c8f4", + "task_name": "RUParaPhraserSTS", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "pearson": 0.643782, + "spearman": 0.728033, + "cosine_pearson": 0.643782, + "cosine_spearman": 0.728033, + "manhattan_pearson": 0.686784, + "manhattan_spearman": 0.711526, + "euclidean_pearson": 0.686961, + "euclidean_spearman": 0.713606, + "main_score": 0.728033, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 1.4227018356323242, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RiaNewsRetrieval.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RiaNewsRetrieval.json new file mode 100644 index 0000000000..3d2db938d7 --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RiaNewsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "82374b0bbacda6114f39ff9c5b925fa1512ca5d7", + "task_name": "RiaNewsRetrieval", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "ndcg_at_1": 0.6767, + "ndcg_at_3": 0.76564, + "ndcg_at_5": 0.78191, + "ndcg_at_10": 0.79441, + "ndcg_at_20": 0.80111, + "ndcg_at_100": 0.80698, + "ndcg_at_1000": 0.80981, + "map_at_1": 0.6767, + "map_at_3": 0.7445, + "map_at_5": 0.75358, + "map_at_10": 0.75883, + "map_at_20": 0.76068, + "map_at_100": 0.76152, + "map_at_1000": 0.76163, + "recall_at_1": 0.6767, + "recall_at_3": 0.8265, + "recall_at_5": 0.8658, + "recall_at_10": 0.904, + "recall_at_20": 0.9304, + "recall_at_100": 0.9616, + "recall_at_1000": 0.9839, + "precision_at_1": 0.6767, + "precision_at_3": 0.2755, + "precision_at_5": 0.17316, + "precision_at_10": 0.0904, + "precision_at_20": 0.04652, + "precision_at_100": 0.00962, + "precision_at_1000": 0.00098, + "mrr_at_1": 0.6767, + "mrr_at_3": 0.7445, + "mrr_at_5": 0.753585, + "mrr_at_10": 0.758827, + "mrr_at_20": 0.760682, + "mrr_at_100": 0.761524, + "mrr_at_1000": 0.76163, + "nauc_ndcg_at_1_max": 0.311644, + "nauc_ndcg_at_1_std": -0.180771, + "nauc_ndcg_at_1_diff1": 0.772113, + "nauc_ndcg_at_3_max": 0.352775, + "nauc_ndcg_at_3_std": -0.165294, + "nauc_ndcg_at_3_diff1": 0.73136, + "nauc_ndcg_at_5_max": 0.355503, + "nauc_ndcg_at_5_std": -0.161412, + "nauc_ndcg_at_5_diff1": 0.730326, + "nauc_ndcg_at_10_max": 0.354776, + "nauc_ndcg_at_10_std": -0.156032, + "nauc_ndcg_at_10_diff1": 0.730812, + "nauc_ndcg_at_20_max": 0.356951, + "nauc_ndcg_at_20_std": -0.151574, + "nauc_ndcg_at_20_diff1": 0.733533, + "nauc_ndcg_at_100_max": 0.357347, + "nauc_ndcg_at_100_std": -0.145535, + "nauc_ndcg_at_100_diff1": 0.734573, + "nauc_ndcg_at_1000_max": 0.354603, + "nauc_ndcg_at_1000_std": -0.149552, + "nauc_ndcg_at_1000_diff1": 0.735627, + "nauc_map_at_1_max": 0.311644, + "nauc_map_at_1_std": -0.180771, + "nauc_map_at_1_diff1": 0.772113, + "nauc_map_at_3_max": 0.342018, + "nauc_map_at_3_std": -0.169714, + "nauc_map_at_3_diff1": 0.74208, + "nauc_map_at_5_max": 0.342986, + "nauc_map_at_5_std": -0.16793, + "nauc_map_at_5_diff1": 0.741961, + "nauc_map_at_10_max": 0.342372, + "nauc_map_at_10_std": -0.166071, + "nauc_map_at_10_diff1": 0.74233, + "nauc_map_at_20_max": 0.342783, + "nauc_map_at_20_std": -0.165096, + "nauc_map_at_20_diff1": 0.743082, + "nauc_map_at_100_max": 0.342755, + "nauc_map_at_100_std": -0.164309, + "nauc_map_at_100_diff1": 0.743191, + "nauc_map_at_1000_max": 0.342666, + "nauc_map_at_1000_std": -0.164441, + "nauc_map_at_1000_diff1": 0.743224, + "nauc_recall_at_1_max": 0.311644, + "nauc_recall_at_1_std": -0.180771, + "nauc_recall_at_1_diff1": 0.772113, + "nauc_recall_at_3_max": 0.394419, + "nauc_recall_at_3_std": -0.147943, + "nauc_recall_at_3_diff1": 0.689817, + "nauc_recall_at_5_max": 0.41531, + "nauc_recall_at_5_std": -0.129324, + "nauc_recall_at_5_diff1": 0.675085, + "nauc_recall_at_10_max": 0.433009, + "nauc_recall_at_10_std": -0.087146, + "nauc_recall_at_10_diff1": 0.659257, + "nauc_recall_at_20_max": 0.484092, + "nauc_recall_at_20_std": -0.016897, + "nauc_recall_at_20_diff1": 0.658942, + "nauc_recall_at_100_max": 0.598118, + "nauc_recall_at_100_std": 0.224141, + "nauc_recall_at_100_diff1": 0.626678, + "nauc_recall_at_1000_max": 0.705465, + "nauc_recall_at_1000_std": 0.408462, + "nauc_recall_at_1000_diff1": 0.564484, + "nauc_precision_at_1_max": 0.311644, + "nauc_precision_at_1_std": -0.180771, + "nauc_precision_at_1_diff1": 0.772113, + "nauc_precision_at_3_max": 0.394419, + "nauc_precision_at_3_std": -0.147943, + "nauc_precision_at_3_diff1": 0.689817, + "nauc_precision_at_5_max": 0.41531, + "nauc_precision_at_5_std": -0.129324, + "nauc_precision_at_5_diff1": 0.675085, + "nauc_precision_at_10_max": 0.433009, + "nauc_precision_at_10_std": -0.087146, + "nauc_precision_at_10_diff1": 0.659257, + "nauc_precision_at_20_max": 0.484092, + "nauc_precision_at_20_std": -0.016897, + "nauc_precision_at_20_diff1": 0.658942, + "nauc_precision_at_100_max": 0.598118, + "nauc_precision_at_100_std": 0.224141, + "nauc_precision_at_100_diff1": 0.626678, + "nauc_precision_at_1000_max": 0.705465, + "nauc_precision_at_1000_std": 0.408462, + "nauc_precision_at_1000_diff1": 0.564484, + "nauc_mrr_at_1_max": 0.311644, + "nauc_mrr_at_1_std": -0.180771, + "nauc_mrr_at_1_diff1": 0.772113, + "nauc_mrr_at_3_max": 0.342018, + "nauc_mrr_at_3_std": -0.169714, + "nauc_mrr_at_3_diff1": 0.74208, + "nauc_mrr_at_5_max": 0.342986, + "nauc_mrr_at_5_std": -0.16793, + "nauc_mrr_at_5_diff1": 0.741961, + "nauc_mrr_at_10_max": 0.342372, + "nauc_mrr_at_10_std": -0.166071, + "nauc_mrr_at_10_diff1": 0.74233, + "nauc_mrr_at_20_max": 0.342783, + "nauc_mrr_at_20_std": -0.165096, + "nauc_mrr_at_20_diff1": 0.743082, + "nauc_mrr_at_100_max": 0.342755, + "nauc_mrr_at_100_std": -0.164309, + "nauc_mrr_at_100_diff1": 0.743191, + "nauc_mrr_at_1000_max": 0.342666, + "nauc_mrr_at_1000_std": -0.164441, + "nauc_mrr_at_1000_diff1": 0.743224, + "main_score": 0.79441, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 1334.6218299865723, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuBQReranking.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuBQReranking.json new file mode 100644 index 0000000000..cae8437aaa --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuBQReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "2e96b8f098fa4b0950fc58eacadeb31c0d0c7fa2", + "task_name": "RuBQReranking", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "map": 0.697326, + "mrr": 0.750981, + "nAUC_map_max": 0.142031, + "nAUC_map_std": 0.073026, + "nAUC_map_diff1": 0.328312, + "nAUC_mrr_max": 0.179991, + "nAUC_mrr_std": 0.07845, + "nAUC_mrr_diff1": 0.351195, + "main_score": 0.697326, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 47.394821643829346, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuBQRetrieval.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuBQRetrieval.json new file mode 100644 index 0000000000..aa5d1280c0 --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuBQRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b", + "task_name": "RuBQRetrieval", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "ndcg_at_1": 0.54019, + "ndcg_at_3": 0.58633, + "ndcg_at_5": 0.63094, + "ndcg_at_10": 0.66982, + "ndcg_at_20": 0.68872, + "ndcg_at_100": 0.70315, + "ndcg_at_1000": 0.70789, + "map_at_1": 0.3806, + "map_at_3": 0.52494, + "map_at_5": 0.56221, + "map_at_10": 0.58516, + "map_at_20": 0.59291, + "map_at_100": 0.59584, + "map_at_1000": 0.59612, + "recall_at_1": 0.3806, + "recall_at_3": 0.62629, + "recall_at_5": 0.72909, + "recall_at_10": 0.83041, + "recall_at_20": 0.89172, + "recall_at_100": 0.95793, + "recall_at_1000": 0.98903, + "precision_at_1": 0.54019, + "precision_at_3": 0.32388, + "precision_at_5": 0.23357, + "precision_at_10": 0.1357, + "precision_at_20": 0.07394, + "precision_at_100": 0.016, + "precision_at_1000": 0.00166, + "mrr_at_1": 0.541371, + "mrr_at_3": 0.642336, + "mrr_at_5": 0.660156, + "mrr_at_10": 0.668873, + "mrr_at_20": 0.671316, + "mrr_at_100": 0.672359, + "mrr_at_1000": 0.67242, + "nauc_ndcg_at_1_max": 0.203256, + "nauc_ndcg_at_1_std": -0.172556, + "nauc_ndcg_at_1_diff1": 0.384521, + "nauc_ndcg_at_3_max": 0.19953, + "nauc_ndcg_at_3_std": -0.144049, + "nauc_ndcg_at_3_diff1": 0.296522, + "nauc_ndcg_at_5_max": 0.209987, + "nauc_ndcg_at_5_std": -0.144117, + "nauc_ndcg_at_5_diff1": 0.294982, + "nauc_ndcg_at_10_max": 0.219681, + "nauc_ndcg_at_10_std": -0.12979, + "nauc_ndcg_at_10_diff1": 0.28551, + "nauc_ndcg_at_20_max": 0.23593, + "nauc_ndcg_at_20_std": -0.111255, + "nauc_ndcg_at_20_diff1": 0.29751, + "nauc_ndcg_at_100_max": 0.229341, + "nauc_ndcg_at_100_std": -0.113465, + "nauc_ndcg_at_100_diff1": 0.299784, + "nauc_ndcg_at_1000_max": 0.226826, + "nauc_ndcg_at_1000_std": -0.119736, + "nauc_ndcg_at_1000_diff1": 0.302262, + "nauc_map_at_1_max": 0.132153, + "nauc_map_at_1_std": -0.162014, + "nauc_map_at_1_diff1": 0.371037, + "nauc_map_at_3_max": 0.177368, + "nauc_map_at_3_std": -0.147361, + "nauc_map_at_3_diff1": 0.309378, + "nauc_map_at_5_max": 0.188352, + "nauc_map_at_5_std": -0.146271, + "nauc_map_at_5_diff1": 0.304289, + "nauc_map_at_10_max": 0.196668, + "nauc_map_at_10_std": -0.137947, + "nauc_map_at_10_diff1": 0.298108, + "nauc_map_at_20_max": 0.202212, + "nauc_map_at_20_std": -0.132599, + "nauc_map_at_20_diff1": 0.301453, + "nauc_map_at_100_max": 0.201765, + "nauc_map_at_100_std": -0.132528, + "nauc_map_at_100_diff1": 0.302006, + "nauc_map_at_1000_max": 0.201717, + "nauc_map_at_1000_std": -0.132759, + "nauc_map_at_1000_diff1": 0.302127, + "nauc_recall_at_1_max": 0.132153, + "nauc_recall_at_1_std": -0.162014, + "nauc_recall_at_1_diff1": 0.371037, + "nauc_recall_at_3_max": 0.178082, + "nauc_recall_at_3_std": -0.122644, + "nauc_recall_at_3_diff1": 0.234021, + "nauc_recall_at_5_max": 0.198761, + "nauc_recall_at_5_std": -0.118142, + "nauc_recall_at_5_diff1": 0.209415, + "nauc_recall_at_10_max": 0.227369, + "nauc_recall_at_10_std": -0.055989, + "nauc_recall_at_10_diff1": 0.137151, + "nauc_recall_at_20_max": 0.358701, + "nauc_recall_at_20_std": 0.101727, + "nauc_recall_at_20_diff1": 0.173898, + "nauc_recall_at_100_max": 0.355327, + "nauc_recall_at_100_std": 0.316109, + "nauc_recall_at_100_diff1": 0.075282, + "nauc_recall_at_1000_max": 0.446726, + "nauc_recall_at_1000_std": 0.659849, + "nauc_recall_at_1000_diff1": -0.110157, + "nauc_precision_at_1_max": 0.203256, + "nauc_precision_at_1_std": -0.172556, + "nauc_precision_at_1_diff1": 0.384521, + "nauc_precision_at_3_max": 0.194085, + "nauc_precision_at_3_std": -0.054857, + "nauc_precision_at_3_diff1": 0.072398, + "nauc_precision_at_5_max": 0.175495, + "nauc_precision_at_5_std": -0.020722, + "nauc_precision_at_5_diff1": 0.007787, + "nauc_precision_at_10_max": 0.153743, + "nauc_precision_at_10_std": 0.033879, + "nauc_precision_at_10_diff1": -0.058091, + "nauc_precision_at_20_max": 0.151271, + "nauc_precision_at_20_std": 0.085618, + "nauc_precision_at_20_diff1": -0.069138, + "nauc_precision_at_100_max": 0.111894, + "nauc_precision_at_100_std": 0.101999, + "nauc_precision_at_100_diff1": -0.09532, + "nauc_precision_at_1000_max": 0.089108, + "nauc_precision_at_1000_std": 0.081204, + "nauc_precision_at_1000_diff1": -0.10017, + "nauc_mrr_at_1_max": 0.205948, + "nauc_mrr_at_1_std": -0.172758, + "nauc_mrr_at_1_diff1": 0.381484, + "nauc_mrr_at_3_max": 0.235298, + "nauc_mrr_at_3_std": -0.155726, + "nauc_mrr_at_3_diff1": 0.328471, + "nauc_mrr_at_5_max": 0.240503, + "nauc_mrr_at_5_std": -0.153964, + "nauc_mrr_at_5_diff1": 0.332812, + "nauc_mrr_at_10_max": 0.237038, + "nauc_mrr_at_10_std": -0.154247, + "nauc_mrr_at_10_diff1": 0.334848, + "nauc_mrr_at_20_max": 0.238921, + "nauc_mrr_at_20_std": -0.151446, + "nauc_mrr_at_20_diff1": 0.337584, + "nauc_mrr_at_100_max": 0.237472, + "nauc_mrr_at_100_std": -0.152489, + "nauc_mrr_at_100_diff1": 0.337312, + "nauc_mrr_at_1000_max": 0.237443, + "nauc_mrr_at_1000_std": -0.152542, + "nauc_mrr_at_1000_diff1": 0.337309, + "main_score": 0.66982, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 52.40764808654785, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuReviewsClassification.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuReviewsClassification.json new file mode 100644 index 0000000000..e77db963ad --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a", + "task_name": "RuReviewsClassification", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "accuracy": 0.627393, + "f1": 0.626576, + "f1_weighted": 0.626592, + "scores_per_experiment": [ + { + "accuracy": 0.674316, + "f1": 0.676814, + "f1_weighted": 0.676838 + }, + { + "accuracy": 0.555176, + "f1": 0.554899, + "f1_weighted": 0.554939 + }, + { + "accuracy": 0.623535, + "f1": 0.629775, + "f1_weighted": 0.629768 + }, + { + "accuracy": 0.672363, + "f1": 0.675029, + "f1_weighted": 0.675045 + }, + { + "accuracy": 0.647461, + "f1": 0.654827, + "f1_weighted": 0.654838 + }, + { + "accuracy": 0.641113, + "f1": 0.63178, + "f1_weighted": 0.631789 + }, + { + "accuracy": 0.598145, + "f1": 0.594676, + "f1_weighted": 0.594708 + }, + { + "accuracy": 0.644043, + "f1": 0.632416, + "f1_weighted": 0.632418 + }, + { + "accuracy": 0.571289, + "f1": 0.574934, + "f1_weighted": 0.574913 + }, + { + "accuracy": 0.646484, + "f1": 0.640612, + "f1_weighted": 0.640667 + } + ], + "main_score": 0.627393, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 3.4404194355010986, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuSTSBenchmarkSTS.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuSTSBenchmarkSTS.json new file mode 100644 index 0000000000..51197b1ee6 --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuSTSBenchmarkSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7cf24f325c6da6195df55bef3d86b5e0616f3018", + "task_name": "RuSTSBenchmarkSTS", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "pearson": 0.825343, + "spearman": 0.828298, + "cosine_pearson": 0.825343, + "cosine_spearman": 0.828301, + "manhattan_pearson": 0.810345, + "manhattan_spearman": 0.813347, + "euclidean_pearson": 0.814606, + "euclidean_spearman": 0.817018, + "main_score": 0.828301, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 0.8910133838653564, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuSciBenchGRNTIClassification.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuSciBenchGRNTIClassification.json new file mode 100644 index 0000000000..0fd7a614e6 --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuSciBenchGRNTIClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "task_name": "RuSciBenchGRNTIClassification", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "accuracy": 0.637158, + "f1": 0.634732, + "f1_weighted": 0.634809, + "scores_per_experiment": [ + { + "accuracy": 0.654297, + "f1": 0.648554, + "f1_weighted": 0.648614 + }, + { + "accuracy": 0.643555, + "f1": 0.641209, + "f1_weighted": 0.641272 + }, + { + "accuracy": 0.618652, + "f1": 0.616146, + "f1_weighted": 0.616225 + }, + { + "accuracy": 0.633789, + "f1": 0.631634, + "f1_weighted": 0.631739 + }, + { + "accuracy": 0.654297, + "f1": 0.654448, + "f1_weighted": 0.654508 + }, + { + "accuracy": 0.628418, + "f1": 0.623117, + "f1_weighted": 0.623192 + }, + { + "accuracy": 0.641113, + "f1": 0.637649, + "f1_weighted": 0.637767 + }, + { + "accuracy": 0.641113, + "f1": 0.637288, + "f1_weighted": 0.63735 + }, + { + "accuracy": 0.633789, + "f1": 0.633196, + "f1_weighted": 0.633303 + }, + { + "accuracy": 0.622559, + "f1": 0.624079, + "f1_weighted": 0.624118 + } + ], + "main_score": 0.637158, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 22.047078847885132, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuSciBenchGRNTIClusteringP2P.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuSciBenchGRNTIClusteringP2P.json new file mode 100644 index 0000000000..887edacc43 --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuSciBenchGRNTIClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "task_name": "RuSciBenchGRNTIClusteringP2P", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.610829, + 0.57444, + 0.574972, + 0.579211, + 0.579573, + 0.617948, + 0.581254, + 0.587415, + 0.582302, + 0.584807 + ] + }, + "v_measure": 0.587275, + "v_measure_std": 0.014153, + "main_score": 0.587275, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 12.826419115066528, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuSciBenchOECDClassification.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuSciBenchOECDClassification.json new file mode 100644 index 0000000000..867cfb8cfe --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuSciBenchOECDClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "task_name": "RuSciBenchOECDClassification", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "accuracy": 0.493604, + "f1": 0.488183, + "f1_weighted": 0.488242, + "scores_per_experiment": [ + { + "accuracy": 0.500488, + "f1": 0.491115, + "f1_weighted": 0.491148 + }, + { + "accuracy": 0.505371, + "f1": 0.499565, + "f1_weighted": 0.499681 + }, + { + "accuracy": 0.486328, + "f1": 0.48045, + "f1_weighted": 0.480455 + }, + { + "accuracy": 0.476562, + "f1": 0.473607, + "f1_weighted": 0.473565 + }, + { + "accuracy": 0.486816, + "f1": 0.478132, + "f1_weighted": 0.478244 + }, + { + "accuracy": 0.499023, + "f1": 0.4938, + "f1_weighted": 0.493902 + }, + { + "accuracy": 0.48877, + "f1": 0.485735, + "f1_weighted": 0.48586 + }, + { + "accuracy": 0.515137, + "f1": 0.512311, + "f1_weighted": 0.51238 + }, + { + "accuracy": 0.494141, + "f1": 0.48719, + "f1_weighted": 0.487236 + }, + { + "accuracy": 0.483398, + "f1": 0.479921, + "f1_weighted": 0.479953 + } + ], + "main_score": 0.493604, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 22.228461503982544, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuSciBenchOECDClusteringP2P.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuSciBenchOECDClusteringP2P.json new file mode 100644 index 0000000000..226c271f69 --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/RuSciBenchOECDClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "task_name": "RuSciBenchOECDClusteringP2P", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.486567, + 0.508092, + 0.502379, + 0.501346, + 0.512536, + 0.505327, + 0.50251, + 0.504797, + 0.504122, + 0.488583 + ] + }, + "v_measure": 0.501626, + "v_measure_std": 0.007675, + "main_score": 0.501626, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 10.79359483718872, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/STS22.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/STS22.json new file mode 100644 index 0000000000..f25ce2e53c --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/STS22.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", + "task_name": "STS22", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "pearson": 0.646143, + "spearman": 0.671822, + "cosine_pearson": 0.646143, + "cosine_spearman": 0.671878, + "manhattan_pearson": 0.576369, + "manhattan_spearman": 0.630154, + "euclidean_pearson": 0.581132, + "euclidean_spearman": 0.634385, + "main_score": 0.671878, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 1.9689741134643555, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/SensitiveTopicsClassification.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/SensitiveTopicsClassification.json new file mode 100644 index 0000000000..217dd2dcc3 --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/SensitiveTopicsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "416b34a802308eac30e4192afc0ff99bb8dcc7f2", + "task_name": "SensitiveTopicsClassification", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "accuracy": 0.291602, + "f1": 0.3183, + "lrap": 0.437328, + "scores_per_experiment": [ + { + "accuracy": 0.254395, + "f1": 0.273231, + "lrap": 0.383898 + }, + { + "accuracy": 0.25293, + "f1": 0.348964, + "lrap": 0.425618 + }, + { + "accuracy": 0.280273, + "f1": 0.256713, + "lrap": 0.414564 + }, + { + "accuracy": 0.296387, + "f1": 0.280969, + "lrap": 0.413683 + }, + { + "accuracy": 0.289062, + "f1": 0.322315, + "lrap": 0.446438 + }, + { + "accuracy": 0.303223, + "f1": 0.337447, + "lrap": 0.45379 + }, + { + "accuracy": 0.326172, + "f1": 0.337867, + "lrap": 0.476725 + }, + { + "accuracy": 0.308105, + "f1": 0.348985, + "lrap": 0.469293 + }, + { + "accuracy": 0.306152, + "f1": 0.332959, + "lrap": 0.430393 + }, + { + "accuracy": 0.299316, + "f1": 0.343554, + "lrap": 0.458876 + } + ], + "main_score": 0.291602, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 33.116036891937256, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/TERRa.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/TERRa.json new file mode 100644 index 0000000000..ed99b35190 --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/TERRa.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "7b58f24536063837d644aab9a023c62199b2a612", + "task_name": "TERRa", + "mteb_version": "1.38.0", + "scores": { + "dev": [ + { + "similarity_accuracy": 0.596091, + "similarity_accuracy_threshold": 0.808648, + "similarity_f1": 0.669643, + "similarity_f1_threshold": 0.704573, + "similarity_precision": 0.508475, + "similarity_recall": 0.980392, + "similarity_ap": 0.561255, + "cosine_accuracy": 0.596091, + "cosine_accuracy_threshold": 0.808648, + "cosine_f1": 0.669643, + "cosine_f1_threshold": 0.704573, + "cosine_precision": 0.508475, + "cosine_recall": 0.980392, + "cosine_ap": 0.561255, + "manhattan_accuracy": 0.560261, + "manhattan_accuracy_threshold": 260.753601, + "manhattan_f1": 0.678984, + "manhattan_f1_threshold": 319.977173, + "manhattan_precision": 0.525, + "manhattan_recall": 0.960784, + "manhattan_ap": 0.563827, + "euclidean_accuracy": 0.570033, + "euclidean_accuracy_threshold": 13.474876, + "euclidean_f1": 0.67426, + "euclidean_f1_threshold": 15.741646, + "euclidean_precision": 0.517483, + "euclidean_recall": 0.96732, + "euclidean_ap": 0.560274, + "dot_accuracy": 0.511401, + "dot_accuracy_threshold": 524.69873, + "dot_f1": 0.669584, + "dot_f1_threshold": 215.723618, + "dot_precision": 0.503289, + "dot_recall": 1.0, + "dot_ap": 0.492207, + "max_accuracy": 0.596091, + "max_f1": 0.678984, + "max_precision": 0.525, + "max_recall": 1.0, + "max_ap": 0.563827, + "main_score": 0.563827, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 0.51468825340271, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/model_meta.json b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/model_meta.json new file mode 100644 index 0000000000..169985475e --- /dev/null +++ b/results/deepvk__USER2-base/0942cf96909b6d52e61f79a01e2d30c7be640b27/model_meta.json @@ -0,0 +1 @@ +{"name": "deepvk/USER2-base", "revision": "0942cf96909b6d52e61f79a01e2d30c7be640b27", "release_date": "2025-04-18", "languages": ["rus-Cyrl"], "n_parameters": 149000000, "memory_usage_mb": 568.0, "max_tokens": 8192.0, "embed_dim": 768, "license": "apache-2.0", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/deepvk/USER2-base", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": {"RedditClustering": [], "RedditClusteringP2P": [], "RedditClustering.v2": [], "RedditClusteringP2P.v2": [], "AmazonPolarityClassification": [], "AmazonReviewsClassification": [], "AmazonCounterfactualClassification": [], "WikipediaRetrievalMultilingual": [], "WikipediaRerankingMultilingual": [], "CodeSearchNetCCRetrieval": [], "COIRCodeSearchNetRetrieval": [], "YahooAnswersTopicsClassification": [], "StackExchangeClustering.v2": [], "StackExchangeClusteringP2P.v2": [], "QuoraRetrieval": [], "Quora-NL": [], "NanoQuoraRetrieval": [], "FQuADRetrieval": [], "MSMARCO": ["train"], "MSMARCOHardNegatives": ["train"], "NanoMSMARCORetrieval": ["train"], "mMARCO-NL": ["train"], "NQ": ["train"], "NQHardNegatives": ["train"], "NanoNQRetrieval": ["train"], "NQ-PL": ["train"], "NQ-NL": ["train"], "HotPotQA": ["test"], "HotPotQAHardNegatives": ["test"], "HotPotQA-PL": ["test"], "HotpotQA-NL": ["test"], "FEVER": ["test"], "FEVERHardNegatives": ["test"], "FEVER-NL": ["test"], "MIRACLReranking": ["train"], "MIRACLRetrieval": ["train"], "MIRACLRetrievalHardNegatives": ["train"], "MrTidyRetrieval": ["train"]}, "adapted_from": "https://huggingface.co/deepvk/RuModernBERT-base", "superseded_by": null, "is_cross_encoder": null, "modalities": ["text"], "loader": "sentence_transformers_loader"} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/CEDRClassification.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/CEDRClassification.json new file mode 100644 index 0000000000..c44e25f782 --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/CEDRClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "c0ba03d058e3e1b2f3fd20518875a4563dd12db4", + "task_name": "CEDRClassification", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "accuracy": 0.394368, + "f1": 0.245227, + "lrap": 0.614304, + "scores_per_experiment": [ + { + "accuracy": 0.347503, + "f1": 0.192479, + "lrap": 0.594049 + }, + { + "accuracy": 0.418172, + "f1": 0.229963, + "lrap": 0.619554 + }, + { + "accuracy": 0.362912, + "f1": 0.20085, + "lrap": 0.62136 + }, + { + "accuracy": 0.411265, + "f1": 0.301559, + "lrap": 0.638842 + }, + { + "accuracy": 0.375133, + "f1": 0.188758, + "lrap": 0.59559 + }, + { + "accuracy": 0.427205, + "f1": 0.27423, + "lrap": 0.611105 + }, + { + "accuracy": 0.449522, + "f1": 0.292897, + "lrap": 0.64102 + }, + { + "accuracy": 0.378852, + "f1": 0.198663, + "lrap": 0.579065 + }, + { + "accuracy": 0.408608, + "f1": 0.347089, + "lrap": 0.646599 + }, + { + "accuracy": 0.364506, + "f1": 0.22578, + "lrap": 0.595855 + } + ], + "main_score": 0.394368, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 10.16298532485962, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/GeoreviewClassification.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/GeoreviewClassification.json new file mode 100644 index 0000000000..9c07c73ae2 --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/GeoreviewClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "3765c0d1de6b7d264bc459433c45e5a75513839c", + "task_name": "GeoreviewClassification", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "accuracy": 0.411035, + "f1": 0.409135, + "f1_weighted": 0.409146, + "scores_per_experiment": [ + { + "accuracy": 0.406738, + "f1": 0.402362, + "f1_weighted": 0.402438 + }, + { + "accuracy": 0.445312, + "f1": 0.444212, + "f1_weighted": 0.444181 + }, + { + "accuracy": 0.434082, + "f1": 0.424592, + "f1_weighted": 0.424616 + }, + { + "accuracy": 0.385254, + "f1": 0.394847, + "f1_weighted": 0.394854 + }, + { + "accuracy": 0.395508, + "f1": 0.39719, + "f1_weighted": 0.397191 + }, + { + "accuracy": 0.394531, + "f1": 0.390939, + "f1_weighted": 0.390959 + }, + { + "accuracy": 0.429199, + "f1": 0.42708, + "f1_weighted": 0.427092 + }, + { + "accuracy": 0.45752, + "f1": 0.451935, + "f1_weighted": 0.451909 + }, + { + "accuracy": 0.366211, + "f1": 0.362506, + "f1_weighted": 0.362526 + }, + { + "accuracy": 0.395996, + "f1": 0.395687, + "f1_weighted": 0.395695 + } + ], + "main_score": 0.411035, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 8.84178900718689, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/GeoreviewClusteringP2P.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/GeoreviewClusteringP2P.json new file mode 100644 index 0000000000..9707c5040b --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/GeoreviewClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "97a313c8fc85b47f13f33e7e9a95c1ad888c7fec", + "task_name": "GeoreviewClusteringP2P", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.66689, + 0.656989, + 0.652653, + 0.665982, + 0.667955, + 0.64937, + 0.666428, + 0.662826, + 0.655198, + 0.674348 + ] + }, + "v_measure": 0.661864, + "v_measure_std": 0.007524, + "main_score": 0.661864, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 7.187486171722412, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/HeadlineClassification.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/HeadlineClassification.json new file mode 100644 index 0000000000..101035ee3d --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/HeadlineClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb", + "task_name": "HeadlineClassification", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "accuracy": 0.743262, + "f1": 0.74412, + "f1_weighted": 0.744128, + "scores_per_experiment": [ + { + "accuracy": 0.685547, + "f1": 0.692534, + "f1_weighted": 0.692565 + }, + { + "accuracy": 0.741699, + "f1": 0.744049, + "f1_weighted": 0.74406 + }, + { + "accuracy": 0.756348, + "f1": 0.754954, + "f1_weighted": 0.754952 + }, + { + "accuracy": 0.766113, + "f1": 0.766242, + "f1_weighted": 0.766243 + }, + { + "accuracy": 0.758301, + "f1": 0.759958, + "f1_weighted": 0.759957 + }, + { + "accuracy": 0.768066, + "f1": 0.767802, + "f1_weighted": 0.76781 + }, + { + "accuracy": 0.727051, + "f1": 0.726475, + "f1_weighted": 0.726478 + }, + { + "accuracy": 0.733398, + "f1": 0.733197, + "f1_weighted": 0.733204 + }, + { + "accuracy": 0.722656, + "f1": 0.722674, + "f1_weighted": 0.722702 + }, + { + "accuracy": 0.773438, + "f1": 0.773314, + "f1_weighted": 0.773311 + } + ], + "main_score": 0.743262, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 4.941124439239502, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/InappropriatenessClassification.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/InappropriatenessClassification.json new file mode 100644 index 0000000000..17a4cc61eb --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/InappropriatenessClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285", + "task_name": "InappropriatenessClassification", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "accuracy": 0.607227, + "f1": 0.604012, + "f1_weighted": 0.604012, + "ap": 0.56625, + "ap_weighted": 0.56625, + "scores_per_experiment": [ + { + "accuracy": 0.605469, + "f1": 0.6026, + "f1_weighted": 0.6026, + "ap": 0.566135, + "ap_weighted": 0.566135 + }, + { + "accuracy": 0.642578, + "f1": 0.641689, + "f1_weighted": 0.641689, + "ap": 0.589776, + "ap_weighted": 0.589776 + }, + { + "accuracy": 0.626465, + "f1": 0.620489, + "f1_weighted": 0.620489, + "ap": 0.584585, + "ap_weighted": 0.584585 + }, + { + "accuracy": 0.574707, + "f1": 0.574224, + "f1_weighted": 0.574224, + "ap": 0.542582, + "ap_weighted": 0.542582 + }, + { + "accuracy": 0.591309, + "f1": 0.587253, + "f1_weighted": 0.587253, + "ap": 0.556053, + "ap_weighted": 0.556053 + }, + { + "accuracy": 0.556152, + "f1": 0.552658, + "f1_weighted": 0.552658, + "ap": 0.530756, + "ap_weighted": 0.530756 + }, + { + "accuracy": 0.644531, + "f1": 0.644302, + "f1_weighted": 0.644302, + "ap": 0.592145, + "ap_weighted": 0.592145 + }, + { + "accuracy": 0.625, + "f1": 0.612347, + "f1_weighted": 0.612347, + "ap": 0.573978, + "ap_weighted": 0.573978 + }, + { + "accuracy": 0.574707, + "f1": 0.573217, + "f1_weighted": 0.573217, + "ap": 0.543683, + "ap_weighted": 0.543683 + }, + { + "accuracy": 0.631348, + "f1": 0.631343, + "f1_weighted": 0.631343, + "ap": 0.582809, + "ap_weighted": 0.582809 + } + ], + "main_score": 0.607227, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 4.830262660980225, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/KinopoiskClassification.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/KinopoiskClassification.json new file mode 100644 index 0000000000..37a15e10a2 --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/KinopoiskClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "5911f26666ac11af46cb9c6849d0dc80a378af24", + "task_name": "KinopoiskClassification", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "accuracy": 0.5224, + "f1": 0.516962, + "f1_weighted": 0.516962, + "scores_per_experiment": [ + { + "accuracy": 0.476, + "f1": 0.470318, + "f1_weighted": 0.470318 + }, + { + "accuracy": 0.535333, + "f1": 0.525038, + "f1_weighted": 0.525038 + }, + { + "accuracy": 0.538, + "f1": 0.527659, + "f1_weighted": 0.527659 + }, + { + "accuracy": 0.502667, + "f1": 0.502006, + "f1_weighted": 0.502006 + }, + { + "accuracy": 0.514667, + "f1": 0.510448, + "f1_weighted": 0.510448 + }, + { + "accuracy": 0.510667, + "f1": 0.511309, + "f1_weighted": 0.511309 + }, + { + "accuracy": 0.553333, + "f1": 0.547363, + "f1_weighted": 0.547363 + }, + { + "accuracy": 0.526, + "f1": 0.523207, + "f1_weighted": 0.523207 + }, + { + "accuracy": 0.536, + "f1": 0.525001, + "f1_weighted": 0.525001 + }, + { + "accuracy": 0.531333, + "f1": 0.527267, + "f1_weighted": 0.527267 + } + ], + "main_score": 0.5224, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 5.780102729797363, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/MIRACLReranking.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/MIRACLReranking.json new file mode 100644 index 0000000000..68176c950f --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "task_name": "MIRACLReranking", + "mteb_version": "1.38.0", + "scores": { + "dev": [ + { + "NDCG@1(MIRACL)": 0.40577, + "NDCG@3(MIRACL)": 0.4352, + "NDCG@5(MIRACL)": 0.464, + "NDCG@10(MIRACL)": 0.50487, + "NDCG@20(MIRACL)": 0.53469, + "NDCG@100(MIRACL)": 0.56006, + "NDCG@1000(MIRACL)": 0.56006, + "MAP@1(MIRACL)": 0.25231, + "MAP@3(MIRACL)": 0.35635, + "MAP@5(MIRACL)": 0.39143, + "MAP@10(MIRACL)": 0.42154, + "MAP@20(MIRACL)": 0.4364, + "MAP@100(MIRACL)": 0.44515, + "MAP@1000(MIRACL)": 0.44515, + "Recall@1(MIRACL)": 0.25231, + "Recall@3(MIRACL)": 0.43256, + "Recall@5(MIRACL)": 0.51981, + "Recall@10(MIRACL)": 0.62587, + "Recall@20(MIRACL)": 0.71035, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "P@1(MIRACL)": 0.40577, + "P@3(MIRACL)": 0.26811, + "P@5(MIRACL)": 0.20946, + "P@10(MIRACL)": 0.13681, + "P@20(MIRACL)": 0.08196, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "nAUC_NDCG@1_max(MIRACL)": 0.206346, + "nAUC_NDCG@1_std(MIRACL)": 0.032804, + "nAUC_NDCG@1_diff1(MIRACL)": 0.216726, + "nAUC_NDCG@3_max(MIRACL)": 0.217311, + "nAUC_NDCG@3_std(MIRACL)": 0.027144, + "nAUC_NDCG@3_diff1(MIRACL)": 0.177638, + "nAUC_NDCG@5_max(MIRACL)": 0.234903, + "nAUC_NDCG@5_std(MIRACL)": 0.049002, + "nAUC_NDCG@5_diff1(MIRACL)": 0.162516, + "nAUC_NDCG@10_max(MIRACL)": 0.258587, + "nAUC_NDCG@10_std(MIRACL)": 0.075449, + "nAUC_NDCG@10_diff1(MIRACL)": 0.122999, + "nAUC_NDCG@20_max(MIRACL)": 0.292273, + "nAUC_NDCG@20_std(MIRACL)": 0.099387, + "nAUC_NDCG@20_diff1(MIRACL)": 0.09316, + "nAUC_NDCG@100_max(MIRACL)": 0.327687, + "nAUC_NDCG@100_std(MIRACL)": 0.101394, + "nAUC_NDCG@100_diff1(MIRACL)": 0.0678, + "nAUC_NDCG@1000_max(MIRACL)": 0.327687, + "nAUC_NDCG@1000_std(MIRACL)": 0.101394, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.0678, + "nAUC_MAP@1_max(MIRACL)": 0.083137, + "nAUC_MAP@1_std(MIRACL)": -0.018062, + "nAUC_MAP@1_diff1(MIRACL)": 0.330773, + "nAUC_MAP@3_max(MIRACL)": 0.145035, + "nAUC_MAP@3_std(MIRACL)": -0.001813, + "nAUC_MAP@3_diff1(MIRACL)": 0.2423, + "nAUC_MAP@5_max(MIRACL)": 0.178207, + "nAUC_MAP@5_std(MIRACL)": 0.01896, + "nAUC_MAP@5_diff1(MIRACL)": 0.219414, + "nAUC_MAP@10_max(MIRACL)": 0.198643, + "nAUC_MAP@10_std(MIRACL)": 0.036187, + "nAUC_MAP@10_diff1(MIRACL)": 0.191637, + "nAUC_MAP@20_max(MIRACL)": 0.215565, + "nAUC_MAP@20_std(MIRACL)": 0.049152, + "nAUC_MAP@20_diff1(MIRACL)": 0.178628, + "nAUC_MAP@100_max(MIRACL)": 0.226596, + "nAUC_MAP@100_std(MIRACL)": 0.050651, + "nAUC_MAP@100_diff1(MIRACL)": 0.171259, + "nAUC_MAP@1000_max(MIRACL)": 0.226596, + "nAUC_MAP@1000_std(MIRACL)": 0.050651, + "nAUC_MAP@1000_diff1(MIRACL)": 0.171259, + "nAUC_Recall@1_max(MIRACL)": 0.083137, + "nAUC_Recall@1_std(MIRACL)": -0.018062, + "nAUC_Recall@1_diff1(MIRACL)": 0.330773, + "nAUC_Recall@3_max(MIRACL)": 0.161424, + "nAUC_Recall@3_std(MIRACL)": 0.016045, + "nAUC_Recall@3_diff1(MIRACL)": 0.166501, + "nAUC_Recall@5_max(MIRACL)": 0.223982, + "nAUC_Recall@5_std(MIRACL)": 0.070164, + "nAUC_Recall@5_diff1(MIRACL)": 0.105792, + "nAUC_Recall@10_max(MIRACL)": 0.289915, + "nAUC_Recall@10_std(MIRACL)": 0.131551, + "nAUC_Recall@10_diff1(MIRACL)": -0.002495, + "nAUC_Recall@20_max(MIRACL)": 0.401705, + "nAUC_Recall@20_std(MIRACL)": 0.214272, + "nAUC_Recall@20_diff1(MIRACL)": -0.117404, + "nAUC_Recall@100_max(MIRACL)": 0.65285, + "nAUC_Recall@100_std(MIRACL)": 0.265749, + "nAUC_Recall@100_diff1(MIRACL)": -0.301197, + "nAUC_Recall@1000_max(MIRACL)": 0.65285, + "nAUC_Recall@1000_std(MIRACL)": 0.265749, + "nAUC_Recall@1000_diff1(MIRACL)": -0.301197, + "nAUC_P@1_max(MIRACL)": 0.206346, + "nAUC_P@1_std(MIRACL)": 0.032804, + "nAUC_P@1_diff1(MIRACL)": 0.216726, + "nAUC_P@3_max(MIRACL)": 0.30617, + "nAUC_P@3_std(MIRACL)": 0.077231, + "nAUC_P@3_diff1(MIRACL)": -0.042335, + "nAUC_P@5_max(MIRACL)": 0.326942, + "nAUC_P@5_std(MIRACL)": 0.111811, + "nAUC_P@5_diff1(MIRACL)": -0.110957, + "nAUC_P@10_max(MIRACL)": 0.322436, + "nAUC_P@10_std(MIRACL)": 0.143905, + "nAUC_P@10_diff1(MIRACL)": -0.192642, + "nAUC_P@20_max(MIRACL)": 0.329325, + "nAUC_P@20_std(MIRACL)": 0.167299, + "nAUC_P@20_diff1(MIRACL)": -0.22772, + "nAUC_P@100_max(MIRACL)": 0.327702, + "nAUC_P@100_std(MIRACL)": 0.125274, + "nAUC_P@100_diff1(MIRACL)": -0.252366, + "nAUC_P@1000_max(MIRACL)": 0.327702, + "nAUC_P@1000_std(MIRACL)": 0.125274, + "nAUC_P@1000_diff1(MIRACL)": -0.252366, + "main_score": 0.50487, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 127.85159993171692, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/MIRACLRetrieval.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/MIRACLRetrieval.json new file mode 100644 index 0000000000..59b110c991 --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/MIRACLRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "main", + "task_name": "MIRACLRetrieval", + "mteb_version": "1.38.0", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.34744, + "ndcg_at_3": 0.35697, + "ndcg_at_5": 0.38432, + "ndcg_at_10": 0.42909, + "ndcg_at_20": 0.47105, + "ndcg_at_100": 0.52208, + "ndcg_at_1000": 0.53824, + "map_at_1": 0.17194, + "map_at_3": 0.26108, + "map_at_5": 0.29503, + "map_at_10": 0.32524, + "map_at_20": 0.34341, + "map_at_100": 0.35686, + "map_at_1000": 0.35809, + "recall_at_1": 0.17194, + "recall_at_3": 0.33589, + "recall_at_5": 0.42833, + "recall_at_10": 0.54962, + "recall_at_20": 0.6707, + "recall_at_100": 0.86322, + "recall_at_1000": 0.95469, + "precision_at_1": 0.34744, + "precision_at_3": 0.24947, + "precision_at_5": 0.20288, + "precision_at_10": 0.13842, + "precision_at_20": 0.08778, + "precision_at_100": 0.02402, + "precision_at_1000": 0.00271, + "mrr_at_1": 0.347444, + "mrr_at_3": 0.453275, + "mrr_at_5": 0.475639, + "mrr_at_10": 0.48973, + "mrr_at_20": 0.495054, + "mrr_at_100": 0.497528, + "mrr_at_1000": 0.497678, + "nauc_ndcg_at_1_max": 0.21126, + "nauc_ndcg_at_1_std": 0.023256, + "nauc_ndcg_at_1_diff1": 0.263681, + "nauc_ndcg_at_3_max": 0.241159, + "nauc_ndcg_at_3_std": 0.026574, + "nauc_ndcg_at_3_diff1": 0.205691, + "nauc_ndcg_at_5_max": 0.251008, + "nauc_ndcg_at_5_std": 0.01775, + "nauc_ndcg_at_5_diff1": 0.212498, + "nauc_ndcg_at_10_max": 0.255622, + "nauc_ndcg_at_10_std": 0.026872, + "nauc_ndcg_at_10_diff1": 0.212907, + "nauc_ndcg_at_20_max": 0.270321, + "nauc_ndcg_at_20_std": 0.049486, + "nauc_ndcg_at_20_diff1": 0.202145, + "nauc_ndcg_at_100_max": 0.299479, + "nauc_ndcg_at_100_std": 0.095422, + "nauc_ndcg_at_100_diff1": 0.20107, + "nauc_ndcg_at_1000_max": 0.293851, + "nauc_ndcg_at_1000_std": 0.089617, + "nauc_ndcg_at_1000_diff1": 0.199252, + "nauc_map_at_1_max": 0.134282, + "nauc_map_at_1_std": -0.04959, + "nauc_map_at_1_diff1": 0.292669, + "nauc_map_at_3_max": 0.184215, + "nauc_map_at_3_std": -0.024652, + "nauc_map_at_3_diff1": 0.228665, + "nauc_map_at_5_max": 0.206756, + "nauc_map_at_5_std": -0.019346, + "nauc_map_at_5_diff1": 0.226205, + "nauc_map_at_10_max": 0.220427, + "nauc_map_at_10_std": -0.007108, + "nauc_map_at_10_diff1": 0.225156, + "nauc_map_at_20_max": 0.23044, + "nauc_map_at_20_std": 0.005034, + "nauc_map_at_20_diff1": 0.219928, + "nauc_map_at_100_max": 0.240452, + "nauc_map_at_100_std": 0.018803, + "nauc_map_at_100_diff1": 0.219084, + "nauc_map_at_1000_max": 0.240381, + "nauc_map_at_1000_std": 0.019087, + "nauc_map_at_1000_diff1": 0.218677, + "nauc_recall_at_1_max": 0.134282, + "nauc_recall_at_1_std": -0.04959, + "nauc_recall_at_1_diff1": 0.292669, + "nauc_recall_at_3_max": 0.185222, + "nauc_recall_at_3_std": 0.001109, + "nauc_recall_at_3_diff1": 0.165304, + "nauc_recall_at_5_max": 0.217954, + "nauc_recall_at_5_std": 0.010065, + "nauc_recall_at_5_diff1": 0.157861, + "nauc_recall_at_10_max": 0.225403, + "nauc_recall_at_10_std": 0.030465, + "nauc_recall_at_10_diff1": 0.145388, + "nauc_recall_at_20_max": 0.23933, + "nauc_recall_at_20_std": 0.077157, + "nauc_recall_at_20_diff1": 0.102116, + "nauc_recall_at_100_max": 0.434831, + "nauc_recall_at_100_std": 0.378232, + "nauc_recall_at_100_diff1": 0.080181, + "nauc_recall_at_1000_max": 0.591589, + "nauc_recall_at_1000_std": 0.645477, + "nauc_recall_at_1000_diff1": -0.061787, + "nauc_precision_at_1_max": 0.21126, + "nauc_precision_at_1_std": 0.023256, + "nauc_precision_at_1_diff1": 0.263681, + "nauc_precision_at_3_max": 0.290643, + "nauc_precision_at_3_std": 0.096381, + "nauc_precision_at_3_diff1": 0.097644, + "nauc_precision_at_5_max": 0.289881, + "nauc_precision_at_5_std": 0.103411, + "nauc_precision_at_5_diff1": 0.068164, + "nauc_precision_at_10_max": 0.273951, + "nauc_precision_at_10_std": 0.139219, + "nauc_precision_at_10_diff1": 0.026568, + "nauc_precision_at_20_max": 0.263474, + "nauc_precision_at_20_std": 0.185878, + "nauc_precision_at_20_diff1": -0.023, + "nauc_precision_at_100_max": 0.234463, + "nauc_precision_at_100_std": 0.245945, + "nauc_precision_at_100_diff1": -0.069824, + "nauc_precision_at_1000_max": 0.174536, + "nauc_precision_at_1000_std": 0.207665, + "nauc_precision_at_1000_diff1": -0.091889, + "nauc_mrr_at_1_max": 0.21126, + "nauc_mrr_at_1_std": 0.023256, + "nauc_mrr_at_1_diff1": 0.263681, + "nauc_mrr_at_3_max": 0.258804, + "nauc_mrr_at_3_std": 0.049912, + "nauc_mrr_at_3_diff1": 0.229564, + "nauc_mrr_at_5_max": 0.264325, + "nauc_mrr_at_5_std": 0.055022, + "nauc_mrr_at_5_diff1": 0.228038, + "nauc_mrr_at_10_max": 0.263235, + "nauc_mrr_at_10_std": 0.056399, + "nauc_mrr_at_10_diff1": 0.2274, + "nauc_mrr_at_20_max": 0.263468, + "nauc_mrr_at_20_std": 0.057578, + "nauc_mrr_at_20_diff1": 0.22712, + "nauc_mrr_at_100_max": 0.263293, + "nauc_mrr_at_100_std": 0.057888, + "nauc_mrr_at_100_diff1": 0.228139, + "nauc_mrr_at_1000_max": 0.263128, + "nauc_mrr_at_1000_std": 0.057689, + "nauc_mrr_at_1000_diff1": 0.228201, + "main_score": 0.42909, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 8435.766399860382, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/MassiveIntentClassification.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/MassiveIntentClassification.json new file mode 100644 index 0000000000..442a536538 --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/MassiveIntentClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6", + "task_name": "MassiveIntentClassification", + "mteb_version": "1.38.0", + "scores": { + "validation": [ + { + "accuracy": 0.666995, + "f1": 0.643967, + "f1_weighted": 0.666747, + "scores_per_experiment": [ + { + "accuracy": 0.680767, + "f1": 0.672953, + "f1_weighted": 0.681914 + }, + { + "accuracy": 0.684702, + "f1": 0.637199, + "f1_weighted": 0.686631 + }, + { + "accuracy": 0.666503, + "f1": 0.648616, + "f1_weighted": 0.665579 + }, + { + "accuracy": 0.689621, + "f1": 0.657123, + "f1_weighted": 0.685811 + }, + { + "accuracy": 0.670438, + "f1": 0.632977, + "f1_weighted": 0.669339 + }, + { + "accuracy": 0.64486, + "f1": 0.634696, + "f1_weighted": 0.643422 + }, + { + "accuracy": 0.645844, + "f1": 0.631151, + "f1_weighted": 0.647155 + }, + { + "accuracy": 0.653222, + "f1": 0.634657, + "f1_weighted": 0.653055 + }, + { + "accuracy": 0.663551, + "f1": 0.644659, + "f1_weighted": 0.661982 + }, + { + "accuracy": 0.670438, + "f1": 0.645636, + "f1_weighted": 0.672579 + } + ], + "main_score": 0.666995, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + } + ], + "test": [ + { + "accuracy": 0.660861, + "f1": 0.636051, + "f1_weighted": 0.663091, + "scores_per_experiment": [ + { + "accuracy": 0.680229, + "f1": 0.660851, + "f1_weighted": 0.681357 + }, + { + "accuracy": 0.677202, + "f1": 0.63632, + "f1_weighted": 0.681052 + }, + { + "accuracy": 0.637525, + "f1": 0.617713, + "f1_weighted": 0.642201 + }, + { + "accuracy": 0.682246, + "f1": 0.649758, + "f1_weighted": 0.682883 + }, + { + "accuracy": 0.661399, + "f1": 0.634073, + "f1_weighted": 0.660421 + }, + { + "accuracy": 0.642569, + "f1": 0.622346, + "f1_weighted": 0.645013 + }, + { + "accuracy": 0.644923, + "f1": 0.634327, + "f1_weighted": 0.650587 + }, + { + "accuracy": 0.659381, + "f1": 0.628652, + "f1_weighted": 0.660902 + }, + { + "accuracy": 0.64963, + "f1": 0.623935, + "f1_weighted": 0.650821 + }, + { + "accuracy": 0.673504, + "f1": 0.652532, + "f1_weighted": 0.675676 + } + ], + "main_score": 0.660861, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 34.685758113861084, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/MassiveScenarioClassification.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/MassiveScenarioClassification.json new file mode 100644 index 0000000000..46e98c24c4 --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/MassiveScenarioClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "task_name": "MassiveScenarioClassification", + "mteb_version": "1.38.0", + "scores": { + "validation": [ + { + "accuracy": 0.706345, + "f1": 0.70788, + "f1_weighted": 0.706186, + "scores_per_experiment": [ + { + "accuracy": 0.716183, + "f1": 0.708444, + "f1_weighted": 0.71397 + }, + { + "accuracy": 0.701426, + "f1": 0.712378, + "f1_weighted": 0.699575 + }, + { + "accuracy": 0.694048, + "f1": 0.708056, + "f1_weighted": 0.697092 + }, + { + "accuracy": 0.703886, + "f1": 0.707915, + "f1_weighted": 0.700573 + }, + { + "accuracy": 0.739793, + "f1": 0.73267, + "f1_weighted": 0.738463 + }, + { + "accuracy": 0.67093, + "f1": 0.670755, + "f1_weighted": 0.666894 + }, + { + "accuracy": 0.700443, + "f1": 0.701363, + "f1_weighted": 0.70557 + }, + { + "accuracy": 0.699951, + "f1": 0.704138, + "f1_weighted": 0.70117 + }, + { + "accuracy": 0.71028, + "f1": 0.709116, + "f1_weighted": 0.709501 + }, + { + "accuracy": 0.726513, + "f1": 0.723962, + "f1_weighted": 0.72905 + } + ], + "main_score": 0.706345, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + } + ], + "test": [ + { + "accuracy": 0.702757, + "f1": 0.70639, + "f1_weighted": 0.701913, + "scores_per_experiment": [ + { + "accuracy": 0.719233, + "f1": 0.720912, + "f1_weighted": 0.715867 + }, + { + "accuracy": 0.694015, + "f1": 0.708772, + "f1_weighted": 0.69054 + }, + { + "accuracy": 0.701076, + "f1": 0.715749, + "f1_weighted": 0.701999 + }, + { + "accuracy": 0.69536, + "f1": 0.695671, + "f1_weighted": 0.692334 + }, + { + "accuracy": 0.737391, + "f1": 0.733093, + "f1_weighted": 0.736535 + }, + { + "accuracy": 0.678211, + "f1": 0.679562, + "f1_weighted": 0.673204 + }, + { + "accuracy": 0.688635, + "f1": 0.687426, + "f1_weighted": 0.691787 + }, + { + "accuracy": 0.685945, + "f1": 0.693319, + "f1_weighted": 0.686075 + }, + { + "accuracy": 0.714526, + "f1": 0.711932, + "f1_weighted": 0.713659 + }, + { + "accuracy": 0.713181, + "f1": 0.71746, + "f1_weighted": 0.717131 + } + ], + "main_score": 0.702757, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 6.287858963012695, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RUParaPhraserSTS.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RUParaPhraserSTS.json new file mode 100644 index 0000000000..d0c733451f --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RUParaPhraserSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "43265056790b8f7c59e0139acb4be0a8dad2c8f4", + "task_name": "RUParaPhraserSTS", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "pearson": 0.616028, + "spearman": 0.696034, + "cosine_pearson": 0.616028, + "cosine_spearman": 0.696033, + "manhattan_pearson": 0.647673, + "manhattan_spearman": 0.674899, + "euclidean_pearson": 0.646733, + "euclidean_spearman": 0.676687, + "main_score": 0.696033, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 2.1534206867218018, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RiaNewsRetrieval.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RiaNewsRetrieval.json new file mode 100644 index 0000000000..d374cc2e97 --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RiaNewsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "82374b0bbacda6114f39ff9c5b925fa1512ca5d7", + "task_name": "RiaNewsRetrieval", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "ndcg_at_1": 0.6106, + "ndcg_at_3": 0.70478, + "ndcg_at_5": 0.72383, + "ndcg_at_10": 0.73988, + "ndcg_at_20": 0.74848, + "ndcg_at_100": 0.75766, + "ndcg_at_1000": 0.76176, + "map_at_1": 0.6106, + "map_at_3": 0.6825, + "map_at_5": 0.69311, + "map_at_10": 0.69981, + "map_at_20": 0.7022, + "map_at_100": 0.70352, + "map_at_1000": 0.70368, + "recall_at_1": 0.6106, + "recall_at_3": 0.7689, + "recall_at_5": 0.815, + "recall_at_10": 0.8642, + "recall_at_20": 0.898, + "recall_at_100": 0.9467, + "recall_at_1000": 0.9792, + "precision_at_1": 0.6106, + "precision_at_3": 0.2563, + "precision_at_5": 0.163, + "precision_at_10": 0.08642, + "precision_at_20": 0.0449, + "precision_at_100": 0.00947, + "precision_at_1000": 0.00098, + "mrr_at_1": 0.6106, + "mrr_at_3": 0.6825, + "mrr_at_5": 0.69311, + "mrr_at_10": 0.699805, + "mrr_at_20": 0.702195, + "mrr_at_100": 0.703523, + "mrr_at_1000": 0.703675, + "nauc_ndcg_at_1_max": 0.269429, + "nauc_ndcg_at_1_std": -0.159385, + "nauc_ndcg_at_1_diff1": 0.728193, + "nauc_ndcg_at_3_max": 0.29875, + "nauc_ndcg_at_3_std": -0.15386, + "nauc_ndcg_at_3_diff1": 0.682707, + "nauc_ndcg_at_5_max": 0.301792, + "nauc_ndcg_at_5_std": -0.148422, + "nauc_ndcg_at_5_diff1": 0.678937, + "nauc_ndcg_at_10_max": 0.303905, + "nauc_ndcg_at_10_std": -0.142205, + "nauc_ndcg_at_10_diff1": 0.678961, + "nauc_ndcg_at_20_max": 0.306832, + "nauc_ndcg_at_20_std": -0.135931, + "nauc_ndcg_at_20_diff1": 0.678929, + "nauc_ndcg_at_100_max": 0.309142, + "nauc_ndcg_at_100_std": -0.130707, + "nauc_ndcg_at_100_diff1": 0.68235, + "nauc_ndcg_at_1000_max": 0.306214, + "nauc_ndcg_at_1000_std": -0.134752, + "nauc_ndcg_at_1000_diff1": 0.683851, + "nauc_map_at_1_max": 0.269429, + "nauc_map_at_1_std": -0.159385, + "nauc_map_at_1_diff1": 0.728193, + "nauc_map_at_3_max": 0.290502, + "nauc_map_at_3_std": -0.155963, + "nauc_map_at_3_diff1": 0.694446, + "nauc_map_at_5_max": 0.291835, + "nauc_map_at_5_std": -0.153242, + "nauc_map_at_5_diff1": 0.692792, + "nauc_map_at_10_max": 0.292298, + "nauc_map_at_10_std": -0.151067, + "nauc_map_at_10_diff1": 0.693081, + "nauc_map_at_20_max": 0.292922, + "nauc_map_at_20_std": -0.149569, + "nauc_map_at_20_diff1": 0.693138, + "nauc_map_at_100_max": 0.293064, + "nauc_map_at_100_std": -0.149032, + "nauc_map_at_100_diff1": 0.693631, + "nauc_map_at_1000_max": 0.292979, + "nauc_map_at_1000_std": -0.149133, + "nauc_map_at_1000_diff1": 0.693677, + "nauc_recall_at_1_max": 0.269429, + "nauc_recall_at_1_std": -0.159385, + "nauc_recall_at_1_diff1": 0.728193, + "nauc_recall_at_3_max": 0.328785, + "nauc_recall_at_3_std": -0.145916, + "nauc_recall_at_3_diff1": 0.640512, + "nauc_recall_at_5_max": 0.34467, + "nauc_recall_at_5_std": -0.12649, + "nauc_recall_at_5_diff1": 0.620119, + "nauc_recall_at_10_max": 0.370501, + "nauc_recall_at_10_std": -0.087042, + "nauc_recall_at_10_diff1": 0.601503, + "nauc_recall_at_20_max": 0.413499, + "nauc_recall_at_20_std": -0.02055, + "nauc_recall_at_20_diff1": 0.57984, + "nauc_recall_at_100_max": 0.561012, + "nauc_recall_at_100_std": 0.191759, + "nauc_recall_at_100_diff1": 0.558873, + "nauc_recall_at_1000_max": 0.719758, + "nauc_recall_at_1000_std": 0.363129, + "nauc_recall_at_1000_diff1": 0.482421, + "nauc_precision_at_1_max": 0.269429, + "nauc_precision_at_1_std": -0.159385, + "nauc_precision_at_1_diff1": 0.728193, + "nauc_precision_at_3_max": 0.328785, + "nauc_precision_at_3_std": -0.145916, + "nauc_precision_at_3_diff1": 0.640512, + "nauc_precision_at_5_max": 0.34467, + "nauc_precision_at_5_std": -0.12649, + "nauc_precision_at_5_diff1": 0.620119, + "nauc_precision_at_10_max": 0.370501, + "nauc_precision_at_10_std": -0.087042, + "nauc_precision_at_10_diff1": 0.601503, + "nauc_precision_at_20_max": 0.413499, + "nauc_precision_at_20_std": -0.02055, + "nauc_precision_at_20_diff1": 0.57984, + "nauc_precision_at_100_max": 0.561012, + "nauc_precision_at_100_std": 0.191759, + "nauc_precision_at_100_diff1": 0.558873, + "nauc_precision_at_1000_max": 0.719758, + "nauc_precision_at_1000_std": 0.363129, + "nauc_precision_at_1000_diff1": 0.482421, + "nauc_mrr_at_1_max": 0.269429, + "nauc_mrr_at_1_std": -0.159385, + "nauc_mrr_at_1_diff1": 0.728193, + "nauc_mrr_at_3_max": 0.290502, + "nauc_mrr_at_3_std": -0.155963, + "nauc_mrr_at_3_diff1": 0.694446, + "nauc_mrr_at_5_max": 0.291835, + "nauc_mrr_at_5_std": -0.153242, + "nauc_mrr_at_5_diff1": 0.692792, + "nauc_mrr_at_10_max": 0.292298, + "nauc_mrr_at_10_std": -0.151067, + "nauc_mrr_at_10_diff1": 0.693081, + "nauc_mrr_at_20_max": 0.292922, + "nauc_mrr_at_20_std": -0.149569, + "nauc_mrr_at_20_diff1": 0.693138, + "nauc_mrr_at_100_max": 0.293064, + "nauc_mrr_at_100_std": -0.149032, + "nauc_mrr_at_100_diff1": 0.693631, + "nauc_mrr_at_1000_max": 0.292979, + "nauc_mrr_at_1000_std": -0.149133, + "nauc_mrr_at_1000_diff1": 0.693677, + "main_score": 0.73988, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 1503.5954048633575, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuBQReranking.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuBQReranking.json new file mode 100644 index 0000000000..b6a149d335 --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuBQReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "2e96b8f098fa4b0950fc58eacadeb31c0d0c7fa2", + "task_name": "RuBQReranking", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "map": 0.660337, + "mrr": 0.715478, + "nAUC_map_max": 0.127696, + "nAUC_map_std": 0.061964, + "nAUC_map_diff1": 0.311351, + "nAUC_mrr_max": 0.168194, + "nAUC_mrr_std": 0.070277, + "nAUC_mrr_diff1": 0.322622, + "main_score": 0.660337, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 34.62532353401184, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuBQRetrieval.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuBQRetrieval.json new file mode 100644 index 0000000000..75a9b23b97 --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuBQRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b", + "task_name": "RuBQRetrieval", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "ndcg_at_1": 0.47104, + "ndcg_at_3": 0.52952, + "ndcg_at_5": 0.57132, + "ndcg_at_10": 0.61142, + "ndcg_at_20": 0.63406, + "ndcg_at_100": 0.653, + "ndcg_at_1000": 0.65929, + "map_at_1": 0.32598, + "map_at_3": 0.46643, + "map_at_5": 0.50106, + "map_at_10": 0.52397, + "map_at_20": 0.5324, + "map_at_100": 0.53632, + "map_at_1000": 0.53666, + "recall_at_1": 0.32598, + "recall_at_3": 0.57584, + "recall_at_5": 0.67303, + "recall_at_10": 0.77789, + "recall_at_20": 0.85408, + "recall_at_100": 0.93995, + "recall_at_1000": 0.98169, + "precision_at_1": 0.47104, + "precision_at_3": 0.29984, + "precision_at_5": 0.21584, + "precision_at_10": 0.12707, + "precision_at_20": 0.07048, + "precision_at_100": 0.01564, + "precision_at_1000": 0.00165, + "mrr_at_1": 0.47104, + "mrr_at_3": 0.582348, + "mrr_at_5": 0.599813, + "mrr_at_10": 0.609827, + "mrr_at_20": 0.613525, + "mrr_at_100": 0.614952, + "mrr_at_1000": 0.615053, + "nauc_ndcg_at_1_max": 0.22003, + "nauc_ndcg_at_1_std": -0.107142, + "nauc_ndcg_at_1_diff1": 0.361082, + "nauc_ndcg_at_3_max": 0.201833, + "nauc_ndcg_at_3_std": -0.125934, + "nauc_ndcg_at_3_diff1": 0.315536, + "nauc_ndcg_at_5_max": 0.217815, + "nauc_ndcg_at_5_std": -0.127261, + "nauc_ndcg_at_5_diff1": 0.304512, + "nauc_ndcg_at_10_max": 0.234647, + "nauc_ndcg_at_10_std": -0.108915, + "nauc_ndcg_at_10_diff1": 0.296101, + "nauc_ndcg_at_20_max": 0.240779, + "nauc_ndcg_at_20_std": -0.094551, + "nauc_ndcg_at_20_diff1": 0.297699, + "nauc_ndcg_at_100_max": 0.240418, + "nauc_ndcg_at_100_std": -0.088257, + "nauc_ndcg_at_100_diff1": 0.303394, + "nauc_ndcg_at_1000_max": 0.235027, + "nauc_ndcg_at_1000_std": -0.096086, + "nauc_ndcg_at_1000_diff1": 0.306269, + "nauc_map_at_1_max": 0.148593, + "nauc_map_at_1_std": -0.12671, + "nauc_map_at_1_diff1": 0.345106, + "nauc_map_at_3_max": 0.185361, + "nauc_map_at_3_std": -0.130084, + "nauc_map_at_3_diff1": 0.326504, + "nauc_map_at_5_max": 0.202175, + "nauc_map_at_5_std": -0.127687, + "nauc_map_at_5_diff1": 0.312788, + "nauc_map_at_10_max": 0.211066, + "nauc_map_at_10_std": -0.118988, + "nauc_map_at_10_diff1": 0.30636, + "nauc_map_at_20_max": 0.212806, + "nauc_map_at_20_std": -0.114017, + "nauc_map_at_20_diff1": 0.30646, + "nauc_map_at_100_max": 0.213042, + "nauc_map_at_100_std": -0.112996, + "nauc_map_at_100_diff1": 0.307896, + "nauc_map_at_1000_max": 0.212818, + "nauc_map_at_1000_std": -0.113265, + "nauc_map_at_1000_diff1": 0.308006, + "nauc_recall_at_1_max": 0.148593, + "nauc_recall_at_1_std": -0.12671, + "nauc_recall_at_1_diff1": 0.345106, + "nauc_recall_at_3_max": 0.174399, + "nauc_recall_at_3_std": -0.124795, + "nauc_recall_at_3_diff1": 0.273639, + "nauc_recall_at_5_max": 0.207076, + "nauc_recall_at_5_std": -0.124692, + "nauc_recall_at_5_diff1": 0.237238, + "nauc_recall_at_10_max": 0.263634, + "nauc_recall_at_10_std": -0.060621, + "nauc_recall_at_10_diff1": 0.191919, + "nauc_recall_at_20_max": 0.321706, + "nauc_recall_at_20_std": 0.037602, + "nauc_recall_at_20_diff1": 0.168928, + "nauc_recall_at_100_max": 0.428528, + "nauc_recall_at_100_std": 0.288874, + "nauc_recall_at_100_diff1": 0.145961, + "nauc_recall_at_1000_max": 0.522267, + "nauc_recall_at_1000_std": 0.432559, + "nauc_recall_at_1000_diff1": 0.180288, + "nauc_precision_at_1_max": 0.22003, + "nauc_precision_at_1_std": -0.107142, + "nauc_precision_at_1_diff1": 0.361082, + "nauc_precision_at_3_max": 0.202565, + "nauc_precision_at_3_std": -0.06051, + "nauc_precision_at_3_diff1": 0.14501, + "nauc_precision_at_5_max": 0.191722, + "nauc_precision_at_5_std": -0.036093, + "nauc_precision_at_5_diff1": 0.054815, + "nauc_precision_at_10_max": 0.177226, + "nauc_precision_at_10_std": 0.022583, + "nauc_precision_at_10_diff1": -0.015402, + "nauc_precision_at_20_max": 0.153153, + "nauc_precision_at_20_std": 0.074844, + "nauc_precision_at_20_diff1": -0.053611, + "nauc_precision_at_100_max": 0.116173, + "nauc_precision_at_100_std": 0.106208, + "nauc_precision_at_100_diff1": -0.084174, + "nauc_precision_at_1000_max": 0.077236, + "nauc_precision_at_1000_std": 0.077372, + "nauc_precision_at_1000_diff1": -0.091744, + "nauc_mrr_at_1_max": 0.22003, + "nauc_mrr_at_1_std": -0.107142, + "nauc_mrr_at_1_diff1": 0.361082, + "nauc_mrr_at_3_max": 0.234177, + "nauc_mrr_at_3_std": -0.112828, + "nauc_mrr_at_3_diff1": 0.333406, + "nauc_mrr_at_5_max": 0.236098, + "nauc_mrr_at_5_std": -0.112699, + "nauc_mrr_at_5_diff1": 0.332683, + "nauc_mrr_at_10_max": 0.238938, + "nauc_mrr_at_10_std": -0.10935, + "nauc_mrr_at_10_diff1": 0.333839, + "nauc_mrr_at_20_max": 0.239087, + "nauc_mrr_at_20_std": -0.108352, + "nauc_mrr_at_20_diff1": 0.334471, + "nauc_mrr_at_100_max": 0.238609, + "nauc_mrr_at_100_std": -0.107894, + "nauc_mrr_at_100_diff1": 0.334605, + "nauc_mrr_at_1000_max": 0.238439, + "nauc_mrr_at_1000_std": -0.108071, + "nauc_mrr_at_1000_diff1": 0.334594, + "main_score": 0.61142, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 65.38883948326111, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuReviewsClassification.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuReviewsClassification.json new file mode 100644 index 0000000000..ca53507c81 --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a", + "task_name": "RuReviewsClassification", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "accuracy": 0.608203, + "f1": 0.606035, + "f1_weighted": 0.606045, + "scores_per_experiment": [ + { + "accuracy": 0.651367, + "f1": 0.650423, + "f1_weighted": 0.650435 + }, + { + "accuracy": 0.549805, + "f1": 0.54792, + "f1_weighted": 0.54797 + }, + { + "accuracy": 0.623047, + "f1": 0.627547, + "f1_weighted": 0.627543 + }, + { + "accuracy": 0.650879, + "f1": 0.653223, + "f1_weighted": 0.653233 + }, + { + "accuracy": 0.613281, + "f1": 0.618769, + "f1_weighted": 0.618761 + }, + { + "accuracy": 0.617188, + "f1": 0.605132, + "f1_weighted": 0.60514 + }, + { + "accuracy": 0.611816, + "f1": 0.60452, + "f1_weighted": 0.604524 + }, + { + "accuracy": 0.598145, + "f1": 0.589239, + "f1_weighted": 0.589233 + }, + { + "accuracy": 0.530762, + "f1": 0.531015, + "f1_weighted": 0.53102 + }, + { + "accuracy": 0.635742, + "f1": 0.632564, + "f1_weighted": 0.632587 + } + ], + "main_score": 0.608203, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 2.6548800468444824, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuSTSBenchmarkSTS.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuSTSBenchmarkSTS.json new file mode 100644 index 0000000000..7ab005cce5 --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuSTSBenchmarkSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7cf24f325c6da6195df55bef3d86b5e0616f3018", + "task_name": "RuSTSBenchmarkSTS", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "pearson": 0.810543, + "spearman": 0.810205, + "cosine_pearson": 0.810543, + "cosine_spearman": 0.810205, + "manhattan_pearson": 0.764555, + "manhattan_spearman": 0.761005, + "euclidean_pearson": 0.768071, + "euclidean_spearman": 0.763987, + "main_score": 0.810205, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 1.3832764625549316, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuSciBenchGRNTIClassification.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuSciBenchGRNTIClassification.json new file mode 100644 index 0000000000..9d156bd13a --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuSciBenchGRNTIClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "task_name": "RuSciBenchGRNTIClassification", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "accuracy": 0.631299, + "f1": 0.626491, + "f1_weighted": 0.626578, + "scores_per_experiment": [ + { + "accuracy": 0.654297, + "f1": 0.647889, + "f1_weighted": 0.647952 + }, + { + "accuracy": 0.621094, + "f1": 0.614536, + "f1_weighted": 0.61462 + }, + { + "accuracy": 0.612305, + "f1": 0.606509, + "f1_weighted": 0.606616 + }, + { + "accuracy": 0.651855, + "f1": 0.64779, + "f1_weighted": 0.647896 + }, + { + "accuracy": 0.638672, + "f1": 0.635138, + "f1_weighted": 0.635208 + }, + { + "accuracy": 0.618652, + "f1": 0.611891, + "f1_weighted": 0.612007 + }, + { + "accuracy": 0.641602, + "f1": 0.638337, + "f1_weighted": 0.638466 + }, + { + "accuracy": 0.611816, + "f1": 0.605718, + "f1_weighted": 0.60578 + }, + { + "accuracy": 0.628906, + "f1": 0.626956, + "f1_weighted": 0.627052 + }, + { + "accuracy": 0.633789, + "f1": 0.630145, + "f1_weighted": 0.630187 + } + ], + "main_score": 0.631299, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 8.551090478897095, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuSciBenchGRNTIClusteringP2P.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuSciBenchGRNTIClusteringP2P.json new file mode 100644 index 0000000000..4e82719bb5 --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuSciBenchGRNTIClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "task_name": "RuSciBenchGRNTIClusteringP2P", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.577944, + 0.574558, + 0.565149, + 0.562063, + 0.549614, + 0.568638, + 0.576729, + 0.538447, + 0.564417, + 0.564854 + ] + }, + "v_measure": 0.564241, + "v_measure_std": 0.011639, + "main_score": 0.564241, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 8.983436822891235, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuSciBenchOECDClassification.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuSciBenchOECDClassification.json new file mode 100644 index 0000000000..45a737cde4 --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuSciBenchOECDClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "task_name": "RuSciBenchOECDClassification", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "accuracy": 0.49165, + "f1": 0.482145, + "f1_weighted": 0.482197, + "scores_per_experiment": [ + { + "accuracy": 0.499023, + "f1": 0.484919, + "f1_weighted": 0.485018 + }, + { + "accuracy": 0.488281, + "f1": 0.482687, + "f1_weighted": 0.482817 + }, + { + "accuracy": 0.481445, + "f1": 0.470575, + "f1_weighted": 0.470658 + }, + { + "accuracy": 0.475586, + "f1": 0.466292, + "f1_weighted": 0.466242 + }, + { + "accuracy": 0.497559, + "f1": 0.485221, + "f1_weighted": 0.485251 + }, + { + "accuracy": 0.502441, + "f1": 0.491987, + "f1_weighted": 0.491976 + }, + { + "accuracy": 0.491211, + "f1": 0.484392, + "f1_weighted": 0.484451 + }, + { + "accuracy": 0.497559, + "f1": 0.491371, + "f1_weighted": 0.49151 + }, + { + "accuracy": 0.484375, + "f1": 0.471235, + "f1_weighted": 0.471332 + }, + { + "accuracy": 0.499023, + "f1": 0.492772, + "f1_weighted": 0.49272 + } + ], + "main_score": 0.49165, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 8.62116026878357, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuSciBenchOECDClusteringP2P.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuSciBenchOECDClusteringP2P.json new file mode 100644 index 0000000000..c599627dbc --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/RuSciBenchOECDClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "task_name": "RuSciBenchOECDClusteringP2P", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.48613, + 0.485414, + 0.488264, + 0.469726, + 0.493389, + 0.484781, + 0.491317, + 0.481613, + 0.487619, + 0.487845 + ] + }, + "v_measure": 0.48561, + "v_measure_std": 0.006152, + "main_score": 0.48561, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 8.203256130218506, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/STS22.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/STS22.json new file mode 100644 index 0000000000..a0784287a1 --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/STS22.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", + "task_name": "STS22", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "pearson": 0.626851, + "spearman": 0.661203, + "cosine_pearson": 0.626851, + "cosine_spearman": 0.661203, + "manhattan_pearson": 0.511332, + "manhattan_spearman": 0.602954, + "euclidean_pearson": 0.517422, + "euclidean_spearman": 0.610789, + "main_score": 0.661203, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 1.4541609287261963, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/SensitiveTopicsClassification.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/SensitiveTopicsClassification.json new file mode 100644 index 0000000000..4248bd91a7 --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/SensitiveTopicsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "416b34a802308eac30e4192afc0ff99bb8dcc7f2", + "task_name": "SensitiveTopicsClassification", + "mteb_version": "1.38.0", + "scores": { + "test": [ + { + "accuracy": 0.274756, + "f1": 0.28232, + "lrap": 0.408933, + "scores_per_experiment": [ + { + "accuracy": 0.259277, + "f1": 0.2432, + "lrap": 0.375231 + }, + { + "accuracy": 0.245117, + "f1": 0.264378, + "lrap": 0.384562 + }, + { + "accuracy": 0.280762, + "f1": 0.238812, + "lrap": 0.393758 + }, + { + "accuracy": 0.261719, + "f1": 0.253739, + "lrap": 0.383152 + }, + { + "accuracy": 0.254883, + "f1": 0.272862, + "lrap": 0.398397 + }, + { + "accuracy": 0.253418, + "f1": 0.290081, + "lrap": 0.409641 + }, + { + "accuracy": 0.291016, + "f1": 0.275205, + "lrap": 0.422729 + }, + { + "accuracy": 0.307617, + "f1": 0.336717, + "lrap": 0.458415 + }, + { + "accuracy": 0.300781, + "f1": 0.312809, + "lrap": 0.423353 + }, + { + "accuracy": 0.292969, + "f1": 0.335395, + "lrap": 0.440091 + } + ], + "main_score": 0.274756, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 31.014525413513184, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/TERRa.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/TERRa.json new file mode 100644 index 0000000000..5a242fc04b --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/TERRa.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "7b58f24536063837d644aab9a023c62199b2a612", + "task_name": "TERRa", + "mteb_version": "1.38.0", + "scores": { + "dev": [ + { + "similarity_accuracy": 0.570033, + "similarity_accuracy_threshold": 0.832086, + "similarity_f1": 0.674058, + "similarity_f1_threshold": 0.754869, + "similarity_precision": 0.510067, + "similarity_recall": 0.993464, + "similarity_ap": 0.540227, + "cosine_accuracy": 0.570033, + "cosine_accuracy_threshold": 0.832086, + "cosine_f1": 0.674058, + "cosine_f1_threshold": 0.754869, + "cosine_precision": 0.510067, + "cosine_recall": 0.993464, + "cosine_ap": 0.540227, + "manhattan_accuracy": 0.550489, + "manhattan_accuracy_threshold": 51.652531, + "manhattan_f1": 0.672941, + "manhattan_f1_threshold": 53.282349, + "manhattan_precision": 0.525735, + "manhattan_recall": 0.934641, + "manhattan_ap": 0.527995, + "euclidean_accuracy": 0.550489, + "euclidean_accuracy_threshold": 3.496766, + "euclidean_f1": 0.674528, + "euclidean_f1_threshold": 3.635141, + "euclidean_precision": 0.527675, + "euclidean_recall": 0.934641, + "euclidean_ap": 0.529482, + "dot_accuracy": 0.508143, + "dot_accuracy_threshold": 22.01572, + "dot_f1": 0.663755, + "dot_f1_threshold": 11.508549, + "dot_precision": 0.498361, + "dot_recall": 0.993464, + "dot_ap": 0.468229, + "max_accuracy": 0.570033, + "max_f1": 0.674528, + "max_precision": 0.527675, + "max_recall": 0.993464, + "max_ap": 0.540227, + "main_score": 0.540227, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 0.35443568229675293, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/model_meta.json b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/model_meta.json new file mode 100644 index 0000000000..7b873f587b --- /dev/null +++ b/results/deepvk__USER2-small/23f65b34cf7632032061f5cc66c14714e6d4cee4/model_meta.json @@ -0,0 +1 @@ +{"name": "deepvk/USER2-small", "revision": "23f65b34cf7632032061f5cc66c14714e6d4cee4", "release_date": "2025-04-18", "languages": ["rus-Cyrl"], "n_parameters": 34400000, "memory_usage_mb": 131.0, "max_tokens": 8192.0, "embed_dim": 384, "license": "apache-2.0", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/deepvk/USER2-small", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": {"RedditClustering": [], "RedditClusteringP2P": [], "RedditClustering.v2": [], "RedditClusteringP2P.v2": [], "AmazonPolarityClassification": [], "AmazonReviewsClassification": [], "AmazonCounterfactualClassification": [], "WikipediaRetrievalMultilingual": [], "WikipediaRerankingMultilingual": [], "CodeSearchNetCCRetrieval": [], "COIRCodeSearchNetRetrieval": [], "YahooAnswersTopicsClassification": [], "StackExchangeClustering.v2": [], "StackExchangeClusteringP2P.v2": [], "QuoraRetrieval": [], "Quora-NL": [], "NanoQuoraRetrieval": [], "FQuADRetrieval": [], "MSMARCO": ["train"], "MSMARCOHardNegatives": ["train"], "NanoMSMARCORetrieval": ["train"], "mMARCO-NL": ["train"], "NQ": ["train"], "NQHardNegatives": ["train"], "NanoNQRetrieval": ["train"], "NQ-PL": ["train"], "NQ-NL": ["train"], "HotPotQA": ["test"], "HotPotQAHardNegatives": ["test"], "HotPotQA-PL": ["test"], "HotpotQA-NL": ["test"], "FEVER": ["test"], "FEVERHardNegatives": ["test"], "FEVER-NL": ["test"], "MIRACLReranking": ["train"], "MIRACLRetrieval": ["train"], "MIRACLRetrievalHardNegatives": ["train"], "MrTidyRetrieval": ["train"]}, "adapted_from": "https://huggingface.co/deepvk/RuModernBERT-small", "superseded_by": null, "is_cross_encoder": null, "modalities": ["text"], "loader": "sentence_transformers_loader"} \ No newline at end of file