diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/AJGT.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/AJGT.json new file mode 100644 index 0000000000..9b3a5f85b2 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/AJGT.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "af3f2fa5462ac461b696cb300d66e07ad366057f", + "task_name": "AJGT", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.523, + "f1": 0.514019, + "f1_weighted": 0.514019, + "ap": 0.512451, + "ap_weighted": 0.512451, + "scores_per_experiment": [ + { + "accuracy": 0.478889, + "f1": 0.4781, + "f1_weighted": 0.4781, + "ap": 0.489928, + "ap_weighted": 0.489928 + }, + { + "accuracy": 0.478333, + "f1": 0.47761, + "f1_weighted": 0.47761, + "ap": 0.489674, + "ap_weighted": 0.489674 + }, + { + "accuracy": 0.564444, + "f1": 0.539365, + "f1_weighted": 0.539365, + "ap": 0.535054, + "ap_weighted": 0.535054 + }, + { + "accuracy": 0.52, + "f1": 0.519231, + "f1_weighted": 0.519231, + "ap": 0.51037, + "ap_weighted": 0.51037 + }, + { + "accuracy": 0.546667, + "f1": 0.538038, + "f1_weighted": 0.538038, + "ap": 0.525044, + "ap_weighted": 0.525044 + }, + { + "accuracy": 0.52, + "f1": 0.519231, + "f1_weighted": 0.519231, + "ap": 0.51037, + "ap_weighted": 0.51037 + }, + { + "accuracy": 0.521667, + "f1": 0.520963, + "f1_weighted": 0.520963, + "ap": 0.511269, + "ap_weighted": 0.511269 + }, + { + "accuracy": 0.521667, + "f1": 0.521003, + "f1_weighted": 0.521003, + "ap": 0.51127, + "ap_weighted": 0.51127 + }, + { + "accuracy": 0.522778, + "f1": 0.522192, + "f1_weighted": 0.522192, + "ap": 0.511874, + "ap_weighted": 0.511874 + }, + { + "accuracy": 0.555556, + "f1": 0.504459, + "f1_weighted": 0.504459, + "ap": 0.529657, + "ap_weighted": 0.529657 + } + ], + "main_score": 0.523, + "hf_subset": "default", + "languages": [ + "ara-Arab" + ] + } + ] + }, + "evaluation_time": 1.7125825881958008, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/AfriSentiClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/AfriSentiClassification.json new file mode 100644 index 0000000000..f48b70a043 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/AfriSentiClassification.json @@ -0,0 +1,755 @@ +{ + "dataset_revision": "b52e930385cf5ed7f063072c3f7bd17b599a16cf", + "task_name": "AfriSentiClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.244822, + "f1": 0.209868, + "f1_weighted": 0.219489, + "scores_per_experiment": [ + { + "accuracy": 0.149575, + "f1": 0.158642, + "f1_weighted": 0.108972 + }, + { + "accuracy": 0.202101, + "f1": 0.208695, + "f1_weighted": 0.167633 + }, + { + "accuracy": 0.250625, + "f1": 0.241571, + "f1_weighted": 0.234756 + }, + { + "accuracy": 0.130065, + "f1": 0.094564, + "f1_weighted": 0.071094 + }, + { + "accuracy": 0.158079, + "f1": 0.163908, + "f1_weighted": 0.11371 + }, + { + "accuracy": 0.284642, + "f1": 0.227547, + "f1_weighted": 0.27735 + }, + { + "accuracy": 0.256628, + "f1": 0.216478, + "f1_weighted": 0.292748 + }, + { + "accuracy": 0.172586, + "f1": 0.172456, + "f1_weighted": 0.094918 + }, + { + "accuracy": 0.56028, + "f1": 0.371308, + "f1_weighted": 0.550272 + }, + { + "accuracy": 0.283642, + "f1": 0.243515, + "f1_weighted": 0.283441 + } + ], + "main_score": 0.244822, + "hf_subset": "amh", + "languages": [ + "amh-Ethi" + ] + }, + { + "accuracy": 0.333925, + "f1": 0.304364, + "f1_weighted": 0.328488, + "scores_per_experiment": [ + { + "accuracy": 0.31524, + "f1": 0.27952, + "f1_weighted": 0.311053 + }, + { + "accuracy": 0.32881, + "f1": 0.322003, + "f1_weighted": 0.325131 + }, + { + "accuracy": 0.378914, + "f1": 0.325049, + "f1_weighted": 0.380038 + }, + { + "accuracy": 0.338205, + "f1": 0.298768, + "f1_weighted": 0.333152 + }, + { + "accuracy": 0.282881, + "f1": 0.251661, + "f1_weighted": 0.267822 + }, + { + "accuracy": 0.258873, + "f1": 0.253038, + "f1_weighted": 0.232981 + }, + { + "accuracy": 0.421712, + "f1": 0.377719, + "f1_weighted": 0.425837 + }, + { + "accuracy": 0.350731, + "f1": 0.332636, + "f1_weighted": 0.356756 + }, + { + "accuracy": 0.294363, + "f1": 0.274119, + "f1_weighted": 0.269782 + }, + { + "accuracy": 0.36952, + "f1": 0.329127, + "f1_weighted": 0.382328 + } + ], + "main_score": 0.333925, + "hf_subset": "arq", + "languages": [ + "arq-Arab" + ] + }, + { + "accuracy": 0.337158, + "f1": 0.306649, + "f1_weighted": 0.314065, + "scores_per_experiment": [ + { + "accuracy": 0.393066, + "f1": 0.383008, + "f1_weighted": 0.386732 + }, + { + "accuracy": 0.290039, + "f1": 0.281585, + "f1_weighted": 0.277724 + }, + { + "accuracy": 0.287598, + "f1": 0.269897, + "f1_weighted": 0.266314 + }, + { + "accuracy": 0.360352, + "f1": 0.283727, + "f1_weighted": 0.304896 + }, + { + "accuracy": 0.292969, + "f1": 0.291537, + "f1_weighted": 0.289589 + }, + { + "accuracy": 0.347656, + "f1": 0.279054, + "f1_weighted": 0.299385 + }, + { + "accuracy": 0.283203, + "f1": 0.279196, + "f1_weighted": 0.275851 + }, + { + "accuracy": 0.379395, + "f1": 0.368519, + "f1_weighted": 0.375503 + }, + { + "accuracy": 0.386719, + "f1": 0.352642, + "f1_weighted": 0.366594 + }, + { + "accuracy": 0.350586, + "f1": 0.277327, + "f1_weighted": 0.298057 + } + ], + "main_score": 0.337158, + "hf_subset": "ary", + "languages": [ + "ary-Arab" + ] + }, + { + "accuracy": 0.600195, + "f1": 0.354864, + "f1_weighted": 0.693993, + "scores_per_experiment": [ + { + "accuracy": 0.715332, + "f1": 0.403747, + "f1_weighted": 0.78091 + }, + { + "accuracy": 0.552246, + "f1": 0.343898, + "f1_weighted": 0.67806 + }, + { + "accuracy": 0.376465, + "f1": 0.262233, + "f1_weighted": 0.529068 + }, + { + "accuracy": 0.633789, + "f1": 0.401727, + "f1_weighted": 0.729491 + }, + { + "accuracy": 0.753418, + "f1": 0.386023, + "f1_weighted": 0.788016 + }, + { + "accuracy": 0.540039, + "f1": 0.325546, + "f1_weighted": 0.642728 + }, + { + "accuracy": 0.602051, + "f1": 0.367596, + "f1_weighted": 0.685555 + }, + { + "accuracy": 0.440918, + "f1": 0.265325, + "f1_weighted": 0.582311 + }, + { + "accuracy": 0.661621, + "f1": 0.403016, + "f1_weighted": 0.739904 + }, + { + "accuracy": 0.726074, + "f1": 0.389525, + "f1_weighted": 0.783884 + } + ], + "main_score": 0.600195, + "hf_subset": "hau", + "languages": [ + "hau-Latn" + ] + }, + { + "accuracy": 0.509326, + "f1": 0.334145, + "f1_weighted": 0.499085, + "scores_per_experiment": [ + { + "accuracy": 0.538086, + "f1": 0.361421, + "f1_weighted": 0.538761 + }, + { + "accuracy": 0.594238, + "f1": 0.405656, + "f1_weighted": 0.608857 + }, + { + "accuracy": 0.427734, + "f1": 0.277469, + "f1_weighted": 0.432218 + }, + { + "accuracy": 0.543945, + "f1": 0.394074, + "f1_weighted": 0.597188 + }, + { + "accuracy": 0.484375, + "f1": 0.293234, + "f1_weighted": 0.422509 + }, + { + "accuracy": 0.524414, + "f1": 0.347321, + "f1_weighted": 0.514492 + }, + { + "accuracy": 0.515625, + "f1": 0.351794, + "f1_weighted": 0.52311 + }, + { + "accuracy": 0.451172, + "f1": 0.341354, + "f1_weighted": 0.509421 + }, + { + "accuracy": 0.562988, + "f1": 0.33888, + "f1_weighted": 0.524503 + }, + { + "accuracy": 0.450684, + "f1": 0.230247, + "f1_weighted": 0.319786 + } + ], + "main_score": 0.509326, + "hf_subset": "ibo", + "languages": [ + "ibo-Latn" + ] + }, + { + "accuracy": 0.364133, + "f1": 0.357448, + "f1_weighted": 0.357019, + "scores_per_experiment": [ + { + "accuracy": 0.382066, + "f1": 0.362524, + "f1_weighted": 0.351439 + }, + { + "accuracy": 0.334308, + "f1": 0.331726, + "f1_weighted": 0.335667 + }, + { + "accuracy": 0.37037, + "f1": 0.360679, + "f1_weighted": 0.356894 + }, + { + "accuracy": 0.359649, + "f1": 0.356162, + "f1_weighted": 0.354932 + }, + { + "accuracy": 0.394737, + "f1": 0.392886, + "f1_weighted": 0.399841 + }, + { + "accuracy": 0.385965, + "f1": 0.373709, + "f1_weighted": 0.382398 + }, + { + "accuracy": 0.374269, + "f1": 0.371409, + "f1_weighted": 0.366483 + }, + { + "accuracy": 0.336257, + "f1": 0.337162, + "f1_weighted": 0.342216 + }, + { + "accuracy": 0.327485, + "f1": 0.317958, + "f1_weighted": 0.310621 + }, + { + "accuracy": 0.376218, + "f1": 0.37026, + "f1_weighted": 0.369698 + } + ], + "main_score": 0.364133, + "hf_subset": "kin", + "languages": [ + "kin-Latn" + ] + }, + { + "accuracy": 0.388965, + "f1": 0.347327, + "f1_weighted": 0.408177, + "scores_per_experiment": [ + { + "accuracy": 0.300293, + "f1": 0.290984, + "f1_weighted": 0.313933 + }, + { + "accuracy": 0.370605, + "f1": 0.336678, + "f1_weighted": 0.397739 + }, + { + "accuracy": 0.543945, + "f1": 0.398074, + "f1_weighted": 0.531259 + }, + { + "accuracy": 0.396973, + "f1": 0.367809, + "f1_weighted": 0.422363 + }, + { + "accuracy": 0.309082, + "f1": 0.301798, + "f1_weighted": 0.324558 + }, + { + "accuracy": 0.397949, + "f1": 0.365979, + "f1_weighted": 0.424038 + }, + { + "accuracy": 0.342285, + "f1": 0.322715, + "f1_weighted": 0.363587 + }, + { + "accuracy": 0.390137, + "f1": 0.364827, + "f1_weighted": 0.415038 + }, + { + "accuracy": 0.393555, + "f1": 0.351217, + "f1_weighted": 0.418232 + }, + { + "accuracy": 0.444824, + "f1": 0.373189, + "f1_weighted": 0.47102 + } + ], + "main_score": 0.388965, + "hf_subset": "por", + "languages": [ + "por-Latn" + ] + }, + { + "accuracy": 0.35249, + "f1": 0.303007, + "f1_weighted": 0.3962, + "scores_per_experiment": [ + { + "accuracy": 0.389648, + "f1": 0.327468, + "f1_weighted": 0.434965 + }, + { + "accuracy": 0.386719, + "f1": 0.334138, + "f1_weighted": 0.432213 + }, + { + "accuracy": 0.352539, + "f1": 0.292537, + "f1_weighted": 0.396832 + }, + { + "accuracy": 0.401855, + "f1": 0.329281, + "f1_weighted": 0.453872 + }, + { + "accuracy": 0.365234, + "f1": 0.296765, + "f1_weighted": 0.409473 + }, + { + "accuracy": 0.274902, + "f1": 0.250157, + "f1_weighted": 0.319035 + }, + { + "accuracy": 0.261719, + "f1": 0.240457, + "f1_weighted": 0.304754 + }, + { + "accuracy": 0.321289, + "f1": 0.289737, + "f1_weighted": 0.358244 + }, + { + "accuracy": 0.412109, + "f1": 0.355118, + "f1_weighted": 0.457003 + }, + { + "accuracy": 0.358887, + "f1": 0.314408, + "f1_weighted": 0.395607 + } + ], + "main_score": 0.35249, + "hf_subset": "pcm", + "languages": [ + "pcm-Latn" + ] + }, + { + "accuracy": 0.35254, + "f1": 0.319682, + "f1_weighted": 0.376311, + "scores_per_experiment": [ + { + "accuracy": 0.371658, + "f1": 0.322596, + "f1_weighted": 0.391696 + }, + { + "accuracy": 0.320856, + "f1": 0.301413, + "f1_weighted": 0.355513 + }, + { + "accuracy": 0.367647, + "f1": 0.338706, + "f1_weighted": 0.39501 + }, + { + "accuracy": 0.382353, + "f1": 0.346252, + "f1_weighted": 0.402838 + }, + { + "accuracy": 0.323529, + "f1": 0.313357, + "f1_weighted": 0.35107 + }, + { + "accuracy": 0.403743, + "f1": 0.347373, + "f1_weighted": 0.432941 + }, + { + "accuracy": 0.350267, + "f1": 0.30384, + "f1_weighted": 0.380656 + }, + { + "accuracy": 0.314171, + "f1": 0.298541, + "f1_weighted": 0.329497 + }, + { + "accuracy": 0.335561, + "f1": 0.295713, + "f1_weighted": 0.347467 + }, + { + "accuracy": 0.355615, + "f1": 0.329026, + "f1_weighted": 0.376421 + } + ], + "main_score": 0.35254, + "hf_subset": "swa", + "languages": [ + "swa-Latn" + ] + }, + { + "accuracy": 0.377977, + "f1": 0.354356, + "f1_weighted": 0.389524, + "scores_per_experiment": [ + { + "accuracy": 0.369863, + "f1": 0.352626, + "f1_weighted": 0.380195 + }, + { + "accuracy": 0.425711, + "f1": 0.400234, + "f1_weighted": 0.440722 + }, + { + "accuracy": 0.382508, + "f1": 0.349392, + "f1_weighted": 0.380412 + }, + { + "accuracy": 0.408851, + "f1": 0.387092, + "f1_weighted": 0.419966 + }, + { + "accuracy": 0.381454, + "f1": 0.355273, + "f1_weighted": 0.397051 + }, + { + "accuracy": 0.395153, + "f1": 0.357656, + "f1_weighted": 0.404004 + }, + { + "accuracy": 0.320337, + "f1": 0.313859, + "f1_weighted": 0.339137 + }, + { + "accuracy": 0.404636, + "f1": 0.363915, + "f1_weighted": 0.410964 + }, + { + "accuracy": 0.386723, + "f1": 0.363643, + "f1_weighted": 0.403617 + }, + { + "accuracy": 0.304531, + "f1": 0.299872, + "f1_weighted": 0.319171 + } + ], + "main_score": 0.377977, + "hf_subset": "twi", + "languages": [ + "twi-Latn" + ] + }, + { + "accuracy": 0.355512, + "f1": 0.332125, + "f1_weighted": 0.368893, + "scores_per_experiment": [ + { + "accuracy": 0.385827, + "f1": 0.357175, + "f1_weighted": 0.398146 + }, + { + "accuracy": 0.366142, + "f1": 0.336828, + "f1_weighted": 0.383707 + }, + { + "accuracy": 0.30315, + "f1": 0.300445, + "f1_weighted": 0.330993 + }, + { + "accuracy": 0.377953, + "f1": 0.340329, + "f1_weighted": 0.388081 + }, + { + "accuracy": 0.311024, + "f1": 0.288129, + "f1_weighted": 0.327526 + }, + { + "accuracy": 0.405512, + "f1": 0.387941, + "f1_weighted": 0.417268 + }, + { + "accuracy": 0.350394, + "f1": 0.311367, + "f1_weighted": 0.357657 + }, + { + "accuracy": 0.334646, + "f1": 0.308907, + "f1_weighted": 0.354068 + }, + { + "accuracy": 0.314961, + "f1": 0.310992, + "f1_weighted": 0.327611 + }, + { + "accuracy": 0.405512, + "f1": 0.379136, + "f1_weighted": 0.403877 + } + ], + "main_score": 0.355512, + "hf_subset": "tso", + "languages": [ + "tso-Latn" + ] + }, + { + "accuracy": 0.351709, + "f1": 0.193424, + "f1_weighted": 0.443959, + "scores_per_experiment": [ + { + "accuracy": 0.32666, + "f1": 0.211345, + "f1_weighted": 0.476602 + }, + { + "accuracy": 0.140625, + "f1": 0.107779, + "f1_weighted": 0.182761 + }, + { + "accuracy": 0.359375, + "f1": 0.219392, + "f1_weighted": 0.489956 + }, + { + "accuracy": 0.415527, + "f1": 0.223797, + "f1_weighted": 0.568565 + }, + { + "accuracy": 0.405762, + "f1": 0.230309, + "f1_weighted": 0.55597 + }, + { + "accuracy": 0.695801, + "f1": 0.305216, + "f1_weighted": 0.779923 + }, + { + "accuracy": 0.05957, + "f1": 0.067547, + "f1_weighted": 0.091719 + }, + { + "accuracy": 0.181152, + "f1": 0.135512, + "f1_weighted": 0.264687 + }, + { + "accuracy": 0.80127, + "f1": 0.325745, + "f1_weighted": 0.841264 + }, + { + "accuracy": 0.131348, + "f1": 0.107594, + "f1_weighted": 0.188147 + } + ], + "main_score": 0.351709, + "hf_subset": "yor", + "languages": [ + "yor-Latn" + ] + } + ] + }, + "evaluation_time": 27.489768981933594, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/AllegroReviews.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/AllegroReviews.json new file mode 100644 index 0000000000..7f9b059532 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/AllegroReviews.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "b89853e6de927b0e3bfa8ecc0e56fe4e02ceafc6", + "task_name": "AllegroReviews", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.227038, + "f1": 0.211415, + "f1_weighted": 0.23726, + "scores_per_experiment": [ + { + "accuracy": 0.201789, + "f1": 0.198178, + "f1_weighted": 0.208374 + }, + { + "accuracy": 0.224652, + "f1": 0.213904, + "f1_weighted": 0.234115 + }, + { + "accuracy": 0.236581, + "f1": 0.230856, + "f1_weighted": 0.2493 + }, + { + "accuracy": 0.219682, + "f1": 0.207634, + "f1_weighted": 0.233123 + }, + { + "accuracy": 0.251491, + "f1": 0.221549, + "f1_weighted": 0.257085 + }, + { + "accuracy": 0.246521, + "f1": 0.201741, + "f1_weighted": 0.253553 + }, + { + "accuracy": 0.224652, + "f1": 0.209054, + "f1_weighted": 0.237657 + }, + { + "accuracy": 0.208748, + "f1": 0.196248, + "f1_weighted": 0.226597 + }, + { + "accuracy": 0.238569, + "f1": 0.227468, + "f1_weighted": 0.25 + }, + { + "accuracy": 0.217694, + "f1": 0.207516, + "f1_weighted": 0.222793 + } + ], + "main_score": 0.227038, + "hf_subset": "default", + "languages": [ + "pol-Latn" + ] + } + ] + }, + "evaluation_time": 8.664746046066284, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/AmazonCounterfactualClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/AmazonCounterfactualClassification.json index 29106be829..a36e293fda 100644 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/AmazonCounterfactualClassification.json +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/AmazonCounterfactualClassification.json @@ -1,17 +1,685 @@ { - "mteb_version": "0.0.2", - "test": { - "en": { - "accuracy": 0.6932835820895522, - "accuracy_stderr": 0.04898274617693545, - "ap": 0.3215146833261545, - "ap_stderr": 0.03230897717658606, - "f1": 0.6338819980986146, - "f1_stderr": 0.040359661625618166, - "main_score": 0.6932835820895522 - }, - "evaluation_time": 10.03 - }, - "mteb_dataset_name": "AmazonCounterfactualClassification", - "dataset_revision": "2d8a100785abf0ae21420d2a55b0c56e3e1ea996" + "dataset_revision": "e8379541af4e31359cca9fbcf4b00f2671dba205", + "task_name": "AmazonCounterfactualClassification", + "mteb_version": "1.34.14", + "scores": { + "validation": [ + { + "accuracy": 0.682282, + "f1": 0.558617, + "f1_weighted": 0.744323, + "ap": 0.182388, + "ap_weighted": 0.182388, + "scores_per_experiment": [ + { + "accuracy": 0.738739, + "f1": 0.595857, + "f1_weighted": 0.787809, + "ap": 0.197928, + "ap_weighted": 0.197928 + }, + { + "accuracy": 0.701201, + "f1": 0.576886, + "f1_weighted": 0.760087, + "ap": 0.197178, + "ap_weighted": 0.197178 + }, + { + "accuracy": 0.722222, + "f1": 0.595272, + "f1_weighted": 0.776338, + "ap": 0.212096, + "ap_weighted": 0.212096 + }, + { + "accuracy": 0.624625, + "f1": 0.511953, + "f1_weighted": 0.699269, + "ap": 0.151783, + "ap_weighted": 0.151783 + }, + { + "accuracy": 0.576577, + "f1": 0.487234, + "f1_weighted": 0.658207, + "ap": 0.15163, + "ap_weighted": 0.15163 + }, + { + "accuracy": 0.719219, + "f1": 0.582627, + "f1_weighted": 0.773354, + "ap": 0.191412, + "ap_weighted": 0.191412 + }, + { + "accuracy": 0.674174, + "f1": 0.544857, + "f1_weighted": 0.738651, + "ap": 0.165154, + "ap_weighted": 0.165154 + }, + { + "accuracy": 0.671171, + "f1": 0.55057, + "f1_weighted": 0.736541, + "ap": 0.176862, + "ap_weighted": 0.176862 + }, + { + "accuracy": 0.743243, + "f1": 0.608202, + "f1_weighted": 0.791941, + "ap": 0.216552, + "ap_weighted": 0.216552 + }, + { + "accuracy": 0.651652, + "f1": 0.532716, + "f1_weighted": 0.721031, + "ap": 0.163285, + "ap_weighted": 0.163285 + } + ], + "main_score": 0.682282, + "hf_subset": "en-ext", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.676418, + "f1": 0.601234, + "f1_weighted": 0.71331, + "ap": 0.270918, + "ap_weighted": 0.270918, + "scores_per_experiment": [ + { + "accuracy": 0.608955, + "f1": 0.546872, + "f1_weighted": 0.656519, + "ap": 0.234033, + "ap_weighted": 0.234033 + }, + { + "accuracy": 0.746269, + "f1": 0.67368, + "f1_weighted": 0.774293, + "ap": 0.342399, + "ap_weighted": 0.342399 + }, + { + "accuracy": 0.564179, + "f1": 0.514666, + "f1_weighted": 0.616006, + "ap": 0.221233, + "ap_weighted": 0.221233 + }, + { + "accuracy": 0.644776, + "f1": 0.581799, + "f1_weighted": 0.687891, + "ap": 0.261642, + "ap_weighted": 0.261642 + }, + { + "accuracy": 0.692537, + "f1": 0.615436, + "f1_weighted": 0.728004, + "ap": 0.278957, + "ap_weighted": 0.278957 + }, + { + "accuracy": 0.719403, + "f1": 0.626613, + "f1_weighted": 0.748296, + "ap": 0.277263, + "ap_weighted": 0.277263 + }, + { + "accuracy": 0.734328, + "f1": 0.650588, + "f1_weighted": 0.762412, + "ap": 0.307231, + "ap_weighted": 0.307231 + }, + { + "accuracy": 0.710448, + "f1": 0.610021, + "f1_weighted": 0.739394, + "ap": 0.257913, + "ap_weighted": 0.257913 + }, + { + "accuracy": 0.701493, + "f1": 0.622816, + "f1_weighted": 0.735432, + "ap": 0.284478, + "ap_weighted": 0.284478 + }, + { + "accuracy": 0.641791, + "f1": 0.569851, + "f1_weighted": 0.68485, + "ap": 0.244032, + "ap_weighted": 0.244032 + } + ], + "main_score": 0.676418, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.634335, + "f1": 0.612434, + "f1_weighted": 0.647507, + "ap": 0.767076, + "ap_weighted": 0.767076, + "scores_per_experiment": [ + { + "accuracy": 0.706009, + "f1": 0.674493, + "f1_weighted": 0.714485, + "ap": 0.793134, + "ap_weighted": 0.793134 + }, + { + "accuracy": 0.660944, + "f1": 0.633276, + "f1_weighted": 0.673049, + "ap": 0.773211, + "ap_weighted": 0.773211 + }, + { + "accuracy": 0.667382, + "f1": 0.639778, + "f1_weighted": 0.679151, + "ap": 0.776889, + "ap_weighted": 0.776889 + }, + { + "accuracy": 0.553648, + "f1": 0.543344, + "f1_weighted": 0.570429, + "ap": 0.737706, + "ap_weighted": 0.737706 + }, + { + "accuracy": 0.650215, + "f1": 0.619195, + "f1_weighted": 0.662109, + "ap": 0.763291, + "ap_weighted": 0.763291 + }, + { + "accuracy": 0.658798, + "f1": 0.634996, + "f1_weighted": 0.6718, + "ap": 0.777255, + "ap_weighted": 0.777255 + }, + { + "accuracy": 0.603004, + "f1": 0.587265, + "f1_weighted": 0.619089, + "ap": 0.757116, + "ap_weighted": 0.757116 + }, + { + "accuracy": 0.581545, + "f1": 0.576472, + "f1_weighted": 0.594774, + "ap": 0.767332, + "ap_weighted": 0.767332 + }, + { + "accuracy": 0.607296, + "f1": 0.588734, + "f1_weighted": 0.623233, + "ap": 0.755259, + "ap_weighted": 0.755259 + }, + { + "accuracy": 0.654506, + "f1": 0.626786, + "f1_weighted": 0.666948, + "ap": 0.769566, + "ap_weighted": 0.769566 + } + ], + "main_score": 0.634335, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ] + }, + { + "accuracy": 0.456009, + "f1": 0.364592, + "f1_weighted": 0.518903, + "ap": 0.102143, + "ap_weighted": 0.102143, + "scores_per_experiment": [ + { + "accuracy": 0.675966, + "f1": 0.48234, + "f1_weighted": 0.736431, + "ap": 0.102755, + "ap_weighted": 0.102755 + }, + { + "accuracy": 0.334764, + "f1": 0.311308, + "f1_weighted": 0.413314, + "ap": 0.103337, + "ap_weighted": 0.103337 + }, + { + "accuracy": 0.238197, + "f1": 0.234358, + "f1_weighted": 0.277873, + "ap": 0.100461, + "ap_weighted": 0.100461 + }, + { + "accuracy": 0.712446, + "f1": 0.499407, + "f1_weighted": 0.761502, + "ap": 0.105401, + "ap_weighted": 0.105401 + }, + { + "accuracy": 0.690987, + "f1": 0.47227, + "f1_weighted": 0.744937, + "ap": 0.097028, + "ap_weighted": 0.097028 + }, + { + "accuracy": 0.321888, + "f1": 0.30022, + "f1_weighted": 0.399047, + "ap": 0.100101, + "ap_weighted": 0.100101 + }, + { + "accuracy": 0.306867, + "f1": 0.293114, + "f1_weighted": 0.372248, + "ap": 0.108163, + "ap_weighted": 0.108163 + }, + { + "accuracy": 0.311159, + "f1": 0.291823, + "f1_weighted": 0.385738, + "ap": 0.099018, + "ap_weighted": 0.099018 + }, + { + "accuracy": 0.304721, + "f1": 0.290489, + "f1_weighted": 0.371138, + "ap": 0.105843, + "ap_weighted": 0.105843 + }, + { + "accuracy": 0.66309, + "f1": 0.470589, + "f1_weighted": 0.726801, + "ap": 0.099323, + "ap_weighted": 0.099323 + } + ], + "main_score": 0.456009, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ] + } + ], + "test": [ + { + "accuracy": 0.69018, + "f1": 0.569854, + "f1_weighted": 0.74927, + "ap": 0.194814, + "ap_weighted": 0.194814, + "scores_per_experiment": [ + { + "accuracy": 0.752624, + "f1": 0.622096, + "f1_weighted": 0.797909, + "ap": 0.23354, + "ap_weighted": 0.23354 + }, + { + "accuracy": 0.701649, + "f1": 0.582801, + "f1_weighted": 0.75907, + "ap": 0.207082, + "ap_weighted": 0.207082 + }, + { + "accuracy": 0.7009, + "f1": 0.5765, + "f1_weighted": 0.758196, + "ap": 0.195997, + "ap_weighted": 0.195997 + }, + { + "accuracy": 0.628186, + "f1": 0.522324, + "f1_weighted": 0.700334, + "ap": 0.165375, + "ap_weighted": 0.165375 + }, + { + "accuracy": 0.598201, + "f1": 0.504541, + "f1_weighted": 0.675066, + "ap": 0.161472, + "ap_weighted": 0.161472 + }, + { + "accuracy": 0.724888, + "f1": 0.600191, + "f1_weighted": 0.776942, + "ap": 0.217968, + "ap_weighted": 0.217968 + }, + { + "accuracy": 0.694153, + "f1": 0.561354, + "f1_weighted": 0.752411, + "ap": 0.176056, + "ap_weighted": 0.176056 + }, + { + "accuracy": 0.681409, + "f1": 0.562718, + "f1_weighted": 0.743061, + "ap": 0.188722, + "ap_weighted": 0.188722 + }, + { + "accuracy": 0.746627, + "f1": 0.617038, + "f1_weighted": 0.793386, + "ap": 0.229497, + "ap_weighted": 0.229497 + }, + { + "accuracy": 0.673163, + "f1": 0.548973, + "f1_weighted": 0.736323, + "ap": 0.172431, + "ap_weighted": 0.172431 + } + ], + "main_score": 0.69018, + "hf_subset": "en-ext", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.692537, + "f1": 0.633314, + "f1_weighted": 0.722182, + "ap": 0.3211, + "ap_weighted": 0.3211, + "scores_per_experiment": [ + { + "accuracy": 0.659701, + "f1": 0.596027, + "f1_weighted": 0.693693, + "ap": 0.280716, + "ap_weighted": 0.280716 + }, + { + "accuracy": 0.723881, + "f1": 0.660763, + "f1_weighted": 0.74987, + "ap": 0.342506, + "ap_weighted": 0.342506 + }, + { + "accuracy": 0.6, + "f1": 0.564213, + "f1_weighted": 0.640261, + "ap": 0.281343, + "ap_weighted": 0.281343 + }, + { + "accuracy": 0.673134, + "f1": 0.619771, + "f1_weighted": 0.706513, + "ap": 0.311324, + "ap_weighted": 0.311324 + }, + { + "accuracy": 0.695522, + "f1": 0.637527, + "f1_weighted": 0.725819, + "ap": 0.323838, + "ap_weighted": 0.323838 + }, + { + "accuracy": 0.732836, + "f1": 0.671765, + "f1_weighted": 0.757982, + "ap": 0.356257, + "ap_weighted": 0.356257 + }, + { + "accuracy": 0.773134, + "f1": 0.701625, + "f1_weighted": 0.790576, + "ap": 0.380239, + "ap_weighted": 0.380239 + }, + { + "accuracy": 0.743284, + "f1": 0.667417, + "f1_weighted": 0.764147, + "ap": 0.338785, + "ap_weighted": 0.338785 + }, + { + "accuracy": 0.668657, + "f1": 0.614059, + "f1_weighted": 0.702455, + "ap": 0.3047, + "ap_weighted": 0.3047 + }, + { + "accuracy": 0.655224, + "f1": 0.599973, + "f1_weighted": 0.690504, + "ap": 0.291294, + "ap_weighted": 0.291294 + } + ], + "main_score": 0.692537, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.620557, + "f1": 0.600785, + "f1_weighted": 0.634836, + "ap": 0.760584, + "ap_weighted": 0.760584, + "scores_per_experiment": [ + { + "accuracy": 0.658458, + "f1": 0.628557, + "f1_weighted": 0.669855, + "ap": 0.767364, + "ap_weighted": 0.767364 + }, + { + "accuracy": 0.616702, + "f1": 0.591498, + "f1_weighted": 0.63126, + "ap": 0.750014, + "ap_weighted": 0.750014 + }, + { + "accuracy": 0.664882, + "f1": 0.640979, + "f1_weighted": 0.67728, + "ap": 0.77883, + "ap_weighted": 0.77883 + }, + { + "accuracy": 0.59636, + "f1": 0.587217, + "f1_weighted": 0.61129, + "ap": 0.763799, + "ap_weighted": 0.763799 + }, + { + "accuracy": 0.61242, + "f1": 0.593004, + "f1_weighted": 0.627838, + "ap": 0.755117, + "ap_weighted": 0.755117 + }, + { + "accuracy": 0.649893, + "f1": 0.630303, + "f1_weighted": 0.663651, + "ap": 0.776687, + "ap_weighted": 0.776687 + }, + { + "accuracy": 0.61242, + "f1": 0.594566, + "f1_weighted": 0.627905, + "ap": 0.757412, + "ap_weighted": 0.757412 + }, + { + "accuracy": 0.59743, + "f1": 0.590901, + "f1_weighted": 0.611154, + "ap": 0.771261, + "ap_weighted": 0.771261 + }, + { + "accuracy": 0.572805, + "f1": 0.55249, + "f1_weighted": 0.589853, + "ap": 0.732012, + "ap_weighted": 0.732012 + }, + { + "accuracy": 0.624197, + "f1": 0.598335, + "f1_weighted": 0.638274, + "ap": 0.753342, + "ap_weighted": 0.753342 + } + ], + "main_score": 0.620557, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ] + }, + { + "accuracy": 0.458994, + "f1": 0.368328, + "f1_weighted": 0.520387, + "ap": 0.105706, + "ap_weighted": 0.105706, + "scores_per_experiment": [ + { + "accuracy": 0.675589, + "f1": 0.470822, + "f1_weighted": 0.732332, + "ap": 0.101349, + "ap_weighted": 0.101349 + }, + { + "accuracy": 0.329764, + "f1": 0.308976, + "f1_weighted": 0.404194, + "ap": 0.106741, + "ap_weighted": 0.106741 + }, + { + "accuracy": 0.251606, + "f1": 0.246485, + "f1_weighted": 0.295835, + "ap": 0.103629, + "ap_weighted": 0.103629 + }, + { + "accuracy": 0.726981, + "f1": 0.508144, + "f1_weighted": 0.768781, + "ap": 0.11079, + "ap_weighted": 0.11079 + }, + { + "accuracy": 0.685225, + "f1": 0.480876, + "f1_weighted": 0.739625, + "ap": 0.10401, + "ap_weighted": 0.10401 + }, + { + "accuracy": 0.321199, + "f1": 0.300145, + "f1_weighted": 0.396579, + "ap": 0.102271, + "ap_weighted": 0.102271 + }, + { + "accuracy": 0.307281, + "f1": 0.294232, + "f1_weighted": 0.37047, + "ap": 0.110895, + "ap_weighted": 0.110895 + }, + { + "accuracy": 0.32227, + "f1": 0.301515, + "f1_weighted": 0.397167, + "ap": 0.103231, + "ap_weighted": 0.103231 + }, + { + "accuracy": 0.309422, + "f1": 0.294687, + "f1_weighted": 0.375676, + "ap": 0.108195, + "ap_weighted": 0.108195 + }, + { + "accuracy": 0.6606, + "f1": 0.477399, + "f1_weighted": 0.723212, + "ap": 0.105949, + "ap_weighted": 0.105949 + } + ], + "main_score": 0.458994, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ] + } + ] + }, + "evaluation_time": 13.947409868240356, + "kg_co2_emissions": null } \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/AmazonPolarityClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/AmazonPolarityClassification.json index 5f63285d6f..a58224efe5 100644 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/AmazonPolarityClassification.json +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/AmazonPolarityClassification.json @@ -1,15 +1,95 @@ { - "test": { - "accuracy": 0.67818775, - "accuracy_stderr": 0.053793244794885745, - "ap": 0.6277363933451441, - "ap_stderr": 0.04450756480512419, - "evaluation_time": 12894.28, - "f1": 0.6695367269475989, - "f1_stderr": 0.06445830484057914, - "main_score": 0.67818775 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "AmazonPolarityClassification", - "dataset_revision": "80714f8dcf8cefc218ef4f8c5a966dd83f75a0e1" + "dataset_revision": "e2d317d38cd51312af73b3d32a06d1a08b442046", + "task_name": "AmazonPolarityClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.73838, + "f1": 0.736566, + "f1_weighted": 0.736566, + "ap": 0.681219, + "ap_weighted": 0.681219, + "scores_per_experiment": [ + { + "accuracy": 0.688937, + "f1": 0.688224, + "f1_weighted": 0.688224, + "ap": 0.62705, + "ap_weighted": 0.62705 + }, + { + "accuracy": 0.697595, + "f1": 0.696965, + "f1_weighted": 0.696965, + "ap": 0.641758, + "ap_weighted": 0.641758 + }, + { + "accuracy": 0.75584, + "f1": 0.75536, + "f1_weighted": 0.75536, + "ap": 0.688049, + "ap_weighted": 0.688049 + }, + { + "accuracy": 0.720585, + "f1": 0.713842, + "f1_weighted": 0.713842, + "ap": 0.680507, + "ap_weighted": 0.680507 + }, + { + "accuracy": 0.761905, + "f1": 0.760772, + "f1_weighted": 0.760772, + "ap": 0.710493, + "ap_weighted": 0.710493 + }, + { + "accuracy": 0.718422, + "f1": 0.713344, + "f1_weighted": 0.713344, + "ap": 0.674228, + "ap_weighted": 0.674228 + }, + { + "accuracy": 0.77248, + "f1": 0.77195, + "f1_weighted": 0.77195, + "ap": 0.718412, + "ap_weighted": 0.718412 + }, + { + "accuracy": 0.772177, + "f1": 0.772064, + "f1_weighted": 0.772064, + "ap": 0.713626, + "ap_weighted": 0.713626 + }, + { + "accuracy": 0.749247, + "f1": 0.746629, + "f1_weighted": 0.746629, + "ap": 0.676251, + "ap_weighted": 0.676251 + }, + { + "accuracy": 0.746613, + "f1": 0.746514, + "f1_weighted": 0.746514, + "ap": 0.681815, + "ap_weighted": 0.681815 + } + ], + "main_score": 0.73838, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1085.325970172882, + "kg_co2_emissions": null } \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/AmazonReviewsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/AmazonReviewsClassification.json index 70488dfc2f..6afb995bff 100644 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/AmazonReviewsClassification.json +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/AmazonReviewsClassification.json @@ -1,15 +1,757 @@ { - "test": { - "en": { - "accuracy": 0.38482, - "accuracy_stderr": 0.020790084174913764, - "f1": 0.3721004327687625, - "f1_stderr": 0.01810540569261934, - "main_score": 0.38482 - }, - "evaluation_time": 86.33 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "AmazonReviewsClassification", - "dataset_revision": "c379a6705fec24a2493fa68e011692605f44e119" + "dataset_revision": "1399c76144fd37290681b995c656ef9b2e06e26d", + "task_name": "AmazonReviewsClassification", + "mteb_version": "1.34.14", + "scores": { + "validation": [ + { + "accuracy": 0.37022, + "f1": 0.363551, + "f1_weighted": 0.363551, + "scores_per_experiment": [ + { + "accuracy": 0.3738, + "f1": 0.371889, + "f1_weighted": 0.371889 + }, + { + "accuracy": 0.3984, + "f1": 0.388755, + "f1_weighted": 0.388755 + }, + { + "accuracy": 0.3774, + "f1": 0.369559, + "f1_weighted": 0.369559 + }, + { + "accuracy": 0.3636, + "f1": 0.360348, + "f1_weighted": 0.360348 + }, + { + "accuracy": 0.3804, + "f1": 0.356885, + "f1_weighted": 0.356885 + }, + { + "accuracy": 0.3508, + "f1": 0.344572, + "f1_weighted": 0.344572 + }, + { + "accuracy": 0.3424, + "f1": 0.342228, + "f1_weighted": 0.342228 + }, + { + "accuracy": 0.4002, + "f1": 0.396734, + "f1_weighted": 0.396734 + }, + { + "accuracy": 0.36, + "f1": 0.354855, + "f1_weighted": 0.354855 + }, + { + "accuracy": 0.3552, + "f1": 0.349683, + "f1_weighted": 0.349683 + } + ], + "main_score": 0.37022, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.31248, + "f1": 0.305391, + "f1_weighted": 0.305391, + "scores_per_experiment": [ + { + "accuracy": 0.3356, + "f1": 0.327623, + "f1_weighted": 0.327623 + }, + { + "accuracy": 0.3262, + "f1": 0.310355, + "f1_weighted": 0.310355 + }, + { + "accuracy": 0.286, + "f1": 0.281311, + "f1_weighted": 0.281311 + }, + { + "accuracy": 0.292, + "f1": 0.283555, + "f1_weighted": 0.283555 + }, + { + "accuracy": 0.3072, + "f1": 0.310985, + "f1_weighted": 0.310985 + }, + { + "accuracy": 0.3284, + "f1": 0.311594, + "f1_weighted": 0.311594 + }, + { + "accuracy": 0.2772, + "f1": 0.273796, + "f1_weighted": 0.273796 + }, + { + "accuracy": 0.3154, + "f1": 0.309291, + "f1_weighted": 0.309291 + }, + { + "accuracy": 0.3224, + "f1": 0.322271, + "f1_weighted": 0.322271 + }, + { + "accuracy": 0.3344, + "f1": 0.323131, + "f1_weighted": 0.323131 + } + ], + "main_score": 0.31248, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ] + }, + { + "accuracy": 0.30996, + "f1": 0.301506, + "f1_weighted": 0.301506, + "scores_per_experiment": [ + { + "accuracy": 0.3192, + "f1": 0.302287, + "f1_weighted": 0.302287 + }, + { + "accuracy": 0.2916, + "f1": 0.293085, + "f1_weighted": 0.293085 + }, + { + "accuracy": 0.323, + "f1": 0.307231, + "f1_weighted": 0.307231 + }, + { + "accuracy": 0.3106, + "f1": 0.311743, + "f1_weighted": 0.311743 + }, + { + "accuracy": 0.306, + "f1": 0.28587, + "f1_weighted": 0.28587 + }, + { + "accuracy": 0.295, + "f1": 0.292734, + "f1_weighted": 0.292734 + }, + { + "accuracy": 0.306, + "f1": 0.295453, + "f1_weighted": 0.295453 + }, + { + "accuracy": 0.326, + "f1": 0.317784, + "f1_weighted": 0.317784 + }, + { + "accuracy": 0.3298, + "f1": 0.319265, + "f1_weighted": 0.319265 + }, + { + "accuracy": 0.2924, + "f1": 0.289606, + "f1_weighted": 0.289606 + } + ], + "main_score": 0.30996, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ] + }, + { + "accuracy": 0.29796, + "f1": 0.293143, + "f1_weighted": 0.293143, + "scores_per_experiment": [ + { + "accuracy": 0.315, + "f1": 0.308191, + "f1_weighted": 0.308191 + }, + { + "accuracy": 0.2936, + "f1": 0.285196, + "f1_weighted": 0.285196 + }, + { + "accuracy": 0.3284, + "f1": 0.324539, + "f1_weighted": 0.324539 + }, + { + "accuracy": 0.2668, + "f1": 0.258642, + "f1_weighted": 0.258642 + }, + { + "accuracy": 0.3222, + "f1": 0.323125, + "f1_weighted": 0.323125 + }, + { + "accuracy": 0.304, + "f1": 0.299609, + "f1_weighted": 0.299609 + }, + { + "accuracy": 0.3036, + "f1": 0.299772, + "f1_weighted": 0.299772 + }, + { + "accuracy": 0.2806, + "f1": 0.277151, + "f1_weighted": 0.277151 + }, + { + "accuracy": 0.2608, + "f1": 0.256186, + "f1_weighted": 0.256186 + }, + { + "accuracy": 0.3046, + "f1": 0.299022, + "f1_weighted": 0.299022 + } + ], + "main_score": 0.29796, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ] + }, + { + "accuracy": 0.2182, + "f1": 0.199591, + "f1_weighted": 0.199591, + "scores_per_experiment": [ + { + "accuracy": 0.2152, + "f1": 0.195857, + "f1_weighted": 0.195857 + }, + { + "accuracy": 0.2186, + "f1": 0.202903, + "f1_weighted": 0.202903 + }, + { + "accuracy": 0.2448, + "f1": 0.232531, + "f1_weighted": 0.232531 + }, + { + "accuracy": 0.2274, + "f1": 0.200345, + "f1_weighted": 0.200345 + }, + { + "accuracy": 0.1994, + "f1": 0.192182, + "f1_weighted": 0.192182 + }, + { + "accuracy": 0.202, + "f1": 0.183987, + "f1_weighted": 0.183987 + }, + { + "accuracy": 0.2248, + "f1": 0.20719, + "f1_weighted": 0.20719 + }, + { + "accuracy": 0.2222, + "f1": 0.192226, + "f1_weighted": 0.192226 + }, + { + "accuracy": 0.197, + "f1": 0.171615, + "f1_weighted": 0.171615 + }, + { + "accuracy": 0.2306, + "f1": 0.217072, + "f1_weighted": 0.217072 + } + ], + "main_score": 0.2182, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ] + }, + { + "accuracy": 0.21482, + "f1": 0.181735, + "f1_weighted": 0.181735, + "scores_per_experiment": [ + { + "accuracy": 0.2134, + "f1": 0.178033, + "f1_weighted": 0.178033 + }, + { + "accuracy": 0.1904, + "f1": 0.165551, + "f1_weighted": 0.165551 + }, + { + "accuracy": 0.2018, + "f1": 0.151679, + "f1_weighted": 0.151679 + }, + { + "accuracy": 0.203, + "f1": 0.166301, + "f1_weighted": 0.166301 + }, + { + "accuracy": 0.1994, + "f1": 0.160065, + "f1_weighted": 0.160065 + }, + { + "accuracy": 0.2354, + "f1": 0.197684, + "f1_weighted": 0.197684 + }, + { + "accuracy": 0.2228, + "f1": 0.187186, + "f1_weighted": 0.187186 + }, + { + "accuracy": 0.234, + "f1": 0.205432, + "f1_weighted": 0.205432 + }, + { + "accuracy": 0.2222, + "f1": 0.208587, + "f1_weighted": 0.208587 + }, + { + "accuracy": 0.2258, + "f1": 0.196833, + "f1_weighted": 0.196833 + } + ], + "main_score": 0.21482, + "hf_subset": "zh", + "languages": [ + "cmn-Hans" + ] + } + ], + "test": [ + { + "accuracy": 0.37062, + "f1": 0.363274, + "f1_weighted": 0.363274, + "scores_per_experiment": [ + { + "accuracy": 0.3806, + "f1": 0.374137, + "f1_weighted": 0.374137 + }, + { + "accuracy": 0.4026, + "f1": 0.391767, + "f1_weighted": 0.391767 + }, + { + "accuracy": 0.3668, + "f1": 0.358229, + "f1_weighted": 0.358229 + }, + { + "accuracy": 0.3768, + "f1": 0.374368, + "f1_weighted": 0.374368 + }, + { + "accuracy": 0.3788, + "f1": 0.357203, + "f1_weighted": 0.357203 + }, + { + "accuracy": 0.3462, + "f1": 0.338787, + "f1_weighted": 0.338787 + }, + { + "accuracy": 0.346, + "f1": 0.345773, + "f1_weighted": 0.345773 + }, + { + "accuracy": 0.4008, + "f1": 0.396911, + "f1_weighted": 0.396911 + }, + { + "accuracy": 0.3524, + "f1": 0.345902, + "f1_weighted": 0.345902 + }, + { + "accuracy": 0.3552, + "f1": 0.349663, + "f1_weighted": 0.349663 + } + ], + "main_score": 0.37062, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.3089, + "f1": 0.301503, + "f1_weighted": 0.301503, + "scores_per_experiment": [ + { + "accuracy": 0.3316, + "f1": 0.322149, + "f1_weighted": 0.322149 + }, + { + "accuracy": 0.3284, + "f1": 0.313445, + "f1_weighted": 0.313445 + }, + { + "accuracy": 0.2784, + "f1": 0.271874, + "f1_weighted": 0.271874 + }, + { + "accuracy": 0.2874, + "f1": 0.280075, + "f1_weighted": 0.280075 + }, + { + "accuracy": 0.313, + "f1": 0.317658, + "f1_weighted": 0.317658 + }, + { + "accuracy": 0.3158, + "f1": 0.299925, + "f1_weighted": 0.299925 + }, + { + "accuracy": 0.2806, + "f1": 0.27737, + "f1_weighted": 0.27737 + }, + { + "accuracy": 0.316, + "f1": 0.309854, + "f1_weighted": 0.309854 + }, + { + "accuracy": 0.3098, + "f1": 0.308702, + "f1_weighted": 0.308702 + }, + { + "accuracy": 0.328, + "f1": 0.313982, + "f1_weighted": 0.313982 + } + ], + "main_score": 0.3089, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ] + }, + { + "accuracy": 0.30538, + "f1": 0.29692, + "f1_weighted": 0.29692, + "scores_per_experiment": [ + { + "accuracy": 0.329, + "f1": 0.313668, + "f1_weighted": 0.313668 + }, + { + "accuracy": 0.297, + "f1": 0.298324, + "f1_weighted": 0.298324 + }, + { + "accuracy": 0.307, + "f1": 0.291661, + "f1_weighted": 0.291661 + }, + { + "accuracy": 0.2994, + "f1": 0.300369, + "f1_weighted": 0.300369 + }, + { + "accuracy": 0.2958, + "f1": 0.273972, + "f1_weighted": 0.273972 + }, + { + "accuracy": 0.2902, + "f1": 0.288803, + "f1_weighted": 0.288803 + }, + { + "accuracy": 0.3144, + "f1": 0.303202, + "f1_weighted": 0.303202 + }, + { + "accuracy": 0.3104, + "f1": 0.302459, + "f1_weighted": 0.302459 + }, + { + "accuracy": 0.3246, + "f1": 0.313604, + "f1_weighted": 0.313604 + }, + { + "accuracy": 0.286, + "f1": 0.283145, + "f1_weighted": 0.283145 + } + ], + "main_score": 0.30538, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ] + }, + { + "accuracy": 0.29884, + "f1": 0.294264, + "f1_weighted": 0.294264, + "scores_per_experiment": [ + { + "accuracy": 0.306, + "f1": 0.299018, + "f1_weighted": 0.299018 + }, + { + "accuracy": 0.3054, + "f1": 0.297167, + "f1_weighted": 0.297167 + }, + { + "accuracy": 0.3148, + "f1": 0.312787, + "f1_weighted": 0.312787 + }, + { + "accuracy": 0.2534, + "f1": 0.24926, + "f1_weighted": 0.24926 + }, + { + "accuracy": 0.3256, + "f1": 0.32629, + "f1_weighted": 0.32629 + }, + { + "accuracy": 0.3172, + "f1": 0.311979, + "f1_weighted": 0.311979 + }, + { + "accuracy": 0.3124, + "f1": 0.309059, + "f1_weighted": 0.309059 + }, + { + "accuracy": 0.2854, + "f1": 0.281386, + "f1_weighted": 0.281386 + }, + { + "accuracy": 0.272, + "f1": 0.264988, + "f1_weighted": 0.264988 + }, + { + "accuracy": 0.2962, + "f1": 0.290705, + "f1_weighted": 0.290705 + } + ], + "main_score": 0.29884, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ] + }, + { + "accuracy": 0.21642, + "f1": 0.197664, + "f1_weighted": 0.197664, + "scores_per_experiment": [ + { + "accuracy": 0.2196, + "f1": 0.196806, + "f1_weighted": 0.196806 + }, + { + "accuracy": 0.216, + "f1": 0.199955, + "f1_weighted": 0.199955 + }, + { + "accuracy": 0.2322, + "f1": 0.219511, + "f1_weighted": 0.219511 + }, + { + "accuracy": 0.226, + "f1": 0.201594, + "f1_weighted": 0.201594 + }, + { + "accuracy": 0.2088, + "f1": 0.202181, + "f1_weighted": 0.202181 + }, + { + "accuracy": 0.2008, + "f1": 0.183542, + "f1_weighted": 0.183542 + }, + { + "accuracy": 0.2158, + "f1": 0.196392, + "f1_weighted": 0.196392 + }, + { + "accuracy": 0.22, + "f1": 0.192428, + "f1_weighted": 0.192428 + }, + { + "accuracy": 0.2026, + "f1": 0.176716, + "f1_weighted": 0.176716 + }, + { + "accuracy": 0.2224, + "f1": 0.207518, + "f1_weighted": 0.207518 + } + ], + "main_score": 0.21642, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ] + }, + { + "accuracy": 0.21482, + "f1": 0.180156, + "f1_weighted": 0.180156, + "scores_per_experiment": [ + { + "accuracy": 0.2192, + "f1": 0.180855, + "f1_weighted": 0.180855 + }, + { + "accuracy": 0.191, + "f1": 0.166241, + "f1_weighted": 0.166241 + }, + { + "accuracy": 0.2122, + "f1": 0.158757, + "f1_weighted": 0.158757 + }, + { + "accuracy": 0.1984, + "f1": 0.156656, + "f1_weighted": 0.156656 + }, + { + "accuracy": 0.206, + "f1": 0.164446, + "f1_weighted": 0.164446 + }, + { + "accuracy": 0.2216, + "f1": 0.185426, + "f1_weighted": 0.185426 + }, + { + "accuracy": 0.2072, + "f1": 0.172392, + "f1_weighted": 0.172392 + }, + { + "accuracy": 0.2232, + "f1": 0.194476, + "f1_weighted": 0.194476 + }, + { + "accuracy": 0.2274, + "f1": 0.208832, + "f1_weighted": 0.208832 + }, + { + "accuracy": 0.242, + "f1": 0.213479, + "f1_weighted": 0.213479 + } + ], + "main_score": 0.21482, + "hf_subset": "zh", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 137.448570728302, + "kg_co2_emissions": null } \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/AngryTweetsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/AngryTweetsClassification.json new file mode 100644 index 0000000000..564123c20a --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/AngryTweetsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "20b0e6081892e78179356fada741b7afa381443d", + "task_name": "AngryTweetsClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.435339, + "f1": 0.424195, + "f1_weighted": 0.432906, + "scores_per_experiment": [ + { + "accuracy": 0.426934, + "f1": 0.422234, + "f1_weighted": 0.429963 + }, + { + "accuracy": 0.400191, + "f1": 0.395728, + "f1_weighted": 0.399207 + }, + { + "accuracy": 0.449857, + "f1": 0.432413, + "f1_weighted": 0.444636 + }, + { + "accuracy": 0.430755, + "f1": 0.428873, + "f1_weighted": 0.432536 + }, + { + "accuracy": 0.417383, + "f1": 0.409466, + "f1_weighted": 0.420128 + }, + { + "accuracy": 0.422159, + "f1": 0.410851, + "f1_weighted": 0.415474 + }, + { + "accuracy": 0.443171, + "f1": 0.435422, + "f1_weighted": 0.44292 + }, + { + "accuracy": 0.43553, + "f1": 0.426199, + "f1_weighted": 0.436379 + }, + { + "accuracy": 0.449857, + "f1": 0.430685, + "f1_weighted": 0.443249 + }, + { + "accuracy": 0.477555, + "f1": 0.450076, + "f1_weighted": 0.464562 + } + ], + "main_score": 0.435339, + "hf_subset": "default", + "languages": [ + "dan-Latn" + ] + } + ] + }, + "evaluation_time": 3.077916145324707, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ArguAna.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ArguAna.json deleted file mode 100644 index 6957acb602..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/ArguAna.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "test": { - "evaluation_time": 54.32, - "map_at_1": 0.26814, - "map_at_10": 0.41989, - "map_at_100": 0.42956, - "map_at_1000": 0.42969, - "map_at_3": 0.36972, - "map_at_5": 0.39757, - "ndcg_at_1": 0.26814, - "ndcg_at_10": 0.50828, - "ndcg_at_100": 0.54973, - "ndcg_at_1000": 0.55251, - "ndcg_at_3": 0.40434, - "ndcg_at_5": 0.45463, - "precision_at_1": 0.26814, - "precision_at_10": 0.07923, - "precision_at_100": 0.00974, - "precision_at_1000": 0.001, - "precision_at_3": 0.16833, - "precision_at_5": 0.12546, - "recall_at_1": 0.26814, - "recall_at_10": 0.79232, - "recall_at_100": 0.9744, - "recall_at_1000": 0.99502, - "recall_at_3": 0.50498, - "recall_at_5": 0.62731 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "ArguAna", - "dataset_revision": "5b3e3697907184a9b77a3c99ee9ea1a9cbb1e4e3" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ArxivClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ArxivClassification.json new file mode 100644 index 0000000000..6741ad22e5 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/ArxivClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f9bd92144ed76200d6eb3ce73a8bd4eba9ffdc85", + "task_name": "ArxivClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.57792, + "f1": 0.553105, + "f1_weighted": 0.560135, + "scores_per_experiment": [ + { + "accuracy": 0.5612, + "f1": 0.536993, + "f1_weighted": 0.545229 + }, + { + "accuracy": 0.6024, + "f1": 0.573598, + "f1_weighted": 0.583203 + }, + { + "accuracy": 0.5924, + "f1": 0.569764, + "f1_weighted": 0.576983 + }, + { + "accuracy": 0.604, + "f1": 0.575769, + "f1_weighted": 0.584295 + }, + { + "accuracy": 0.5836, + "f1": 0.547581, + "f1_weighted": 0.557656 + }, + { + "accuracy": 0.572, + "f1": 0.549614, + "f1_weighted": 0.555252 + }, + { + "accuracy": 0.594, + "f1": 0.570918, + "f1_weighted": 0.579294 + }, + { + "accuracy": 0.5724, + "f1": 0.550469, + "f1_weighted": 0.554016 + }, + { + "accuracy": 0.5316, + "f1": 0.505431, + "f1_weighted": 0.509906 + }, + { + "accuracy": 0.5656, + "f1": 0.55091, + "f1_weighted": 0.555517 + } + ], + "main_score": 0.57792, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 299.5171709060669, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ArxivClusteringP2P.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ArxivClusteringP2P.json deleted file mode 100644 index 69069129d7..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/ArxivClusteringP2P.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "test": { - "evaluation_time": 5598.18, - "v_measure": 0.3548695236674728, - "v_measure_std": 0.1364667933053161 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "ArxivClusteringP2P", - "dataset_revision": "0bbdb47bcbe3a90093699aefeed338a0f28a7ee8" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ArxivClusteringS2S.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ArxivClusteringS2S.json deleted file mode 100644 index d7c6238fad..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/ArxivClusteringS2S.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "test": { - "evaluation_time": 518.5, - "v_measure": 0.27180108855677926, - "v_measure_std": 0.14368416196949751 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "ArxivClusteringS2S", - "dataset_revision": "b73bd54100e5abfa6e3a23dcafb46fe4d2438dc3" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/AskUbuntuDupQuestions.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/AskUbuntuDupQuestions.json deleted file mode 100644 index 7323bcb525..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/AskUbuntuDupQuestions.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "test": { - "evaluation_time": 5.97, - "map": 0.6086144157716742, - "mrr": 0.7360319658796113 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "AskUbuntuDupQuestions", - "dataset_revision": "4d853f94cd57d85ec13805aeeac3ae3e5eb4c49c" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/BIOSSES.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/BIOSSES.json deleted file mode 100644 index ad99996711..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/BIOSSES.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "test": { - "cos_sim": { - "pearson": 0.8055684313101614, - "spearman": 0.7900420980306923 - }, - "euclidean": { - "pearson": 0.7852051241139544, - "spearman": 0.7900420980306923 - }, - "evaluation_time": 2.61, - "manhattan": { - "pearson": 0.78630271953975, - "spearman": 0.7868010603260279 - } - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "BIOSSES", - "dataset_revision": "9ee918f184421b6bd48b78f6c714d86546106103" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/Banking77Classification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/Banking77Classification.json index 90667fe01d..e3a945d959 100644 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/Banking77Classification.json +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/Banking77Classification.json @@ -1,13 +1,73 @@ { - "test": { - "accuracy": 0.7925649350649351, - "accuracy_stderr": 0.010860328945921847, - "evaluation_time": 29.68, - "f1": 0.784267382525839, - "f1_stderr": 0.01140292267067428, - "main_score": 0.7925649350649351 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "Banking77Classification", - "dataset_revision": "44fa15921b4c889113cc5df03dd4901b49161ab7" + "dataset_revision": "0fd18e25b25c072e09e0d92ab615fda904d66300", + "task_name": "Banking77Classification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.790714, + "f1": 0.782329, + "f1_weighted": 0.782329, + "scores_per_experiment": [ + { + "accuracy": 0.783117, + "f1": 0.773475, + "f1_weighted": 0.773475 + }, + { + "accuracy": 0.803896, + "f1": 0.796872, + "f1_weighted": 0.796872 + }, + { + "accuracy": 0.796753, + "f1": 0.790149, + "f1_weighted": 0.790149 + }, + { + "accuracy": 0.788961, + "f1": 0.781573, + "f1_weighted": 0.781573 + }, + { + "accuracy": 0.788312, + "f1": 0.778588, + "f1_weighted": 0.778588 + }, + { + "accuracy": 0.803896, + "f1": 0.798686, + "f1_weighted": 0.798686 + }, + { + "accuracy": 0.783442, + "f1": 0.774393, + "f1_weighted": 0.774393 + }, + { + "accuracy": 0.780844, + "f1": 0.76898, + "f1_weighted": 0.76898 + }, + { + "accuracy": 0.790909, + "f1": 0.783328, + "f1_weighted": 0.783328 + }, + { + "accuracy": 0.787013, + "f1": 0.777247, + "f1_weighted": 0.777247 + } + ], + "main_score": 0.790714, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 17.07085871696472, + "kg_co2_emissions": null } \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/BengaliDocumentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/BengaliDocumentClassification.json new file mode 100644 index 0000000000..c668277890 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/BengaliDocumentClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "1c6e67433da618073295b7c90f1c55fa8e78f35c", + "task_name": "BengaliDocumentClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.105225, + "f1": 0.076524, + "f1_weighted": 0.106198, + "scores_per_experiment": [ + { + "accuracy": 0.088867, + "f1": 0.068936, + "f1_weighted": 0.055946 + }, + { + "accuracy": 0.124023, + "f1": 0.099205, + "f1_weighted": 0.11596 + }, + { + "accuracy": 0.118652, + "f1": 0.082852, + "f1_weighted": 0.163355 + }, + { + "accuracy": 0.128906, + "f1": 0.094031, + "f1_weighted": 0.134508 + }, + { + "accuracy": 0.113281, + "f1": 0.094247, + "f1_weighted": 0.122192 + }, + { + "accuracy": 0.112793, + "f1": 0.077721, + "f1_weighted": 0.126304 + }, + { + "accuracy": 0.07666, + "f1": 0.056263, + "f1_weighted": 0.077725 + }, + { + "accuracy": 0.099609, + "f1": 0.06958, + "f1_weighted": 0.063221 + }, + { + "accuracy": 0.071289, + "f1": 0.058182, + "f1_weighted": 0.043058 + }, + { + "accuracy": 0.118164, + "f1": 0.064227, + "f1_weighted": 0.159715 + } + ], + "main_score": 0.105225, + "hf_subset": "default", + "languages": [ + "ben-Beng" + ] + } + ] + }, + "evaluation_time": 54.286158084869385, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/BengaliHateSpeechClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/BengaliHateSpeechClassification.json new file mode 100644 index 0000000000..8878204b3a --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/BengaliHateSpeechClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "99612296bc093f0720cac7d7cbfcb67eecf1ca2f", + "task_name": "BengaliHateSpeechClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.3021, + "f1": 0.179511, + "f1_weighted": 0.240116, + "scores_per_experiment": [ + { + "accuracy": 0.379883, + "f1": 0.222345, + "f1_weighted": 0.324414 + }, + { + "accuracy": 0.372559, + "f1": 0.196948, + "f1_weighted": 0.302027 + }, + { + "accuracy": 0.317383, + "f1": 0.188984, + "f1_weighted": 0.273867 + }, + { + "accuracy": 0.319336, + "f1": 0.161817, + "f1_weighted": 0.251567 + }, + { + "accuracy": 0.310059, + "f1": 0.178026, + "f1_weighted": 0.25958 + }, + { + "accuracy": 0.222656, + "f1": 0.145533, + "f1_weighted": 0.150166 + }, + { + "accuracy": 0.20459, + "f1": 0.149488, + "f1_weighted": 0.128598 + }, + { + "accuracy": 0.375488, + "f1": 0.215781, + "f1_weighted": 0.31202 + }, + { + "accuracy": 0.189941, + "f1": 0.147589, + "f1_weighted": 0.116087 + }, + { + "accuracy": 0.329102, + "f1": 0.188594, + "f1_weighted": 0.282829 + } + ], + "main_score": 0.179511, + "hf_subset": "default", + "languages": [ + "ben-Beng" + ] + } + ] + }, + "evaluation_time": 3.4576399326324463, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/BengaliSentimentAnalysis.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/BengaliSentimentAnalysis.json new file mode 100644 index 0000000000..6c0fe992fe --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/BengaliSentimentAnalysis.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "a4b3685b1854cc26c554dda4c7cb918a36a6fb6c", + "task_name": "BengaliSentimentAnalysis", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.461768, + "f1": 0.438021, + "f1_weighted": 0.459742, + "ap": 0.727173, + "ap_weighted": 0.727173, + "scores_per_experiment": [ + { + "accuracy": 0.480957, + "f1": 0.473421, + "f1_weighted": 0.501104, + "ap": 0.73274, + "ap_weighted": 0.73274 + }, + { + "accuracy": 0.396973, + "f1": 0.393499, + "f1_weighted": 0.373327, + "ap": 0.734341, + "ap_weighted": 0.734341 + }, + { + "accuracy": 0.491699, + "f1": 0.461762, + "f1_weighted": 0.517545, + "ap": 0.712195, + "ap_weighted": 0.712195 + }, + { + "accuracy": 0.529785, + "f1": 0.46698, + "f1_weighted": 0.547385, + "ap": 0.70781, + "ap_weighted": 0.70781 + }, + { + "accuracy": 0.473145, + "f1": 0.470265, + "f1_weighted": 0.487429, + "ap": 0.739743, + "ap_weighted": 0.739743 + }, + { + "accuracy": 0.580566, + "f1": 0.493032, + "f1_weighted": 0.585607, + "ap": 0.717002, + "ap_weighted": 0.717002 + }, + { + "accuracy": 0.401855, + "f1": 0.398673, + "f1_weighted": 0.379447, + "ap": 0.736466, + "ap_weighted": 0.736466 + }, + { + "accuracy": 0.519043, + "f1": 0.497611, + "f1_weighted": 0.543211, + "ap": 0.732585, + "ap_weighted": 0.732585 + }, + { + "accuracy": 0.363281, + "f1": 0.347142, + "f1_weighted": 0.302034, + "ap": 0.736557, + "ap_weighted": 0.736557 + }, + { + "accuracy": 0.380371, + "f1": 0.377825, + "f1_weighted": 0.360336, + "ap": 0.722287, + "ap_weighted": 0.722287 + } + ], + "main_score": 0.438021, + "hf_subset": "default", + "languages": [ + "ben-Beng" + ] + } + ] + }, + "evaluation_time": 2.920525312423706, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/BiorxivClusteringP2P.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/BiorxivClusteringP2P.json deleted file mode 100644 index 2ca2e383ab..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/BiorxivClusteringP2P.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "test": { - "evaluation_time": 806.58, - "v_measure": 0.2765822038298308, - "v_measure_std": 0.008913004746369184 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "BiorxivClusteringP2P", - "dataset_revision": "11d0121201d1f1f280e8cc8f3d98fb9c4d9f9c55" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/BiorxivClusteringS2S.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/BiorxivClusteringS2S.json deleted file mode 100644 index cfa9510156..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/BiorxivClusteringS2S.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "test": { - "evaluation_time": 64.32, - "v_measure": 0.23251707197080881, - "v_measure_std": 0.006139698393558528 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "BiorxivClusteringS2S", - "dataset_revision": "c0fab014e1bcb8d3a5e31b2088972a1e01547dc1" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/BulgarianStoreReviewSentimentClassfication.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/BulgarianStoreReviewSentimentClassfication.json new file mode 100644 index 0000000000..f547385e5c --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/BulgarianStoreReviewSentimentClassfication.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "701984d6c6efea0e14a1c7850ef70e464c5577c0", + "task_name": "BulgarianStoreReviewSentimentClassfication", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.277473, + "f1": 0.207181, + "f1_weighted": 0.324319, + "scores_per_experiment": [ + { + "accuracy": 0.456044, + "f1": 0.278467, + "f1_weighted": 0.480944 + }, + { + "accuracy": 0.208791, + "f1": 0.187378, + "f1_weighted": 0.272262 + }, + { + "accuracy": 0.307692, + "f1": 0.206443, + "f1_weighted": 0.362662 + }, + { + "accuracy": 0.230769, + "f1": 0.219347, + "f1_weighted": 0.295004 + }, + { + "accuracy": 0.214286, + "f1": 0.152225, + "f1_weighted": 0.259627 + }, + { + "accuracy": 0.296703, + "f1": 0.232393, + "f1_weighted": 0.341471 + }, + { + "accuracy": 0.291209, + "f1": 0.210507, + "f1_weighted": 0.358214 + }, + { + "accuracy": 0.313187, + "f1": 0.216548, + "f1_weighted": 0.364288 + }, + { + "accuracy": 0.10989, + "f1": 0.112981, + "f1_weighted": 0.116859 + }, + { + "accuracy": 0.346154, + "f1": 0.255519, + "f1_weighted": 0.39186 + } + ], + "main_score": 0.277473, + "hf_subset": "default", + "languages": [ + "bul-Cyrl" + ] + } + ] + }, + "evaluation_time": 3.5148472785949707, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CBD.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CBD.json new file mode 100644 index 0000000000..dad897574e --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CBD.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "36ddb419bcffe6a5374c3891957912892916f28d", + "task_name": "CBD", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.5112, + "f1": 0.44144, + "f1_weighted": 0.581687, + "ap": 0.150718, + "ap_weighted": 0.150718, + "scores_per_experiment": [ + { + "accuracy": 0.532, + "f1": 0.46408, + "f1_weighted": 0.603736, + "ap": 0.159316, + "ap_weighted": 0.159316 + }, + { + "accuracy": 0.546, + "f1": 0.425488, + "f1_weighted": 0.618097, + "ap": 0.125411, + "ap_weighted": 0.125411 + }, + { + "accuracy": 0.496, + "f1": 0.432716, + "f1_weighted": 0.57141, + "ap": 0.145035, + "ap_weighted": 0.145035 + }, + { + "accuracy": 0.566, + "f1": 0.487201, + "f1_weighted": 0.634346, + "ap": 0.166167, + "ap_weighted": 0.166167 + }, + { + "accuracy": 0.546, + "f1": 0.441833, + "f1_weighted": 0.618339, + "ap": 0.132867, + "ap_weighted": 0.132867 + }, + { + "accuracy": 0.49, + "f1": 0.430183, + "f1_weighted": 0.565325, + "ap": 0.145853, + "ap_weighted": 0.145853 + }, + { + "accuracy": 0.4, + "f1": 0.378975, + "f1_weighted": 0.462619, + "ap": 0.153632, + "ap_weighted": 0.153632 + }, + { + "accuracy": 0.437, + "f1": 0.406978, + "f1_weighted": 0.504649, + "ap": 0.158812, + "ap_weighted": 0.158812 + }, + { + "accuracy": 0.582, + "f1": 0.493503, + "f1_weighted": 0.648479, + "ap": 0.164007, + "ap_weighted": 0.164007 + }, + { + "accuracy": 0.517, + "f1": 0.453446, + "f1_weighted": 0.589873, + "ap": 0.156078, + "ap_weighted": 0.156078 + } + ], + "main_score": 0.5112, + "hf_subset": "default", + "languages": [ + "pol-Latn" + ] + } + ] + }, + "evaluation_time": 1.921954870223999, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackAndroidRetrieval.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackAndroidRetrieval.json deleted file mode 100644 index 346153c87f..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackAndroidRetrieval.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "mteb_version": "0.0.2", - "test": { - "evaluation_time": 83.52, - "map_at_1": 0.28129, - "map_at_10": 0.37444, - "map_at_100": 0.38811, - "map_at_1000": 0.38944, - "map_at_3": 0.34401, - "map_at_5": 0.36117, - "ndcg_at_1": 0.34621, - "ndcg_at_10": 0.43296, - "ndcg_at_100": 0.48803, - "ndcg_at_1000": 0.50993, - "ndcg_at_3": 0.38691, - "ndcg_at_5": 0.40893, - "precision_at_1": 0.34621, - "precision_at_10": 0.08155, - "precision_at_100": 0.01352, - "precision_at_1000": 0.00182, - "precision_at_3": 0.1836, - "precision_at_5": 0.13276, - "recall_at_1": 0.28129, - "recall_at_10": 0.54292, - "recall_at_100": 0.77943, - "recall_at_1000": 0.91964, - "recall_at_3": 0.40916, - "recall_at_5": 0.46899 - }, - "mteb_dataset_name": "CQADupstackAndroidRetrieval", - "dataset_revision": "2b9f5791698b5be7bc5e10535c8690f20043c3db" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackEnglishRetrieval.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackEnglishRetrieval.json deleted file mode 100644 index 84d5570510..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackEnglishRetrieval.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "mteb_version": "0.0.2", - "test": { - "evaluation_time": 125.72, - "map_at_1": 0.29078, - "map_at_10": 0.38406, - "map_at_100": 0.39552, - "map_at_1000": 0.39683, - "map_at_3": 0.35679, - "map_at_5": 0.37304, - "ndcg_at_1": 0.36561, - "ndcg_at_10": 0.43934, - "ndcg_at_100": 0.48332, - "ndcg_at_1000": 0.50518, - "ndcg_at_3": 0.40069, - "ndcg_at_5": 0.41992, - "precision_at_1": 0.36561, - "precision_at_10": 0.08153, - "precision_at_100": 0.01327, - "precision_at_1000": 0.00181, - "precision_at_3": 0.19193, - "precision_at_5": 0.13618, - "recall_at_1": 0.29078, - "recall_at_10": 0.53325, - "recall_at_100": 0.72218, - "recall_at_1000": 0.86298, - "recall_at_3": 0.41865, - "recall_at_5": 0.47235 - }, - "mteb_dataset_name": "CQADupstackEnglishRetrieval", - "dataset_revision": "2b9f5791698b5be7bc5e10535c8690f20043c3db" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackGamingRetrieval.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackGamingRetrieval.json deleted file mode 100644 index 31e4969636..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackGamingRetrieval.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "mteb_version": "0.0.2", - "test": { - "evaluation_time": 133.23, - "map_at_1": 0.35891, - "map_at_10": 0.46846, - "map_at_100": 0.47957, - "map_at_1000": 0.48023, - "map_at_3": 0.43596, - "map_at_5": 0.45524, - "ndcg_at_1": 0.40878, - "ndcg_at_10": 0.52465, - "ndcg_at_100": 0.57037, - "ndcg_at_1000": 0.58468, - "ndcg_at_3": 0.46831, - "ndcg_at_5": 0.49779, - "precision_at_1": 0.40878, - "precision_at_10": 0.08408, - "precision_at_100": 0.01165, - "precision_at_1000": 0.00134, - "precision_at_3": 0.20648, - "precision_at_5": 0.14445, - "recall_at_1": 0.35891, - "recall_at_10": 0.65795, - "recall_at_100": 0.85612, - "recall_at_1000": 0.95881, - "recall_at_3": 0.50922, - "recall_at_5": 0.58031 - }, - "mteb_dataset_name": "CQADupstackGamingRetrieval", - "dataset_revision": "2b9f5791698b5be7bc5e10535c8690f20043c3db" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackGisRetrieval.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackGisRetrieval.json deleted file mode 100644 index cd7590d9f1..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackGisRetrieval.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "mteb_version": "0.0.2", - "test": { - "evaluation_time": 218.35, - "map_at_1": 0.19385, - "map_at_10": 0.26492, - "map_at_100": 0.27374, - "map_at_1000": 0.27469, - "map_at_3": 0.24172, - "map_at_5": 0.25483, - "ndcg_at_1": 0.20791, - "ndcg_at_10": 0.30678, - "ndcg_at_100": 0.3534, - "ndcg_at_1000": 0.37985, - "ndcg_at_3": 0.26131, - "ndcg_at_5": 0.28427, - "precision_at_1": 0.20791, - "precision_at_10": 0.0478, - "precision_at_100": 0.00747, - "precision_at_1000": 0.00101, - "precision_at_3": 0.11073, - "precision_at_5": 0.0791, - "recall_at_1": 0.19385, - "recall_at_10": 0.4223, - "recall_at_100": 0.64026, - "recall_at_1000": 0.84359, - "recall_at_3": 0.29978, - "recall_at_5": 0.35581 - }, - "mteb_dataset_name": "CQADupstackGisRetrieval", - "dataset_revision": "2b9f5791698b5be7bc5e10535c8690f20043c3db" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackMathematicaRetrieval.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackMathematicaRetrieval.json deleted file mode 100644 index 82adae8ae3..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackMathematicaRetrieval.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "mteb_version": "0.0.2", - "test": { - "evaluation_time": 137.9, - "map_at_1": 0.13242, - "map_at_10": 0.19493, - "map_at_100": 0.2054, - "map_at_1000": 0.20672, - "map_at_3": 0.17408, - "map_at_5": 0.18716, - "ndcg_at_1": 0.16542, - "ndcg_at_10": 0.23683, - "ndcg_at_100": 0.2908, - "ndcg_at_1000": 0.3241, - "ndcg_at_3": 0.19719, - "ndcg_at_5": 0.21943, - "precision_at_1": 0.16542, - "precision_at_10": 0.04316, - "precision_at_100": 0.00811, - "precision_at_1000": 0.00125, - "precision_at_3": 0.09328, - "precision_at_5": 0.0709, - "recall_at_1": 0.13242, - "recall_at_10": 0.32847, - "recall_at_100": 0.57018, - "recall_at_1000": 0.80793, - "recall_at_3": 0.22191, - "recall_at_5": 0.27666 - }, - "mteb_dataset_name": "CQADupstackMathematicaRetrieval", - "dataset_revision": "2b9f5791698b5be7bc5e10535c8690f20043c3db" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackPhysicsRetrieval.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackPhysicsRetrieval.json deleted file mode 100644 index d35720f942..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackPhysicsRetrieval.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "mteb_version": "0.0.2", - "test": { - "evaluation_time": 249.99, - "map_at_1": 0.23849, - "map_at_10": 0.31553, - "map_at_100": 0.32759, - "map_at_1000": 0.32887, - "map_at_3": 0.28964, - "map_at_5": 0.30359, - "ndcg_at_1": 0.29163, - "ndcg_at_10": 0.36785, - "ndcg_at_100": 0.42203, - "ndcg_at_1000": 0.44811, - "ndcg_at_3": 0.32226, - "ndcg_at_5": 0.3431, - "precision_at_1": 0.29163, - "precision_at_10": 0.06554, - "precision_at_100": 0.01091, - "precision_at_1000": 0.00149, - "precision_at_3": 0.14918, - "precision_at_5": 0.10683, - "recall_at_1": 0.23849, - "recall_at_10": 0.47457, - "recall_at_100": 0.70546, - "recall_at_1000": 0.87941, - "recall_at_3": 0.34552, - "recall_at_5": 0.39947 - }, - "mteb_dataset_name": "CQADupstackPhysicsRetrieval", - "dataset_revision": "2b9f5791698b5be7bc5e10535c8690f20043c3db" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackProgrammersRetrieval.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackProgrammersRetrieval.json deleted file mode 100644 index 3a68c3793a..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackProgrammersRetrieval.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "mteb_version": "0.0.2", - "test": { - "evaluation_time": 195.83, - "map_at_1": 0.23936, - "map_at_10": 0.32323, - "map_at_100": 0.33568, - "map_at_1000": 0.3369, - "map_at_3": 0.29356, - "map_at_5": 0.31207, - "ndcg_at_1": 0.29452, - "ndcg_at_10": 0.37578, - "ndcg_at_100": 0.43044, - "ndcg_at_1000": 0.45732, - "ndcg_at_3": 0.32721, - "ndcg_at_5": 0.35377, - "precision_at_1": 0.29452, - "precision_at_10": 0.06861, - "precision_at_100": 0.01119, - "precision_at_1000": 0.00154, - "precision_at_3": 0.15487, - "precision_at_5": 0.11438, - "recall_at_1": 0.23936, - "recall_at_10": 0.48276, - "recall_at_100": 0.71781, - "recall_at_1000": 0.90343, - "recall_at_3": 0.349, - "recall_at_5": 0.41739 - }, - "mteb_dataset_name": "CQADupstackProgrammersRetrieval", - "dataset_revision": "2b9f5791698b5be7bc5e10535c8690f20043c3db" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackRetrieval.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackRetrieval.json deleted file mode 100644 index b1c4f086de..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackRetrieval.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "dataset_revision": "1", - "task_name": "CQADupstackRetrieval", - "mteb_version": "0.0.2", - "scores": { - "test": [ - { - "ndcg_at_10": 0.345487, - "main_score": 0.345487, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 2686.69, - "kg_co2_emissions": NaN -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackStatsRetrieval.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackStatsRetrieval.json deleted file mode 100644 index fdc6e4012e..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackStatsRetrieval.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "mteb_version": "0.0.2", - "test": { - "evaluation_time": 326.9, - "map_at_1": 0.19572, - "map_at_10": 0.25178, - "map_at_100": 0.25963, - "map_at_1000": 0.26058, - "map_at_3": 0.23351, - "map_at_5": 0.24406, - "ndcg_at_1": 0.21626, - "ndcg_at_10": 0.28496, - "ndcg_at_100": 0.32557, - "ndcg_at_1000": 0.35173, - "ndcg_at_3": 0.25077, - "ndcg_at_5": 0.26703, - "precision_at_1": 0.21626, - "precision_at_10": 0.04448, - "precision_at_100": 0.00695, - "precision_at_1000": 0.00098, - "precision_at_3": 0.10685, - "precision_at_5": 0.07485, - "recall_at_1": 0.19572, - "recall_at_10": 0.36872, - "recall_at_100": 0.55446, - "recall_at_1000": 0.75098, - "recall_at_3": 0.27451, - "recall_at_5": 0.31514 - }, - "mteb_dataset_name": "CQADupstackStatsRetrieval", - "dataset_revision": "2b9f5791698b5be7bc5e10535c8690f20043c3db" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackTexRetrieval.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackTexRetrieval.json deleted file mode 100644 index 130f342dc8..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackTexRetrieval.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "mteb_version": "0.0.2", - "test": { - "evaluation_time": 540.74, - "map_at_1": 0.13565, - "map_at_10": 0.19416, - "map_at_100": 0.20265, - "map_at_1000": 0.20381, - "map_at_3": 0.17426, - "map_at_5": 0.1849, - "ndcg_at_1": 0.16586, - "ndcg_at_10": 0.23353, - "ndcg_at_100": 0.27623, - "ndcg_at_1000": 0.30717, - "ndcg_at_3": 0.19703, - "ndcg_at_5": 0.21301, - "precision_at_1": 0.16586, - "precision_at_10": 0.04326, - "precision_at_100": 0.00736, - "precision_at_1000": 0.00114, - "precision_at_3": 0.09348, - "precision_at_5": 0.06841, - "recall_at_1": 0.13565, - "recall_at_10": 0.32033, - "recall_at_100": 0.5171, - "recall_at_1000": 0.74403, - "recall_at_3": 0.21754, - "recall_at_5": 0.25865 - }, - "mteb_dataset_name": "CQADupstackTexRetrieval", - "dataset_revision": "2b9f5791698b5be7bc5e10535c8690f20043c3db" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackUnixRetrieval.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackUnixRetrieval.json deleted file mode 100644 index efb9fd7355..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackUnixRetrieval.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "mteb_version": "0.0.2", - "test": { - "evaluation_time": 279.97, - "map_at_1": 0.22957, - "map_at_10": 0.28989, - "map_at_100": 0.30073, - "map_at_1000": 0.30187, - "map_at_3": 0.27019, - "map_at_5": 0.27988, - "ndcg_at_1": 0.26119, - "ndcg_at_10": 0.32965, - "ndcg_at_100": 0.38317, - "ndcg_at_1000": 0.41188, - "ndcg_at_3": 0.29164, - "ndcg_at_5": 0.30662, - "precision_at_1": 0.26119, - "precision_at_10": 0.05233, - "precision_at_100": 0.00886, - "precision_at_1000": 0.00126, - "precision_at_3": 0.12531, - "precision_at_5": 0.08657, - "recall_at_1": 0.22957, - "recall_at_10": 0.41921, - "recall_at_100": 0.658, - "recall_at_1000": 0.8626, - "recall_at_3": 0.31377, - "recall_at_5": 0.35242 - }, - "mteb_dataset_name": "CQADupstackUnixRetrieval", - "dataset_revision": "2b9f5791698b5be7bc5e10535c8690f20043c3db" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackWebmastersRetrieval.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackWebmastersRetrieval.json deleted file mode 100644 index e1be2cd186..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackWebmastersRetrieval.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "mteb_version": "0.0.2", - "test": { - "evaluation_time": 85.69, - "map_at_1": 0.21651, - "map_at_10": 0.28958, - "map_at_100": 0.30292, - "map_at_1000": 0.305, - "map_at_3": 0.26185, - "map_at_5": 0.27675, - "ndcg_at_1": 0.26482, - "ndcg_at_10": 0.34206, - "ndcg_at_100": 0.39652, - "ndcg_at_1000": 0.42778, - "ndcg_at_3": 0.29862, - "ndcg_at_5": 0.31871, - "precision_at_1": 0.26482, - "precision_at_10": 0.06917, - "precision_at_100": 0.0135, - "precision_at_1000": 0.00224, - "precision_at_3": 0.14097, - "precision_at_5": 0.10356, - "recall_at_1": 0.21651, - "recall_at_10": 0.43578, - "recall_at_100": 0.69747, - "recall_at_1000": 0.90562, - "recall_at_3": 0.31537, - "recall_at_5": 0.3694 - }, - "mteb_dataset_name": "CQADupstackWebmastersRetrieval", - "dataset_revision": "2b9f5791698b5be7bc5e10535c8690f20043c3db" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackWordpressRetrieval.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackWordpressRetrieval.json deleted file mode 100644 index ef10464e3f..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/CQADupstackWordpressRetrieval.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "mteb_version": "0.0.2", - "test": { - "evaluation_time": 308.85, - "map_at_1": 0.17443, - "map_at_10": 0.23308, - "map_at_100": 0.24215, - "map_at_1000": 0.24315, - "map_at_3": 0.21368, - "map_at_5": 0.22287, - "ndcg_at_1": 0.19039, - "ndcg_at_10": 0.27146, - "ndcg_at_100": 0.31967, - "ndcg_at_1000": 0.34729, - "ndcg_at_3": 0.2323, - "ndcg_at_5": 0.24737, - "precision_at_1": 0.19039, - "precision_at_10": 0.04344, - "precision_at_100": 0.00738, - "precision_at_1000": 0.00107, - "precision_at_3": 0.10043, - "precision_at_5": 0.06913, - "recall_at_1": 0.17443, - "recall_at_10": 0.37363, - "recall_at_100": 0.5995, - "recall_at_1000": 0.81126, - "recall_at_3": 0.26275, - "recall_at_5": 0.30102 - }, - "mteb_dataset_name": "CQADupstackWordpressRetrieval", - "dataset_revision": "2b9f5791698b5be7bc5e10535c8690f20043c3db" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CSFDCZMovieReviewSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CSFDCZMovieReviewSentimentClassification.json new file mode 100644 index 0000000000..7a6d4e7a87 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CSFDCZMovieReviewSentimentClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "dd2ede6faaea338ef6b1e2966f06808656975a23", + "task_name": "CSFDCZMovieReviewSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.212988, + "f1": 0.207271, + "f1_weighted": 0.207609, + "scores_per_experiment": [ + { + "accuracy": 0.199707, + "f1": 0.196021, + "f1_weighted": 0.196877 + }, + { + "accuracy": 0.211426, + "f1": 0.204862, + "f1_weighted": 0.205237 + }, + { + "accuracy": 0.216797, + "f1": 0.214166, + "f1_weighted": 0.214715 + }, + { + "accuracy": 0.202637, + "f1": 0.194354, + "f1_weighted": 0.195134 + }, + { + "accuracy": 0.229004, + "f1": 0.221685, + "f1_weighted": 0.221879 + }, + { + "accuracy": 0.191406, + "f1": 0.187754, + "f1_weighted": 0.188432 + }, + { + "accuracy": 0.213867, + "f1": 0.208791, + "f1_weighted": 0.209008 + }, + { + "accuracy": 0.220215, + "f1": 0.219649, + "f1_weighted": 0.219298 + }, + { + "accuracy": 0.216797, + "f1": 0.205839, + "f1_weighted": 0.207162 + }, + { + "accuracy": 0.228027, + "f1": 0.219589, + "f1_weighted": 0.218345 + } + ], + "main_score": 0.212988, + "hf_subset": "default", + "languages": [ + "ces-Latn" + ] + } + ] + }, + "evaluation_time": 14.491014003753662, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CSFDSKMovieReviewSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CSFDSKMovieReviewSentimentClassification.json new file mode 100644 index 0000000000..a77005c1f3 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CSFDSKMovieReviewSentimentClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "23a20c659d868740ef9c54854de631fe19cd5c17", + "task_name": "CSFDSKMovieReviewSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.211475, + "f1": 0.205938, + "f1_weighted": 0.206536, + "scores_per_experiment": [ + { + "accuracy": 0.196777, + "f1": 0.189925, + "f1_weighted": 0.191291 + }, + { + "accuracy": 0.195312, + "f1": 0.194383, + "f1_weighted": 0.194857 + }, + { + "accuracy": 0.225586, + "f1": 0.220178, + "f1_weighted": 0.220956 + }, + { + "accuracy": 0.216309, + "f1": 0.213251, + "f1_weighted": 0.212535 + }, + { + "accuracy": 0.20166, + "f1": 0.197868, + "f1_weighted": 0.199007 + }, + { + "accuracy": 0.214844, + "f1": 0.196407, + "f1_weighted": 0.197456 + }, + { + "accuracy": 0.196777, + "f1": 0.195254, + "f1_weighted": 0.196141 + }, + { + "accuracy": 0.225586, + "f1": 0.216688, + "f1_weighted": 0.217094 + }, + { + "accuracy": 0.21875, + "f1": 0.214161, + "f1_weighted": 0.214985 + }, + { + "accuracy": 0.223145, + "f1": 0.221265, + "f1_weighted": 0.221039 + } + ], + "main_score": 0.211475, + "hf_subset": "default", + "languages": [ + "slk-Latn" + ] + } + ] + }, + "evaluation_time": 13.880476236343384, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADAffiliateLicenseLicenseeLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADAffiliateLicenseLicenseeLegalBenchClassification.json new file mode 100644 index 0000000000..475b186675 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADAffiliateLicenseLicenseeLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADAffiliateLicenseLicenseeLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.858586, + "f1": 0.856477, + "f1_weighted": 0.856477, + "ap": 0.782787, + "ap_weighted": 0.782787, + "scores_per_experiment": [ + { + "accuracy": 0.858586, + "f1": 0.856477, + "f1_weighted": 0.856477, + "ap": 0.782787, + "ap_weighted": 0.782787 + }, + { + "accuracy": 0.858586, + "f1": 0.856477, + "f1_weighted": 0.856477, + "ap": 0.782787, + "ap_weighted": 0.782787 + }, + { + "accuracy": 0.858586, + "f1": 0.856477, + "f1_weighted": 0.856477, + "ap": 0.782787, + "ap_weighted": 0.782787 + }, + { + "accuracy": 0.858586, + "f1": 0.856477, + "f1_weighted": 0.856477, + "ap": 0.782787, + "ap_weighted": 0.782787 + }, + { + "accuracy": 0.858586, + "f1": 0.856477, + "f1_weighted": 0.856477, + "ap": 0.782787, + "ap_weighted": 0.782787 + }, + { + "accuracy": 0.858586, + "f1": 0.856477, + "f1_weighted": 0.856477, + "ap": 0.782787, + "ap_weighted": 0.782787 + }, + { + "accuracy": 0.858586, + "f1": 0.856477, + "f1_weighted": 0.856477, + "ap": 0.782787, + "ap_weighted": 0.782787 + }, + { + "accuracy": 0.858586, + "f1": 0.856477, + "f1_weighted": 0.856477, + "ap": 0.782787, + "ap_weighted": 0.782787 + }, + { + "accuracy": 0.858586, + "f1": 0.856477, + "f1_weighted": 0.856477, + "ap": 0.782787, + "ap_weighted": 0.782787 + }, + { + "accuracy": 0.858586, + "f1": 0.856477, + "f1_weighted": 0.856477, + "ap": 0.782787, + "ap_weighted": 0.782787 + } + ], + "main_score": 0.858586, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.4315497875213623, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADAffiliateLicenseLicensorLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADAffiliateLicenseLicensorLegalBenchClassification.json new file mode 100644 index 0000000000..1a4d9d95e1 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADAffiliateLicenseLicensorLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADAffiliateLicenseLicensorLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.806818, + "f1": 0.804776, + "f1_weighted": 0.804776, + "ap": 0.771753, + "ap_weighted": 0.771753, + "scores_per_experiment": [ + { + "accuracy": 0.806818, + "f1": 0.804776, + "f1_weighted": 0.804776, + "ap": 0.771753, + "ap_weighted": 0.771753 + }, + { + "accuracy": 0.806818, + "f1": 0.804776, + "f1_weighted": 0.804776, + "ap": 0.771753, + "ap_weighted": 0.771753 + }, + { + "accuracy": 0.806818, + "f1": 0.804776, + "f1_weighted": 0.804776, + "ap": 0.771753, + "ap_weighted": 0.771753 + }, + { + "accuracy": 0.806818, + "f1": 0.804776, + "f1_weighted": 0.804776, + "ap": 0.771753, + "ap_weighted": 0.771753 + }, + { + "accuracy": 0.806818, + "f1": 0.804776, + "f1_weighted": 0.804776, + "ap": 0.771753, + "ap_weighted": 0.771753 + }, + { + "accuracy": 0.806818, + "f1": 0.804776, + "f1_weighted": 0.804776, + "ap": 0.771753, + "ap_weighted": 0.771753 + }, + { + "accuracy": 0.806818, + "f1": 0.804776, + "f1_weighted": 0.804776, + "ap": 0.771753, + "ap_weighted": 0.771753 + }, + { + "accuracy": 0.806818, + "f1": 0.804776, + "f1_weighted": 0.804776, + "ap": 0.771753, + "ap_weighted": 0.771753 + }, + { + "accuracy": 0.806818, + "f1": 0.804776, + "f1_weighted": 0.804776, + "ap": 0.771753, + "ap_weighted": 0.771753 + }, + { + "accuracy": 0.806818, + "f1": 0.804776, + "f1_weighted": 0.804776, + "ap": 0.771753, + "ap_weighted": 0.771753 + } + ], + "main_score": 0.806818, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.082200050354004, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADAntiAssignmentLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADAntiAssignmentLegalBenchClassification.json new file mode 100644 index 0000000000..3f85978907 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADAntiAssignmentLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADAntiAssignmentLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.872014, + "f1": 0.87196, + "f1_weighted": 0.87196, + "ap": 0.818956, + "ap_weighted": 0.818956, + "scores_per_experiment": [ + { + "accuracy": 0.872014, + "f1": 0.87196, + "f1_weighted": 0.87196, + "ap": 0.818956, + "ap_weighted": 0.818956 + }, + { + "accuracy": 0.872014, + "f1": 0.87196, + "f1_weighted": 0.87196, + "ap": 0.818956, + "ap_weighted": 0.818956 + }, + { + "accuracy": 0.872014, + "f1": 0.87196, + "f1_weighted": 0.87196, + "ap": 0.818956, + "ap_weighted": 0.818956 + }, + { + "accuracy": 0.872014, + "f1": 0.87196, + "f1_weighted": 0.87196, + "ap": 0.818956, + "ap_weighted": 0.818956 + }, + { + "accuracy": 0.872014, + "f1": 0.87196, + "f1_weighted": 0.87196, + "ap": 0.818956, + "ap_weighted": 0.818956 + }, + { + "accuracy": 0.872014, + "f1": 0.87196, + "f1_weighted": 0.87196, + "ap": 0.818956, + "ap_weighted": 0.818956 + }, + { + "accuracy": 0.872014, + "f1": 0.87196, + "f1_weighted": 0.87196, + "ap": 0.818956, + "ap_weighted": 0.818956 + }, + { + "accuracy": 0.872014, + "f1": 0.87196, + "f1_weighted": 0.87196, + "ap": 0.818956, + "ap_weighted": 0.818956 + }, + { + "accuracy": 0.872014, + "f1": 0.87196, + "f1_weighted": 0.87196, + "ap": 0.818956, + "ap_weighted": 0.818956 + }, + { + "accuracy": 0.872014, + "f1": 0.87196, + "f1_weighted": 0.87196, + "ap": 0.818956, + "ap_weighted": 0.818956 + } + ], + "main_score": 0.872014, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.8698766231536865, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADAuditRightsLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADAuditRightsLegalBenchClassification.json new file mode 100644 index 0000000000..88c899b85e --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADAuditRightsLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADAuditRightsLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.839638, + "f1": 0.839633, + "f1_weighted": 0.839633, + "ap": 0.78386, + "ap_weighted": 0.78386, + "scores_per_experiment": [ + { + "accuracy": 0.839638, + "f1": 0.839633, + "f1_weighted": 0.839633, + "ap": 0.78386, + "ap_weighted": 0.78386 + }, + { + "accuracy": 0.839638, + "f1": 0.839633, + "f1_weighted": 0.839633, + "ap": 0.78386, + "ap_weighted": 0.78386 + }, + { + "accuracy": 0.839638, + "f1": 0.839633, + "f1_weighted": 0.839633, + "ap": 0.78386, + "ap_weighted": 0.78386 + }, + { + "accuracy": 0.839638, + "f1": 0.839633, + "f1_weighted": 0.839633, + "ap": 0.78386, + "ap_weighted": 0.78386 + }, + { + "accuracy": 0.839638, + "f1": 0.839633, + "f1_weighted": 0.839633, + "ap": 0.78386, + "ap_weighted": 0.78386 + }, + { + "accuracy": 0.839638, + "f1": 0.839633, + "f1_weighted": 0.839633, + "ap": 0.78386, + "ap_weighted": 0.78386 + }, + { + "accuracy": 0.839638, + "f1": 0.839633, + "f1_weighted": 0.839633, + "ap": 0.78386, + "ap_weighted": 0.78386 + }, + { + "accuracy": 0.839638, + "f1": 0.839633, + "f1_weighted": 0.839633, + "ap": 0.78386, + "ap_weighted": 0.78386 + }, + { + "accuracy": 0.839638, + "f1": 0.839633, + "f1_weighted": 0.839633, + "ap": 0.78386, + "ap_weighted": 0.78386 + }, + { + "accuracy": 0.839638, + "f1": 0.839633, + "f1_weighted": 0.839633, + "ap": 0.78386, + "ap_weighted": 0.78386 + } + ], + "main_score": 0.839638, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.155146360397339, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADCapOnLiabilityLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADCapOnLiabilityLegalBenchClassification.json new file mode 100644 index 0000000000..ae029b7ec1 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADCapOnLiabilityLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADCapOnLiabilityLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.808989, + "f1": 0.808074, + "f1_weighted": 0.808074, + "ap": 0.738388, + "ap_weighted": 0.738388, + "scores_per_experiment": [ + { + "accuracy": 0.808989, + "f1": 0.808074, + "f1_weighted": 0.808074, + "ap": 0.738388, + "ap_weighted": 0.738388 + }, + { + "accuracy": 0.808989, + "f1": 0.808074, + "f1_weighted": 0.808074, + "ap": 0.738388, + "ap_weighted": 0.738388 + }, + { + "accuracy": 0.808989, + "f1": 0.808074, + "f1_weighted": 0.808074, + "ap": 0.738388, + "ap_weighted": 0.738388 + }, + { + "accuracy": 0.808989, + "f1": 0.808074, + "f1_weighted": 0.808074, + "ap": 0.738388, + "ap_weighted": 0.738388 + }, + { + "accuracy": 0.808989, + "f1": 0.808074, + "f1_weighted": 0.808074, + "ap": 0.738388, + "ap_weighted": 0.738388 + }, + { + "accuracy": 0.808989, + "f1": 0.808074, + "f1_weighted": 0.808074, + "ap": 0.738388, + "ap_weighted": 0.738388 + }, + { + "accuracy": 0.808989, + "f1": 0.808074, + "f1_weighted": 0.808074, + "ap": 0.738388, + "ap_weighted": 0.738388 + }, + { + "accuracy": 0.808989, + "f1": 0.808074, + "f1_weighted": 0.808074, + "ap": 0.738388, + "ap_weighted": 0.738388 + }, + { + "accuracy": 0.808989, + "f1": 0.808074, + "f1_weighted": 0.808074, + "ap": 0.738388, + "ap_weighted": 0.738388 + }, + { + "accuracy": 0.808989, + "f1": 0.808074, + "f1_weighted": 0.808074, + "ap": 0.738388, + "ap_weighted": 0.738388 + } + ], + "main_score": 0.808989, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.6639349460601807, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADChangeOfControlLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADChangeOfControlLegalBenchClassification.json new file mode 100644 index 0000000000..b75ddc56bd --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADChangeOfControlLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADChangeOfControlLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.713942, + "f1": 0.711136, + "f1_weighted": 0.711136, + "ap": 0.645206, + "ap_weighted": 0.645206, + "scores_per_experiment": [ + { + "accuracy": 0.713942, + "f1": 0.711136, + "f1_weighted": 0.711136, + "ap": 0.645206, + "ap_weighted": 0.645206 + }, + { + "accuracy": 0.713942, + "f1": 0.711136, + "f1_weighted": 0.711136, + "ap": 0.645206, + "ap_weighted": 0.645206 + }, + { + "accuracy": 0.713942, + "f1": 0.711136, + "f1_weighted": 0.711136, + "ap": 0.645206, + "ap_weighted": 0.645206 + }, + { + "accuracy": 0.713942, + "f1": 0.711136, + "f1_weighted": 0.711136, + "ap": 0.645206, + "ap_weighted": 0.645206 + }, + { + "accuracy": 0.713942, + "f1": 0.711136, + "f1_weighted": 0.711136, + "ap": 0.645206, + "ap_weighted": 0.645206 + }, + { + "accuracy": 0.713942, + "f1": 0.711136, + "f1_weighted": 0.711136, + "ap": 0.645206, + "ap_weighted": 0.645206 + }, + { + "accuracy": 0.713942, + "f1": 0.711136, + "f1_weighted": 0.711136, + "ap": 0.645206, + "ap_weighted": 0.645206 + }, + { + "accuracy": 0.713942, + "f1": 0.711136, + "f1_weighted": 0.711136, + "ap": 0.645206, + "ap_weighted": 0.645206 + }, + { + "accuracy": 0.713942, + "f1": 0.711136, + "f1_weighted": 0.711136, + "ap": 0.645206, + "ap_weighted": 0.645206 + }, + { + "accuracy": 0.713942, + "f1": 0.711136, + "f1_weighted": 0.711136, + "ap": 0.645206, + "ap_weighted": 0.645206 + } + ], + "main_score": 0.713942, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.5710744857788086, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADCompetitiveRestrictionExceptionLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADCompetitiveRestrictionExceptionLegalBenchClassification.json new file mode 100644 index 0000000000..afdad6c58a --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADCompetitiveRestrictionExceptionLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADCompetitiveRestrictionExceptionLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.636364, + "f1": 0.636243, + "f1_weighted": 0.636243, + "ap": 0.587479, + "ap_weighted": 0.587479, + "scores_per_experiment": [ + { + "accuracy": 0.636364, + "f1": 0.636243, + "f1_weighted": 0.636243, + "ap": 0.587479, + "ap_weighted": 0.587479 + }, + { + "accuracy": 0.636364, + "f1": 0.636243, + "f1_weighted": 0.636243, + "ap": 0.587479, + "ap_weighted": 0.587479 + }, + { + "accuracy": 0.636364, + "f1": 0.636243, + "f1_weighted": 0.636243, + "ap": 0.587479, + "ap_weighted": 0.587479 + }, + { + "accuracy": 0.636364, + "f1": 0.636243, + "f1_weighted": 0.636243, + "ap": 0.587479, + "ap_weighted": 0.587479 + }, + { + "accuracy": 0.636364, + "f1": 0.636243, + "f1_weighted": 0.636243, + "ap": 0.587479, + "ap_weighted": 0.587479 + }, + { + "accuracy": 0.636364, + "f1": 0.636243, + "f1_weighted": 0.636243, + "ap": 0.587479, + "ap_weighted": 0.587479 + }, + { + "accuracy": 0.636364, + "f1": 0.636243, + "f1_weighted": 0.636243, + "ap": 0.587479, + "ap_weighted": 0.587479 + }, + { + "accuracy": 0.636364, + "f1": 0.636243, + "f1_weighted": 0.636243, + "ap": 0.587479, + "ap_weighted": 0.587479 + }, + { + "accuracy": 0.636364, + "f1": 0.636243, + "f1_weighted": 0.636243, + "ap": 0.587479, + "ap_weighted": 0.587479 + }, + { + "accuracy": 0.636364, + "f1": 0.636243, + "f1_weighted": 0.636243, + "ap": 0.587479, + "ap_weighted": 0.587479 + } + ], + "main_score": 0.636364, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.2201423645019531, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADCovenantNotToSueLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADCovenantNotToSueLegalBenchClassification.json new file mode 100644 index 0000000000..037c9c36cf --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADCovenantNotToSueLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADCovenantNotToSueLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.659091, + "f1": 0.624891, + "f1_weighted": 0.624891, + "ap": 0.595326, + "ap_weighted": 0.595326, + "scores_per_experiment": [ + { + "accuracy": 0.659091, + "f1": 0.624891, + "f1_weighted": 0.624891, + "ap": 0.595326, + "ap_weighted": 0.595326 + }, + { + "accuracy": 0.659091, + "f1": 0.624891, + "f1_weighted": 0.624891, + "ap": 0.595326, + "ap_weighted": 0.595326 + }, + { + "accuracy": 0.659091, + "f1": 0.624891, + "f1_weighted": 0.624891, + "ap": 0.595326, + "ap_weighted": 0.595326 + }, + { + "accuracy": 0.659091, + "f1": 0.624891, + "f1_weighted": 0.624891, + "ap": 0.595326, + "ap_weighted": 0.595326 + }, + { + "accuracy": 0.659091, + "f1": 0.624891, + "f1_weighted": 0.624891, + "ap": 0.595326, + "ap_weighted": 0.595326 + }, + { + "accuracy": 0.659091, + "f1": 0.624891, + "f1_weighted": 0.624891, + "ap": 0.595326, + "ap_weighted": 0.595326 + }, + { + "accuracy": 0.659091, + "f1": 0.624891, + "f1_weighted": 0.624891, + "ap": 0.595326, + "ap_weighted": 0.595326 + }, + { + "accuracy": 0.659091, + "f1": 0.624891, + "f1_weighted": 0.624891, + "ap": 0.595326, + "ap_weighted": 0.595326 + }, + { + "accuracy": 0.659091, + "f1": 0.624891, + "f1_weighted": 0.624891, + "ap": 0.595326, + "ap_weighted": 0.595326 + }, + { + "accuracy": 0.659091, + "f1": 0.624891, + "f1_weighted": 0.624891, + "ap": 0.595326, + "ap_weighted": 0.595326 + } + ], + "main_score": 0.659091, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.2964534759521484, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADEffectiveDateLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADEffectiveDateLegalBenchClassification.json new file mode 100644 index 0000000000..f4cc23ff74 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADEffectiveDateLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADEffectiveDateLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.665254, + "f1": 0.663896, + "f1_weighted": 0.663896, + "ap": 0.613913, + "ap_weighted": 0.613913, + "scores_per_experiment": [ + { + "accuracy": 0.665254, + "f1": 0.663896, + "f1_weighted": 0.663896, + "ap": 0.613913, + "ap_weighted": 0.613913 + }, + { + "accuracy": 0.665254, + "f1": 0.663896, + "f1_weighted": 0.663896, + "ap": 0.613913, + "ap_weighted": 0.613913 + }, + { + "accuracy": 0.665254, + "f1": 0.663896, + "f1_weighted": 0.663896, + "ap": 0.613913, + "ap_weighted": 0.613913 + }, + { + "accuracy": 0.665254, + "f1": 0.663896, + "f1_weighted": 0.663896, + "ap": 0.613913, + "ap_weighted": 0.613913 + }, + { + "accuracy": 0.665254, + "f1": 0.663896, + "f1_weighted": 0.663896, + "ap": 0.613913, + "ap_weighted": 0.613913 + }, + { + "accuracy": 0.665254, + "f1": 0.663896, + "f1_weighted": 0.663896, + "ap": 0.613913, + "ap_weighted": 0.613913 + }, + { + "accuracy": 0.665254, + "f1": 0.663896, + "f1_weighted": 0.663896, + "ap": 0.613913, + "ap_weighted": 0.613913 + }, + { + "accuracy": 0.665254, + "f1": 0.663896, + "f1_weighted": 0.663896, + "ap": 0.613913, + "ap_weighted": 0.613913 + }, + { + "accuracy": 0.665254, + "f1": 0.663896, + "f1_weighted": 0.663896, + "ap": 0.613913, + "ap_weighted": 0.613913 + }, + { + "accuracy": 0.665254, + "f1": 0.663896, + "f1_weighted": 0.663896, + "ap": 0.613913, + "ap_weighted": 0.613913 + } + ], + "main_score": 0.665254, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.040006399154663, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADExclusivityLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADExclusivityLegalBenchClassification.json new file mode 100644 index 0000000000..39c62a5723 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADExclusivityLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADExclusivityLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.69685, + "f1": 0.687325, + "f1_weighted": 0.687325, + "ap": 0.627149, + "ap_weighted": 0.627149, + "scores_per_experiment": [ + { + "accuracy": 0.69685, + "f1": 0.687325, + "f1_weighted": 0.687325, + "ap": 0.627149, + "ap_weighted": 0.627149 + }, + { + "accuracy": 0.69685, + "f1": 0.687325, + "f1_weighted": 0.687325, + "ap": 0.627149, + "ap_weighted": 0.627149 + }, + { + "accuracy": 0.69685, + "f1": 0.687325, + "f1_weighted": 0.687325, + "ap": 0.627149, + "ap_weighted": 0.627149 + }, + { + "accuracy": 0.69685, + "f1": 0.687325, + "f1_weighted": 0.687325, + "ap": 0.627149, + "ap_weighted": 0.627149 + }, + { + "accuracy": 0.69685, + "f1": 0.687325, + "f1_weighted": 0.687325, + "ap": 0.627149, + "ap_weighted": 0.627149 + }, + { + "accuracy": 0.69685, + "f1": 0.687325, + "f1_weighted": 0.687325, + "ap": 0.627149, + "ap_weighted": 0.627149 + }, + { + "accuracy": 0.69685, + "f1": 0.687325, + "f1_weighted": 0.687325, + "ap": 0.627149, + "ap_weighted": 0.627149 + }, + { + "accuracy": 0.69685, + "f1": 0.687325, + "f1_weighted": 0.687325, + "ap": 0.627149, + "ap_weighted": 0.627149 + }, + { + "accuracy": 0.69685, + "f1": 0.687325, + "f1_weighted": 0.687325, + "ap": 0.627149, + "ap_weighted": 0.627149 + }, + { + "accuracy": 0.69685, + "f1": 0.687325, + "f1_weighted": 0.687325, + "ap": 0.627149, + "ap_weighted": 0.627149 + } + ], + "main_score": 0.69685, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.2402026653289795, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADExpirationDateLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADExpirationDateLegalBenchClassification.json new file mode 100644 index 0000000000..8bfab6a291 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADExpirationDateLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADExpirationDateLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.874429, + "f1": 0.873264, + "f1_weighted": 0.873264, + "ap": 0.860679, + "ap_weighted": 0.860679, + "scores_per_experiment": [ + { + "accuracy": 0.874429, + "f1": 0.873264, + "f1_weighted": 0.873264, + "ap": 0.860679, + "ap_weighted": 0.860679 + }, + { + "accuracy": 0.874429, + "f1": 0.873264, + "f1_weighted": 0.873264, + "ap": 0.860679, + "ap_weighted": 0.860679 + }, + { + "accuracy": 0.874429, + "f1": 0.873264, + "f1_weighted": 0.873264, + "ap": 0.860679, + "ap_weighted": 0.860679 + }, + { + "accuracy": 0.874429, + "f1": 0.873264, + "f1_weighted": 0.873264, + "ap": 0.860679, + "ap_weighted": 0.860679 + }, + { + "accuracy": 0.874429, + "f1": 0.873264, + "f1_weighted": 0.873264, + "ap": 0.860679, + "ap_weighted": 0.860679 + }, + { + "accuracy": 0.874429, + "f1": 0.873264, + "f1_weighted": 0.873264, + "ap": 0.860679, + "ap_weighted": 0.860679 + }, + { + "accuracy": 0.874429, + "f1": 0.873264, + "f1_weighted": 0.873264, + "ap": 0.860679, + "ap_weighted": 0.860679 + }, + { + "accuracy": 0.874429, + "f1": 0.873264, + "f1_weighted": 0.873264, + "ap": 0.860679, + "ap_weighted": 0.860679 + }, + { + "accuracy": 0.874429, + "f1": 0.873264, + "f1_weighted": 0.873264, + "ap": 0.860679, + "ap_weighted": 0.860679 + }, + { + "accuracy": 0.874429, + "f1": 0.873264, + "f1_weighted": 0.873264, + "ap": 0.860679, + "ap_weighted": 0.860679 + } + ], + "main_score": 0.874429, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.204254627227783, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADGoverningLawLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADGoverningLawLegalBenchClassification.json new file mode 100644 index 0000000000..dde574a84d --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADGoverningLawLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADGoverningLawLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.953196, + "f1": 0.953186, + "f1_weighted": 0.953186, + "ap": 0.926065, + "ap_weighted": 0.926065, + "scores_per_experiment": [ + { + "accuracy": 0.953196, + "f1": 0.953186, + "f1_weighted": 0.953186, + "ap": 0.926065, + "ap_weighted": 0.926065 + }, + { + "accuracy": 0.953196, + "f1": 0.953186, + "f1_weighted": 0.953186, + "ap": 0.926065, + "ap_weighted": 0.926065 + }, + { + "accuracy": 0.953196, + "f1": 0.953186, + "f1_weighted": 0.953186, + "ap": 0.926065, + "ap_weighted": 0.926065 + }, + { + "accuracy": 0.953196, + "f1": 0.953186, + "f1_weighted": 0.953186, + "ap": 0.926065, + "ap_weighted": 0.926065 + }, + { + "accuracy": 0.953196, + "f1": 0.953186, + "f1_weighted": 0.953186, + "ap": 0.926065, + "ap_weighted": 0.926065 + }, + { + "accuracy": 0.953196, + "f1": 0.953186, + "f1_weighted": 0.953186, + "ap": 0.926065, + "ap_weighted": 0.926065 + }, + { + "accuracy": 0.953196, + "f1": 0.953186, + "f1_weighted": 0.953186, + "ap": 0.926065, + "ap_weighted": 0.926065 + }, + { + "accuracy": 0.953196, + "f1": 0.953186, + "f1_weighted": 0.953186, + "ap": 0.926065, + "ap_weighted": 0.926065 + }, + { + "accuracy": 0.953196, + "f1": 0.953186, + "f1_weighted": 0.953186, + "ap": 0.926065, + "ap_weighted": 0.926065 + }, + { + "accuracy": 0.953196, + "f1": 0.953186, + "f1_weighted": 0.953186, + "ap": 0.926065, + "ap_weighted": 0.926065 + } + ], + "main_score": 0.953196, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.1512935161590576, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADIPOwnershipAssignmentLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADIPOwnershipAssignmentLegalBenchClassification.json new file mode 100644 index 0000000000..d7aad7aa7d --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADIPOwnershipAssignmentLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADIPOwnershipAssignmentLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.756944, + "f1": 0.752164, + "f1_weighted": 0.752164, + "ap": 0.68014, + "ap_weighted": 0.68014, + "scores_per_experiment": [ + { + "accuracy": 0.756944, + "f1": 0.752164, + "f1_weighted": 0.752164, + "ap": 0.68014, + "ap_weighted": 0.68014 + }, + { + "accuracy": 0.756944, + "f1": 0.752164, + "f1_weighted": 0.752164, + "ap": 0.68014, + "ap_weighted": 0.68014 + }, + { + "accuracy": 0.756944, + "f1": 0.752164, + "f1_weighted": 0.752164, + "ap": 0.68014, + "ap_weighted": 0.68014 + }, + { + "accuracy": 0.756944, + "f1": 0.752164, + "f1_weighted": 0.752164, + "ap": 0.68014, + "ap_weighted": 0.68014 + }, + { + "accuracy": 0.756944, + "f1": 0.752164, + "f1_weighted": 0.752164, + "ap": 0.68014, + "ap_weighted": 0.68014 + }, + { + "accuracy": 0.756944, + "f1": 0.752164, + "f1_weighted": 0.752164, + "ap": 0.68014, + "ap_weighted": 0.68014 + }, + { + "accuracy": 0.756944, + "f1": 0.752164, + "f1_weighted": 0.752164, + "ap": 0.68014, + "ap_weighted": 0.68014 + }, + { + "accuracy": 0.756944, + "f1": 0.752164, + "f1_weighted": 0.752164, + "ap": 0.68014, + "ap_weighted": 0.68014 + }, + { + "accuracy": 0.756944, + "f1": 0.752164, + "f1_weighted": 0.752164, + "ap": 0.68014, + "ap_weighted": 0.68014 + }, + { + "accuracy": 0.756944, + "f1": 0.752164, + "f1_weighted": 0.752164, + "ap": 0.68014, + "ap_weighted": 0.68014 + } + ], + "main_score": 0.756944, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.299515724182129, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADInsuranceLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADInsuranceLegalBenchClassification.json new file mode 100644 index 0000000000..1f6d853492 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADInsuranceLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADInsuranceLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.915534, + "f1": 0.915457, + "f1_weighted": 0.915457, + "ap": 0.891495, + "ap_weighted": 0.891495, + "scores_per_experiment": [ + { + "accuracy": 0.915534, + "f1": 0.915457, + "f1_weighted": 0.915457, + "ap": 0.891495, + "ap_weighted": 0.891495 + }, + { + "accuracy": 0.915534, + "f1": 0.915457, + "f1_weighted": 0.915457, + "ap": 0.891495, + "ap_weighted": 0.891495 + }, + { + "accuracy": 0.915534, + "f1": 0.915457, + "f1_weighted": 0.915457, + "ap": 0.891495, + "ap_weighted": 0.891495 + }, + { + "accuracy": 0.915534, + "f1": 0.915457, + "f1_weighted": 0.915457, + "ap": 0.891495, + "ap_weighted": 0.891495 + }, + { + "accuracy": 0.915534, + "f1": 0.915457, + "f1_weighted": 0.915457, + "ap": 0.891495, + "ap_weighted": 0.891495 + }, + { + "accuracy": 0.915534, + "f1": 0.915457, + "f1_weighted": 0.915457, + "ap": 0.891495, + "ap_weighted": 0.891495 + }, + { + "accuracy": 0.915534, + "f1": 0.915457, + "f1_weighted": 0.915457, + "ap": 0.891495, + "ap_weighted": 0.891495 + }, + { + "accuracy": 0.915534, + "f1": 0.915457, + "f1_weighted": 0.915457, + "ap": 0.891495, + "ap_weighted": 0.891495 + }, + { + "accuracy": 0.915534, + "f1": 0.915457, + "f1_weighted": 0.915457, + "ap": 0.891495, + "ap_weighted": 0.891495 + }, + { + "accuracy": 0.915534, + "f1": 0.915457, + "f1_weighted": 0.915457, + "ap": 0.891495, + "ap_weighted": 0.891495 + } + ], + "main_score": 0.915534, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.6995105743408203, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADIrrevocableOrPerpetualLicenseLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADIrrevocableOrPerpetualLicenseLegalBenchClassification.json new file mode 100644 index 0000000000..a59b3a7bff --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADIrrevocableOrPerpetualLicenseLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADIrrevocableOrPerpetualLicenseLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.8, + "f1": 0.796638, + "f1_weighted": 0.796638, + "ap": 0.721591, + "ap_weighted": 0.721591, + "scores_per_experiment": [ + { + "accuracy": 0.8, + "f1": 0.796638, + "f1_weighted": 0.796638, + "ap": 0.721591, + "ap_weighted": 0.721591 + }, + { + "accuracy": 0.8, + "f1": 0.796638, + "f1_weighted": 0.796638, + "ap": 0.721591, + "ap_weighted": 0.721591 + }, + { + "accuracy": 0.8, + "f1": 0.796638, + "f1_weighted": 0.796638, + "ap": 0.721591, + "ap_weighted": 0.721591 + }, + { + "accuracy": 0.8, + "f1": 0.796638, + "f1_weighted": 0.796638, + "ap": 0.721591, + "ap_weighted": 0.721591 + }, + { + "accuracy": 0.8, + "f1": 0.796638, + "f1_weighted": 0.796638, + "ap": 0.721591, + "ap_weighted": 0.721591 + }, + { + "accuracy": 0.8, + "f1": 0.796638, + "f1_weighted": 0.796638, + "ap": 0.721591, + "ap_weighted": 0.721591 + }, + { + "accuracy": 0.8, + "f1": 0.796638, + "f1_weighted": 0.796638, + "ap": 0.721591, + "ap_weighted": 0.721591 + }, + { + "accuracy": 0.8, + "f1": 0.796638, + "f1_weighted": 0.796638, + "ap": 0.721591, + "ap_weighted": 0.721591 + }, + { + "accuracy": 0.8, + "f1": 0.796638, + "f1_weighted": 0.796638, + "ap": 0.721591, + "ap_weighted": 0.721591 + }, + { + "accuracy": 0.8, + "f1": 0.796638, + "f1_weighted": 0.796638, + "ap": 0.721591, + "ap_weighted": 0.721591 + } + ], + "main_score": 0.8, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.5313284397125244, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADJointIPOwnershipLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADJointIPOwnershipLegalBenchClassification.json new file mode 100644 index 0000000000..edf674f3e2 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADJointIPOwnershipLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADJointIPOwnershipLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.817708, + "f1": 0.815501, + "f1_weighted": 0.815501, + "ap": 0.788056, + "ap_weighted": 0.788056, + "scores_per_experiment": [ + { + "accuracy": 0.817708, + "f1": 0.815501, + "f1_weighted": 0.815501, + "ap": 0.788056, + "ap_weighted": 0.788056 + }, + { + "accuracy": 0.817708, + "f1": 0.815501, + "f1_weighted": 0.815501, + "ap": 0.788056, + "ap_weighted": 0.788056 + }, + { + "accuracy": 0.817708, + "f1": 0.815501, + "f1_weighted": 0.815501, + "ap": 0.788056, + "ap_weighted": 0.788056 + }, + { + "accuracy": 0.817708, + "f1": 0.815501, + "f1_weighted": 0.815501, + "ap": 0.788056, + "ap_weighted": 0.788056 + }, + { + "accuracy": 0.817708, + "f1": 0.815501, + "f1_weighted": 0.815501, + "ap": 0.788056, + "ap_weighted": 0.788056 + }, + { + "accuracy": 0.817708, + "f1": 0.815501, + "f1_weighted": 0.815501, + "ap": 0.788056, + "ap_weighted": 0.788056 + }, + { + "accuracy": 0.817708, + "f1": 0.815501, + "f1_weighted": 0.815501, + "ap": 0.788056, + "ap_weighted": 0.788056 + }, + { + "accuracy": 0.817708, + "f1": 0.815501, + "f1_weighted": 0.815501, + "ap": 0.788056, + "ap_weighted": 0.788056 + }, + { + "accuracy": 0.817708, + "f1": 0.815501, + "f1_weighted": 0.815501, + "ap": 0.788056, + "ap_weighted": 0.788056 + }, + { + "accuracy": 0.817708, + "f1": 0.815501, + "f1_weighted": 0.815501, + "ap": 0.788056, + "ap_weighted": 0.788056 + } + ], + "main_score": 0.817708, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.3193516731262207, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADLicenseGrantLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADLicenseGrantLegalBenchClassification.json new file mode 100644 index 0000000000..80c17d601e --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADLicenseGrantLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADLicenseGrantLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.739255, + "f1": 0.737314, + "f1_weighted": 0.737314, + "ap": 0.688755, + "ap_weighted": 0.688755, + "scores_per_experiment": [ + { + "accuracy": 0.739255, + "f1": 0.737314, + "f1_weighted": 0.737314, + "ap": 0.688755, + "ap_weighted": 0.688755 + }, + { + "accuracy": 0.739255, + "f1": 0.737314, + "f1_weighted": 0.737314, + "ap": 0.688755, + "ap_weighted": 0.688755 + }, + { + "accuracy": 0.739255, + "f1": 0.737314, + "f1_weighted": 0.737314, + "ap": 0.688755, + "ap_weighted": 0.688755 + }, + { + "accuracy": 0.739255, + "f1": 0.737314, + "f1_weighted": 0.737314, + "ap": 0.688755, + "ap_weighted": 0.688755 + }, + { + "accuracy": 0.739255, + "f1": 0.737314, + "f1_weighted": 0.737314, + "ap": 0.688755, + "ap_weighted": 0.688755 + }, + { + "accuracy": 0.739255, + "f1": 0.737314, + "f1_weighted": 0.737314, + "ap": 0.688755, + "ap_weighted": 0.688755 + }, + { + "accuracy": 0.739255, + "f1": 0.737314, + "f1_weighted": 0.737314, + "ap": 0.688755, + "ap_weighted": 0.688755 + }, + { + "accuracy": 0.739255, + "f1": 0.737314, + "f1_weighted": 0.737314, + "ap": 0.688755, + "ap_weighted": 0.688755 + }, + { + "accuracy": 0.739255, + "f1": 0.737314, + "f1_weighted": 0.737314, + "ap": 0.688755, + "ap_weighted": 0.688755 + }, + { + "accuracy": 0.739255, + "f1": 0.737314, + "f1_weighted": 0.737314, + "ap": 0.688755, + "ap_weighted": 0.688755 + } + ], + "main_score": 0.739255, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.8042378425598145, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADLiquidatedDamagesLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADLiquidatedDamagesLegalBenchClassification.json new file mode 100644 index 0000000000..0dc7232143 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADLiquidatedDamagesLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADLiquidatedDamagesLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.822727, + "f1": 0.820016, + "f1_weighted": 0.820016, + "ap": 0.74499, + "ap_weighted": 0.74499, + "scores_per_experiment": [ + { + "accuracy": 0.822727, + "f1": 0.820016, + "f1_weighted": 0.820016, + "ap": 0.74499, + "ap_weighted": 0.74499 + }, + { + "accuracy": 0.822727, + "f1": 0.820016, + "f1_weighted": 0.820016, + "ap": 0.74499, + "ap_weighted": 0.74499 + }, + { + "accuracy": 0.822727, + "f1": 0.820016, + "f1_weighted": 0.820016, + "ap": 0.74499, + "ap_weighted": 0.74499 + }, + { + "accuracy": 0.822727, + "f1": 0.820016, + "f1_weighted": 0.820016, + "ap": 0.74499, + "ap_weighted": 0.74499 + }, + { + "accuracy": 0.822727, + "f1": 0.820016, + "f1_weighted": 0.820016, + "ap": 0.74499, + "ap_weighted": 0.74499 + }, + { + "accuracy": 0.822727, + "f1": 0.820016, + "f1_weighted": 0.820016, + "ap": 0.74499, + "ap_weighted": 0.74499 + }, + { + "accuracy": 0.822727, + "f1": 0.820016, + "f1_weighted": 0.820016, + "ap": 0.74499, + "ap_weighted": 0.74499 + }, + { + "accuracy": 0.822727, + "f1": 0.820016, + "f1_weighted": 0.820016, + "ap": 0.74499, + "ap_weighted": 0.74499 + }, + { + "accuracy": 0.822727, + "f1": 0.820016, + "f1_weighted": 0.820016, + "ap": 0.74499, + "ap_weighted": 0.74499 + }, + { + "accuracy": 0.822727, + "f1": 0.820016, + "f1_weighted": 0.820016, + "ap": 0.74499, + "ap_weighted": 0.74499 + } + ], + "main_score": 0.822727, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.121999979019165, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADMinimumCommitmentLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADMinimumCommitmentLegalBenchClassification.json new file mode 100644 index 0000000000..a2caa07dfd --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADMinimumCommitmentLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADMinimumCommitmentLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.715026, + "f1": 0.714011, + "f1_weighted": 0.714011, + "ap": 0.648826, + "ap_weighted": 0.648826, + "scores_per_experiment": [ + { + "accuracy": 0.715026, + "f1": 0.714011, + "f1_weighted": 0.714011, + "ap": 0.648826, + "ap_weighted": 0.648826 + }, + { + "accuracy": 0.715026, + "f1": 0.714011, + "f1_weighted": 0.714011, + "ap": 0.648826, + "ap_weighted": 0.648826 + }, + { + "accuracy": 0.715026, + "f1": 0.714011, + "f1_weighted": 0.714011, + "ap": 0.648826, + "ap_weighted": 0.648826 + }, + { + "accuracy": 0.715026, + "f1": 0.714011, + "f1_weighted": 0.714011, + "ap": 0.648826, + "ap_weighted": 0.648826 + }, + { + "accuracy": 0.715026, + "f1": 0.714011, + "f1_weighted": 0.714011, + "ap": 0.648826, + "ap_weighted": 0.648826 + }, + { + "accuracy": 0.715026, + "f1": 0.714011, + "f1_weighted": 0.714011, + "ap": 0.648826, + "ap_weighted": 0.648826 + }, + { + "accuracy": 0.715026, + "f1": 0.714011, + "f1_weighted": 0.714011, + "ap": 0.648826, + "ap_weighted": 0.648826 + }, + { + "accuracy": 0.715026, + "f1": 0.714011, + "f1_weighted": 0.714011, + "ap": 0.648826, + "ap_weighted": 0.648826 + }, + { + "accuracy": 0.715026, + "f1": 0.714011, + "f1_weighted": 0.714011, + "ap": 0.648826, + "ap_weighted": 0.648826 + }, + { + "accuracy": 0.715026, + "f1": 0.714011, + "f1_weighted": 0.714011, + "ap": 0.648826, + "ap_weighted": 0.648826 + } + ], + "main_score": 0.715026, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.306438446044922, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADMostFavoredNationLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADMostFavoredNationLegalBenchClassification.json new file mode 100644 index 0000000000..5ee16645d5 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADMostFavoredNationLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADMostFavoredNationLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.765625, + "f1": 0.762787, + "f1_weighted": 0.762787, + "ap": 0.690705, + "ap_weighted": 0.690705, + "scores_per_experiment": [ + { + "accuracy": 0.765625, + "f1": 0.762787, + "f1_weighted": 0.762787, + "ap": 0.690705, + "ap_weighted": 0.690705 + }, + { + "accuracy": 0.765625, + "f1": 0.762787, + "f1_weighted": 0.762787, + "ap": 0.690705, + "ap_weighted": 0.690705 + }, + { + "accuracy": 0.765625, + "f1": 0.762787, + "f1_weighted": 0.762787, + "ap": 0.690705, + "ap_weighted": 0.690705 + }, + { + "accuracy": 0.765625, + "f1": 0.762787, + "f1_weighted": 0.762787, + "ap": 0.690705, + "ap_weighted": 0.690705 + }, + { + "accuracy": 0.765625, + "f1": 0.762787, + "f1_weighted": 0.762787, + "ap": 0.690705, + "ap_weighted": 0.690705 + }, + { + "accuracy": 0.765625, + "f1": 0.762787, + "f1_weighted": 0.762787, + "ap": 0.690705, + "ap_weighted": 0.690705 + }, + { + "accuracy": 0.765625, + "f1": 0.762787, + "f1_weighted": 0.762787, + "ap": 0.690705, + "ap_weighted": 0.690705 + }, + { + "accuracy": 0.765625, + "f1": 0.762787, + "f1_weighted": 0.762787, + "ap": 0.690705, + "ap_weighted": 0.690705 + }, + { + "accuracy": 0.765625, + "f1": 0.762787, + "f1_weighted": 0.762787, + "ap": 0.690705, + "ap_weighted": 0.690705 + }, + { + "accuracy": 0.765625, + "f1": 0.762787, + "f1_weighted": 0.762787, + "ap": 0.690705, + "ap_weighted": 0.690705 + } + ], + "main_score": 0.765625, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.7067184448242188, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADNoSolicitOfCustomersLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADNoSolicitOfCustomersLegalBenchClassification.json new file mode 100644 index 0000000000..ae99df497a --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADNoSolicitOfCustomersLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADNoSolicitOfCustomersLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.702381, + "f1": 0.686333, + "f1_weighted": 0.686333, + "ap": 0.629391, + "ap_weighted": 0.629391, + "scores_per_experiment": [ + { + "accuracy": 0.702381, + "f1": 0.686333, + "f1_weighted": 0.686333, + "ap": 0.629391, + "ap_weighted": 0.629391 + }, + { + "accuracy": 0.702381, + "f1": 0.686333, + "f1_weighted": 0.686333, + "ap": 0.629391, + "ap_weighted": 0.629391 + }, + { + "accuracy": 0.702381, + "f1": 0.686333, + "f1_weighted": 0.686333, + "ap": 0.629391, + "ap_weighted": 0.629391 + }, + { + "accuracy": 0.702381, + "f1": 0.686333, + "f1_weighted": 0.686333, + "ap": 0.629391, + "ap_weighted": 0.629391 + }, + { + "accuracy": 0.702381, + "f1": 0.686333, + "f1_weighted": 0.686333, + "ap": 0.629391, + "ap_weighted": 0.629391 + }, + { + "accuracy": 0.702381, + "f1": 0.686333, + "f1_weighted": 0.686333, + "ap": 0.629391, + "ap_weighted": 0.629391 + }, + { + "accuracy": 0.702381, + "f1": 0.686333, + "f1_weighted": 0.686333, + "ap": 0.629391, + "ap_weighted": 0.629391 + }, + { + "accuracy": 0.702381, + "f1": 0.686333, + "f1_weighted": 0.686333, + "ap": 0.629391, + "ap_weighted": 0.629391 + }, + { + "accuracy": 0.702381, + "f1": 0.686333, + "f1_weighted": 0.686333, + "ap": 0.629391, + "ap_weighted": 0.629391 + }, + { + "accuracy": 0.702381, + "f1": 0.686333, + "f1_weighted": 0.686333, + "ap": 0.629391, + "ap_weighted": 0.629391 + } + ], + "main_score": 0.702381, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.7409155368804932, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADNoSolicitOfEmployeesLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADNoSolicitOfEmployeesLegalBenchClassification.json new file mode 100644 index 0000000000..b998641652 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADNoSolicitOfEmployeesLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADNoSolicitOfEmployeesLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.873239, + "f1": 0.871609, + "f1_weighted": 0.871609, + "ap": 0.800308, + "ap_weighted": 0.800308, + "scores_per_experiment": [ + { + "accuracy": 0.873239, + "f1": 0.871609, + "f1_weighted": 0.871609, + "ap": 0.800308, + "ap_weighted": 0.800308 + }, + { + "accuracy": 0.873239, + "f1": 0.871609, + "f1_weighted": 0.871609, + "ap": 0.800308, + "ap_weighted": 0.800308 + }, + { + "accuracy": 0.873239, + "f1": 0.871609, + "f1_weighted": 0.871609, + "ap": 0.800308, + "ap_weighted": 0.800308 + }, + { + "accuracy": 0.873239, + "f1": 0.871609, + "f1_weighted": 0.871609, + "ap": 0.800308, + "ap_weighted": 0.800308 + }, + { + "accuracy": 0.873239, + "f1": 0.871609, + "f1_weighted": 0.871609, + "ap": 0.800308, + "ap_weighted": 0.800308 + }, + { + "accuracy": 0.873239, + "f1": 0.871609, + "f1_weighted": 0.871609, + "ap": 0.800308, + "ap_weighted": 0.800308 + }, + { + "accuracy": 0.873239, + "f1": 0.871609, + "f1_weighted": 0.871609, + "ap": 0.800308, + "ap_weighted": 0.800308 + }, + { + "accuracy": 0.873239, + "f1": 0.871609, + "f1_weighted": 0.871609, + "ap": 0.800308, + "ap_weighted": 0.800308 + }, + { + "accuracy": 0.873239, + "f1": 0.871609, + "f1_weighted": 0.871609, + "ap": 0.800308, + "ap_weighted": 0.800308 + }, + { + "accuracy": 0.873239, + "f1": 0.871609, + "f1_weighted": 0.871609, + "ap": 0.800308, + "ap_weighted": 0.800308 + } + ], + "main_score": 0.873239, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.1701767444610596, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADNonCompeteLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADNonCompeteLegalBenchClassification.json new file mode 100644 index 0000000000..8e31e1146d --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADNonCompeteLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADNonCompeteLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.647059, + "f1": 0.63901, + "f1_weighted": 0.63901, + "ap": 0.590182, + "ap_weighted": 0.590182, + "scores_per_experiment": [ + { + "accuracy": 0.647059, + "f1": 0.63901, + "f1_weighted": 0.63901, + "ap": 0.590182, + "ap_weighted": 0.590182 + }, + { + "accuracy": 0.647059, + "f1": 0.63901, + "f1_weighted": 0.63901, + "ap": 0.590182, + "ap_weighted": 0.590182 + }, + { + "accuracy": 0.647059, + "f1": 0.63901, + "f1_weighted": 0.63901, + "ap": 0.590182, + "ap_weighted": 0.590182 + }, + { + "accuracy": 0.647059, + "f1": 0.63901, + "f1_weighted": 0.63901, + "ap": 0.590182, + "ap_weighted": 0.590182 + }, + { + "accuracy": 0.647059, + "f1": 0.63901, + "f1_weighted": 0.63901, + "ap": 0.590182, + "ap_weighted": 0.590182 + }, + { + "accuracy": 0.647059, + "f1": 0.63901, + "f1_weighted": 0.63901, + "ap": 0.590182, + "ap_weighted": 0.590182 + }, + { + "accuracy": 0.647059, + "f1": 0.63901, + "f1_weighted": 0.63901, + "ap": 0.590182, + "ap_weighted": 0.590182 + }, + { + "accuracy": 0.647059, + "f1": 0.63901, + "f1_weighted": 0.63901, + "ap": 0.590182, + "ap_weighted": 0.590182 + }, + { + "accuracy": 0.647059, + "f1": 0.63901, + "f1_weighted": 0.63901, + "ap": 0.590182, + "ap_weighted": 0.590182 + }, + { + "accuracy": 0.647059, + "f1": 0.63901, + "f1_weighted": 0.63901, + "ap": 0.590182, + "ap_weighted": 0.590182 + } + ], + "main_score": 0.647059, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.8290743827819824, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADNonDisparagementLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADNonDisparagementLegalBenchClassification.json new file mode 100644 index 0000000000..97cf0f03f3 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADNonDisparagementLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADNonDisparagementLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.77, + "f1": 0.769424, + "f1_weighted": 0.769424, + "ap": 0.701273, + "ap_weighted": 0.701273, + "scores_per_experiment": [ + { + "accuracy": 0.77, + "f1": 0.769424, + "f1_weighted": 0.769424, + "ap": 0.701273, + "ap_weighted": 0.701273 + }, + { + "accuracy": 0.77, + "f1": 0.769424, + "f1_weighted": 0.769424, + "ap": 0.701273, + "ap_weighted": 0.701273 + }, + { + "accuracy": 0.77, + "f1": 0.769424, + "f1_weighted": 0.769424, + "ap": 0.701273, + "ap_weighted": 0.701273 + }, + { + "accuracy": 0.77, + "f1": 0.769424, + "f1_weighted": 0.769424, + "ap": 0.701273, + "ap_weighted": 0.701273 + }, + { + "accuracy": 0.77, + "f1": 0.769424, + "f1_weighted": 0.769424, + "ap": 0.701273, + "ap_weighted": 0.701273 + }, + { + "accuracy": 0.77, + "f1": 0.769424, + "f1_weighted": 0.769424, + "ap": 0.701273, + "ap_weighted": 0.701273 + }, + { + "accuracy": 0.77, + "f1": 0.769424, + "f1_weighted": 0.769424, + "ap": 0.701273, + "ap_weighted": 0.701273 + }, + { + "accuracy": 0.77, + "f1": 0.769424, + "f1_weighted": 0.769424, + "ap": 0.701273, + "ap_weighted": 0.701273 + }, + { + "accuracy": 0.77, + "f1": 0.769424, + "f1_weighted": 0.769424, + "ap": 0.701273, + "ap_weighted": 0.701273 + }, + { + "accuracy": 0.77, + "f1": 0.769424, + "f1_weighted": 0.769424, + "ap": 0.701273, + "ap_weighted": 0.701273 + } + ], + "main_score": 0.77, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.8143632411956787, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADNonTransferableLicenseLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADNonTransferableLicenseLegalBenchClassification.json new file mode 100644 index 0000000000..d8dee97d67 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADNonTransferableLicenseLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADNonTransferableLicenseLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.848708, + "f1": 0.848541, + "f1_weighted": 0.848541, + "ap": 0.788378, + "ap_weighted": 0.788378, + "scores_per_experiment": [ + { + "accuracy": 0.848708, + "f1": 0.848541, + "f1_weighted": 0.848541, + "ap": 0.788378, + "ap_weighted": 0.788378 + }, + { + "accuracy": 0.848708, + "f1": 0.848541, + "f1_weighted": 0.848541, + "ap": 0.788378, + "ap_weighted": 0.788378 + }, + { + "accuracy": 0.848708, + "f1": 0.848541, + "f1_weighted": 0.848541, + "ap": 0.788378, + "ap_weighted": 0.788378 + }, + { + "accuracy": 0.848708, + "f1": 0.848541, + "f1_weighted": 0.848541, + "ap": 0.788378, + "ap_weighted": 0.788378 + }, + { + "accuracy": 0.848708, + "f1": 0.848541, + "f1_weighted": 0.848541, + "ap": 0.788378, + "ap_weighted": 0.788378 + }, + { + "accuracy": 0.848708, + "f1": 0.848541, + "f1_weighted": 0.848541, + "ap": 0.788378, + "ap_weighted": 0.788378 + }, + { + "accuracy": 0.848708, + "f1": 0.848541, + "f1_weighted": 0.848541, + "ap": 0.788378, + "ap_weighted": 0.788378 + }, + { + "accuracy": 0.848708, + "f1": 0.848541, + "f1_weighted": 0.848541, + "ap": 0.788378, + "ap_weighted": 0.788378 + }, + { + "accuracy": 0.848708, + "f1": 0.848541, + "f1_weighted": 0.848541, + "ap": 0.788378, + "ap_weighted": 0.788378 + }, + { + "accuracy": 0.848708, + "f1": 0.848541, + "f1_weighted": 0.848541, + "ap": 0.788378, + "ap_weighted": 0.788378 + } + ], + "main_score": 0.848708, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.8279833793640137, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADNoticePeriodToTerminateRenewalLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADNoticePeriodToTerminateRenewalLegalBenchClassification.json new file mode 100644 index 0000000000..1723ce00e5 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADNoticePeriodToTerminateRenewalLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADNoticePeriodToTerminateRenewalLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.900901, + "f1": 0.90009, + "f1_weighted": 0.90009, + "ap": 0.836634, + "ap_weighted": 0.836634, + "scores_per_experiment": [ + { + "accuracy": 0.900901, + "f1": 0.90009, + "f1_weighted": 0.90009, + "ap": 0.836634, + "ap_weighted": 0.836634 + }, + { + "accuracy": 0.900901, + "f1": 0.90009, + "f1_weighted": 0.90009, + "ap": 0.836634, + "ap_weighted": 0.836634 + }, + { + "accuracy": 0.900901, + "f1": 0.90009, + "f1_weighted": 0.90009, + "ap": 0.836634, + "ap_weighted": 0.836634 + }, + { + "accuracy": 0.900901, + "f1": 0.90009, + "f1_weighted": 0.90009, + "ap": 0.836634, + "ap_weighted": 0.836634 + }, + { + "accuracy": 0.900901, + "f1": 0.90009, + "f1_weighted": 0.90009, + "ap": 0.836634, + "ap_weighted": 0.836634 + }, + { + "accuracy": 0.900901, + "f1": 0.90009, + "f1_weighted": 0.90009, + "ap": 0.836634, + "ap_weighted": 0.836634 + }, + { + "accuracy": 0.900901, + "f1": 0.90009, + "f1_weighted": 0.90009, + "ap": 0.836634, + "ap_weighted": 0.836634 + }, + { + "accuracy": 0.900901, + "f1": 0.90009, + "f1_weighted": 0.90009, + "ap": 0.836634, + "ap_weighted": 0.836634 + }, + { + "accuracy": 0.900901, + "f1": 0.90009, + "f1_weighted": 0.90009, + "ap": 0.836634, + "ap_weighted": 0.836634 + }, + { + "accuracy": 0.900901, + "f1": 0.90009, + "f1_weighted": 0.90009, + "ap": 0.836634, + "ap_weighted": 0.836634 + } + ], + "main_score": 0.900901, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.1870026588439941, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADPostTerminationServicesLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADPostTerminationServicesLegalBenchClassification.json new file mode 100644 index 0000000000..132198cc51 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADPostTerminationServicesLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADPostTerminationServicesLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.595297, + "f1": 0.54485, + "f1_weighted": 0.54485, + "ap": 0.5531, + "ap_weighted": 0.5531, + "scores_per_experiment": [ + { + "accuracy": 0.595297, + "f1": 0.54485, + "f1_weighted": 0.54485, + "ap": 0.5531, + "ap_weighted": 0.5531 + }, + { + "accuracy": 0.595297, + "f1": 0.54485, + "f1_weighted": 0.54485, + "ap": 0.5531, + "ap_weighted": 0.5531 + }, + { + "accuracy": 0.595297, + "f1": 0.54485, + "f1_weighted": 0.54485, + "ap": 0.5531, + "ap_weighted": 0.5531 + }, + { + "accuracy": 0.595297, + "f1": 0.54485, + "f1_weighted": 0.54485, + "ap": 0.5531, + "ap_weighted": 0.5531 + }, + { + "accuracy": 0.595297, + "f1": 0.54485, + "f1_weighted": 0.54485, + "ap": 0.5531, + "ap_weighted": 0.5531 + }, + { + "accuracy": 0.595297, + "f1": 0.54485, + "f1_weighted": 0.54485, + "ap": 0.5531, + "ap_weighted": 0.5531 + }, + { + "accuracy": 0.595297, + "f1": 0.54485, + "f1_weighted": 0.54485, + "ap": 0.5531, + "ap_weighted": 0.5531 + }, + { + "accuracy": 0.595297, + "f1": 0.54485, + "f1_weighted": 0.54485, + "ap": 0.5531, + "ap_weighted": 0.5531 + }, + { + "accuracy": 0.595297, + "f1": 0.54485, + "f1_weighted": 0.54485, + "ap": 0.5531, + "ap_weighted": 0.5531 + }, + { + "accuracy": 0.595297, + "f1": 0.54485, + "f1_weighted": 0.54485, + "ap": 0.5531, + "ap_weighted": 0.5531 + } + ], + "main_score": 0.595297, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.3799021244049072, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADPriceRestrictionsLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADPriceRestrictionsLegalBenchClassification.json new file mode 100644 index 0000000000..f1379da0b0 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADPriceRestrictionsLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADPriceRestrictionsLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.891304, + "f1": 0.891253, + "f1_weighted": 0.891253, + "ap": 0.842391, + "ap_weighted": 0.842391, + "scores_per_experiment": [ + { + "accuracy": 0.891304, + "f1": 0.891253, + "f1_weighted": 0.891253, + "ap": 0.842391, + "ap_weighted": 0.842391 + }, + { + "accuracy": 0.891304, + "f1": 0.891253, + "f1_weighted": 0.891253, + "ap": 0.842391, + "ap_weighted": 0.842391 + }, + { + "accuracy": 0.891304, + "f1": 0.891253, + "f1_weighted": 0.891253, + "ap": 0.842391, + "ap_weighted": 0.842391 + }, + { + "accuracy": 0.891304, + "f1": 0.891253, + "f1_weighted": 0.891253, + "ap": 0.842391, + "ap_weighted": 0.842391 + }, + { + "accuracy": 0.891304, + "f1": 0.891253, + "f1_weighted": 0.891253, + "ap": 0.842391, + "ap_weighted": 0.842391 + }, + { + "accuracy": 0.891304, + "f1": 0.891253, + "f1_weighted": 0.891253, + "ap": 0.842391, + "ap_weighted": 0.842391 + }, + { + "accuracy": 0.891304, + "f1": 0.891253, + "f1_weighted": 0.891253, + "ap": 0.842391, + "ap_weighted": 0.842391 + }, + { + "accuracy": 0.891304, + "f1": 0.891253, + "f1_weighted": 0.891253, + "ap": 0.842391, + "ap_weighted": 0.842391 + }, + { + "accuracy": 0.891304, + "f1": 0.891253, + "f1_weighted": 0.891253, + "ap": 0.842391, + "ap_weighted": 0.842391 + }, + { + "accuracy": 0.891304, + "f1": 0.891253, + "f1_weighted": 0.891253, + "ap": 0.842391, + "ap_weighted": 0.842391 + } + ], + "main_score": 0.891304, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.6577069759368896, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADRenewalTermLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADRenewalTermLegalBenchClassification.json new file mode 100644 index 0000000000..0cccaa7004 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADRenewalTermLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADRenewalTermLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.930052, + "f1": 0.93004, + "f1_weighted": 0.93004, + "ap": 0.8953, + "ap_weighted": 0.8953, + "scores_per_experiment": [ + { + "accuracy": 0.930052, + "f1": 0.93004, + "f1_weighted": 0.93004, + "ap": 0.8953, + "ap_weighted": 0.8953 + }, + { + "accuracy": 0.930052, + "f1": 0.93004, + "f1_weighted": 0.93004, + "ap": 0.8953, + "ap_weighted": 0.8953 + }, + { + "accuracy": 0.930052, + "f1": 0.93004, + "f1_weighted": 0.93004, + "ap": 0.8953, + "ap_weighted": 0.8953 + }, + { + "accuracy": 0.930052, + "f1": 0.93004, + "f1_weighted": 0.93004, + "ap": 0.8953, + "ap_weighted": 0.8953 + }, + { + "accuracy": 0.930052, + "f1": 0.93004, + "f1_weighted": 0.93004, + "ap": 0.8953, + "ap_weighted": 0.8953 + }, + { + "accuracy": 0.930052, + "f1": 0.93004, + "f1_weighted": 0.93004, + "ap": 0.8953, + "ap_weighted": 0.8953 + }, + { + "accuracy": 0.930052, + "f1": 0.93004, + "f1_weighted": 0.93004, + "ap": 0.8953, + "ap_weighted": 0.8953 + }, + { + "accuracy": 0.930052, + "f1": 0.93004, + "f1_weighted": 0.93004, + "ap": 0.8953, + "ap_weighted": 0.8953 + }, + { + "accuracy": 0.930052, + "f1": 0.93004, + "f1_weighted": 0.93004, + "ap": 0.8953, + "ap_weighted": 0.8953 + }, + { + "accuracy": 0.930052, + "f1": 0.93004, + "f1_weighted": 0.93004, + "ap": 0.8953, + "ap_weighted": 0.8953 + } + ], + "main_score": 0.930052, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.3864939212799072, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADRevenueProfitSharingLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADRevenueProfitSharingLegalBenchClassification.json new file mode 100644 index 0000000000..c413b3435a --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADRevenueProfitSharingLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADRevenueProfitSharingLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.861757, + "f1": 0.860893, + "f1_weighted": 0.860893, + "ap": 0.836234, + "ap_weighted": 0.836234, + "scores_per_experiment": [ + { + "accuracy": 0.861757, + "f1": 0.860893, + "f1_weighted": 0.860893, + "ap": 0.836234, + "ap_weighted": 0.836234 + }, + { + "accuracy": 0.861757, + "f1": 0.860893, + "f1_weighted": 0.860893, + "ap": 0.836234, + "ap_weighted": 0.836234 + }, + { + "accuracy": 0.861757, + "f1": 0.860893, + "f1_weighted": 0.860893, + "ap": 0.836234, + "ap_weighted": 0.836234 + }, + { + "accuracy": 0.861757, + "f1": 0.860893, + "f1_weighted": 0.860893, + "ap": 0.836234, + "ap_weighted": 0.836234 + }, + { + "accuracy": 0.861757, + "f1": 0.860893, + "f1_weighted": 0.860893, + "ap": 0.836234, + "ap_weighted": 0.836234 + }, + { + "accuracy": 0.861757, + "f1": 0.860893, + "f1_weighted": 0.860893, + "ap": 0.836234, + "ap_weighted": 0.836234 + }, + { + "accuracy": 0.861757, + "f1": 0.860893, + "f1_weighted": 0.860893, + "ap": 0.836234, + "ap_weighted": 0.836234 + }, + { + "accuracy": 0.861757, + "f1": 0.860893, + "f1_weighted": 0.860893, + "ap": 0.836234, + "ap_weighted": 0.836234 + }, + { + "accuracy": 0.861757, + "f1": 0.860893, + "f1_weighted": 0.860893, + "ap": 0.836234, + "ap_weighted": 0.836234 + }, + { + "accuracy": 0.861757, + "f1": 0.860893, + "f1_weighted": 0.860893, + "ap": 0.836234, + "ap_weighted": 0.836234 + } + ], + "main_score": 0.861757, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.2420923709869385, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADRofrRofoRofnLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADRofrRofoRofnLegalBenchClassification.json new file mode 100644 index 0000000000..09516693e7 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADRofrRofoRofnLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADRofrRofoRofnLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.64058, + "f1": 0.639486, + "f1_weighted": 0.639486, + "ap": 0.592499, + "ap_weighted": 0.592499, + "scores_per_experiment": [ + { + "accuracy": 0.64058, + "f1": 0.639486, + "f1_weighted": 0.639486, + "ap": 0.592499, + "ap_weighted": 0.592499 + }, + { + "accuracy": 0.64058, + "f1": 0.639486, + "f1_weighted": 0.639486, + "ap": 0.592499, + "ap_weighted": 0.592499 + }, + { + "accuracy": 0.64058, + "f1": 0.639486, + "f1_weighted": 0.639486, + "ap": 0.592499, + "ap_weighted": 0.592499 + }, + { + "accuracy": 0.64058, + "f1": 0.639486, + "f1_weighted": 0.639486, + "ap": 0.592499, + "ap_weighted": 0.592499 + }, + { + "accuracy": 0.64058, + "f1": 0.639486, + "f1_weighted": 0.639486, + "ap": 0.592499, + "ap_weighted": 0.592499 + }, + { + "accuracy": 0.64058, + "f1": 0.639486, + "f1_weighted": 0.639486, + "ap": 0.592499, + "ap_weighted": 0.592499 + }, + { + "accuracy": 0.64058, + "f1": 0.639486, + "f1_weighted": 0.639486, + "ap": 0.592499, + "ap_weighted": 0.592499 + }, + { + "accuracy": 0.64058, + "f1": 0.639486, + "f1_weighted": 0.639486, + "ap": 0.592499, + "ap_weighted": 0.592499 + }, + { + "accuracy": 0.64058, + "f1": 0.639486, + "f1_weighted": 0.639486, + "ap": 0.592499, + "ap_weighted": 0.592499 + }, + { + "accuracy": 0.64058, + "f1": 0.639486, + "f1_weighted": 0.639486, + "ap": 0.592499, + "ap_weighted": 0.592499 + } + ], + "main_score": 0.64058, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.1234817504882812, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADSourceCodeEscrowLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADSourceCodeEscrowLegalBenchClassification.json new file mode 100644 index 0000000000..14f49a3ea6 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADSourceCodeEscrowLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADSourceCodeEscrowLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.762712, + "f1": 0.760995, + "f1_weighted": 0.760995, + "ap": 0.714459, + "ap_weighted": 0.714459, + "scores_per_experiment": [ + { + "accuracy": 0.762712, + "f1": 0.760995, + "f1_weighted": 0.760995, + "ap": 0.714459, + "ap_weighted": 0.714459 + }, + { + "accuracy": 0.762712, + "f1": 0.760995, + "f1_weighted": 0.760995, + "ap": 0.714459, + "ap_weighted": 0.714459 + }, + { + "accuracy": 0.762712, + "f1": 0.760995, + "f1_weighted": 0.760995, + "ap": 0.714459, + "ap_weighted": 0.714459 + }, + { + "accuracy": 0.762712, + "f1": 0.760995, + "f1_weighted": 0.760995, + "ap": 0.714459, + "ap_weighted": 0.714459 + }, + { + "accuracy": 0.762712, + "f1": 0.760995, + "f1_weighted": 0.760995, + "ap": 0.714459, + "ap_weighted": 0.714459 + }, + { + "accuracy": 0.762712, + "f1": 0.760995, + "f1_weighted": 0.760995, + "ap": 0.714459, + "ap_weighted": 0.714459 + }, + { + "accuracy": 0.762712, + "f1": 0.760995, + "f1_weighted": 0.760995, + "ap": 0.714459, + "ap_weighted": 0.714459 + }, + { + "accuracy": 0.762712, + "f1": 0.760995, + "f1_weighted": 0.760995, + "ap": 0.714459, + "ap_weighted": 0.714459 + }, + { + "accuracy": 0.762712, + "f1": 0.760995, + "f1_weighted": 0.760995, + "ap": 0.714459, + "ap_weighted": 0.714459 + }, + { + "accuracy": 0.762712, + "f1": 0.760995, + "f1_weighted": 0.760995, + "ap": 0.714459, + "ap_weighted": 0.714459 + } + ], + "main_score": 0.762712, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.9123802185058594, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADTerminationForConvenienceLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADTerminationForConvenienceLegalBenchClassification.json new file mode 100644 index 0000000000..c790b9cb8c --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADTerminationForConvenienceLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADTerminationForConvenienceLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.904651, + "f1": 0.904626, + "f1_weighted": 0.904626, + "ap": 0.871579, + "ap_weighted": 0.871579, + "scores_per_experiment": [ + { + "accuracy": 0.904651, + "f1": 0.904626, + "f1_weighted": 0.904626, + "ap": 0.871579, + "ap_weighted": 0.871579 + }, + { + "accuracy": 0.904651, + "f1": 0.904626, + "f1_weighted": 0.904626, + "ap": 0.871579, + "ap_weighted": 0.871579 + }, + { + "accuracy": 0.904651, + "f1": 0.904626, + "f1_weighted": 0.904626, + "ap": 0.871579, + "ap_weighted": 0.871579 + }, + { + "accuracy": 0.904651, + "f1": 0.904626, + "f1_weighted": 0.904626, + "ap": 0.871579, + "ap_weighted": 0.871579 + }, + { + "accuracy": 0.904651, + "f1": 0.904626, + "f1_weighted": 0.904626, + "ap": 0.871579, + "ap_weighted": 0.871579 + }, + { + "accuracy": 0.904651, + "f1": 0.904626, + "f1_weighted": 0.904626, + "ap": 0.871579, + "ap_weighted": 0.871579 + }, + { + "accuracy": 0.904651, + "f1": 0.904626, + "f1_weighted": 0.904626, + "ap": 0.871579, + "ap_weighted": 0.871579 + }, + { + "accuracy": 0.904651, + "f1": 0.904626, + "f1_weighted": 0.904626, + "ap": 0.871579, + "ap_weighted": 0.871579 + }, + { + "accuracy": 0.904651, + "f1": 0.904626, + "f1_weighted": 0.904626, + "ap": 0.871579, + "ap_weighted": 0.871579 + }, + { + "accuracy": 0.904651, + "f1": 0.904626, + "f1_weighted": 0.904626, + "ap": 0.871579, + "ap_weighted": 0.871579 + } + ], + "main_score": 0.904651, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.362945795059204, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADThirdPartyBeneficiaryLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADThirdPartyBeneficiaryLegalBenchClassification.json new file mode 100644 index 0000000000..4abc2c962a --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADThirdPartyBeneficiaryLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADThirdPartyBeneficiaryLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.926471, + "f1": 0.926455, + "f1_weighted": 0.926455, + "ap": 0.900624, + "ap_weighted": 0.900624, + "scores_per_experiment": [ + { + "accuracy": 0.926471, + "f1": 0.926455, + "f1_weighted": 0.926455, + "ap": 0.900624, + "ap_weighted": 0.900624 + }, + { + "accuracy": 0.926471, + "f1": 0.926455, + "f1_weighted": 0.926455, + "ap": 0.900624, + "ap_weighted": 0.900624 + }, + { + "accuracy": 0.926471, + "f1": 0.926455, + "f1_weighted": 0.926455, + "ap": 0.900624, + "ap_weighted": 0.900624 + }, + { + "accuracy": 0.926471, + "f1": 0.926455, + "f1_weighted": 0.926455, + "ap": 0.900624, + "ap_weighted": 0.900624 + }, + { + "accuracy": 0.926471, + "f1": 0.926455, + "f1_weighted": 0.926455, + "ap": 0.900624, + "ap_weighted": 0.900624 + }, + { + "accuracy": 0.926471, + "f1": 0.926455, + "f1_weighted": 0.926455, + "ap": 0.900624, + "ap_weighted": 0.900624 + }, + { + "accuracy": 0.926471, + "f1": 0.926455, + "f1_weighted": 0.926455, + "ap": 0.900624, + "ap_weighted": 0.900624 + }, + { + "accuracy": 0.926471, + "f1": 0.926455, + "f1_weighted": 0.926455, + "ap": 0.900624, + "ap_weighted": 0.900624 + }, + { + "accuracy": 0.926471, + "f1": 0.926455, + "f1_weighted": 0.926455, + "ap": 0.900624, + "ap_weighted": 0.900624 + }, + { + "accuracy": 0.926471, + "f1": 0.926455, + "f1_weighted": 0.926455, + "ap": 0.900624, + "ap_weighted": 0.900624 + } + ], + "main_score": 0.926471, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.6996288299560547, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADUncappedLiabilityLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADUncappedLiabilityLegalBenchClassification.json new file mode 100644 index 0000000000..9ec078422d --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADUncappedLiabilityLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADUncappedLiabilityLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.789116, + "f1": 0.779951, + "f1_weighted": 0.779951, + "ap": 0.703917, + "ap_weighted": 0.703917, + "scores_per_experiment": [ + { + "accuracy": 0.789116, + "f1": 0.779951, + "f1_weighted": 0.779951, + "ap": 0.703917, + "ap_weighted": 0.703917 + }, + { + "accuracy": 0.789116, + "f1": 0.779951, + "f1_weighted": 0.779951, + "ap": 0.703917, + "ap_weighted": 0.703917 + }, + { + "accuracy": 0.789116, + "f1": 0.779951, + "f1_weighted": 0.779951, + "ap": 0.703917, + "ap_weighted": 0.703917 + }, + { + "accuracy": 0.789116, + "f1": 0.779951, + "f1_weighted": 0.779951, + "ap": 0.703917, + "ap_weighted": 0.703917 + }, + { + "accuracy": 0.789116, + "f1": 0.779951, + "f1_weighted": 0.779951, + "ap": 0.703917, + "ap_weighted": 0.703917 + }, + { + "accuracy": 0.789116, + "f1": 0.779951, + "f1_weighted": 0.779951, + "ap": 0.703917, + "ap_weighted": 0.703917 + }, + { + "accuracy": 0.789116, + "f1": 0.779951, + "f1_weighted": 0.779951, + "ap": 0.703917, + "ap_weighted": 0.703917 + }, + { + "accuracy": 0.789116, + "f1": 0.779951, + "f1_weighted": 0.779951, + "ap": 0.703917, + "ap_weighted": 0.703917 + }, + { + "accuracy": 0.789116, + "f1": 0.779951, + "f1_weighted": 0.779951, + "ap": 0.703917, + "ap_weighted": 0.703917 + }, + { + "accuracy": 0.789116, + "f1": 0.779951, + "f1_weighted": 0.779951, + "ap": 0.703917, + "ap_weighted": 0.703917 + } + ], + "main_score": 0.789116, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.5023176670074463, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADUnlimitedAllYouCanEatLicenseLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADUnlimitedAllYouCanEatLicenseLegalBenchClassification.json new file mode 100644 index 0000000000..56cfda0ab5 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADUnlimitedAllYouCanEatLicenseLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADUnlimitedAllYouCanEatLicenseLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.75, + "f1": 0.749565, + "f1_weighted": 0.749565, + "ap": 0.682692, + "ap_weighted": 0.682692, + "scores_per_experiment": [ + { + "accuracy": 0.75, + "f1": 0.749565, + "f1_weighted": 0.749565, + "ap": 0.682692, + "ap_weighted": 0.682692 + }, + { + "accuracy": 0.75, + "f1": 0.749565, + "f1_weighted": 0.749565, + "ap": 0.682692, + "ap_weighted": 0.682692 + }, + { + "accuracy": 0.75, + "f1": 0.749565, + "f1_weighted": 0.749565, + "ap": 0.682692, + "ap_weighted": 0.682692 + }, + { + "accuracy": 0.75, + "f1": 0.749565, + "f1_weighted": 0.749565, + "ap": 0.682692, + "ap_weighted": 0.682692 + }, + { + "accuracy": 0.75, + "f1": 0.749565, + "f1_weighted": 0.749565, + "ap": 0.682692, + "ap_weighted": 0.682692 + }, + { + "accuracy": 0.75, + "f1": 0.749565, + "f1_weighted": 0.749565, + "ap": 0.682692, + "ap_weighted": 0.682692 + }, + { + "accuracy": 0.75, + "f1": 0.749565, + "f1_weighted": 0.749565, + "ap": 0.682692, + "ap_weighted": 0.682692 + }, + { + "accuracy": 0.75, + "f1": 0.749565, + "f1_weighted": 0.749565, + "ap": 0.682692, + "ap_weighted": 0.682692 + }, + { + "accuracy": 0.75, + "f1": 0.749565, + "f1_weighted": 0.749565, + "ap": 0.682692, + "ap_weighted": 0.682692 + }, + { + "accuracy": 0.75, + "f1": 0.749565, + "f1_weighted": 0.749565, + "ap": 0.682692, + "ap_weighted": 0.682692 + } + ], + "main_score": 0.75, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.6104097366333008, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADVolumeRestrictionLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADVolumeRestrictionLegalBenchClassification.json new file mode 100644 index 0000000000..255bc29d52 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADVolumeRestrictionLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADVolumeRestrictionLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.714286, + "f1": 0.69234, + "f1_weighted": 0.69234, + "ap": 0.705714, + "ap_weighted": 0.705714, + "scores_per_experiment": [ + { + "accuracy": 0.714286, + "f1": 0.69234, + "f1_weighted": 0.69234, + "ap": 0.705714, + "ap_weighted": 0.705714 + }, + { + "accuracy": 0.714286, + "f1": 0.69234, + "f1_weighted": 0.69234, + "ap": 0.705714, + "ap_weighted": 0.705714 + }, + { + "accuracy": 0.714286, + "f1": 0.69234, + "f1_weighted": 0.69234, + "ap": 0.705714, + "ap_weighted": 0.705714 + }, + { + "accuracy": 0.714286, + "f1": 0.69234, + "f1_weighted": 0.69234, + "ap": 0.705714, + "ap_weighted": 0.705714 + }, + { + "accuracy": 0.714286, + "f1": 0.69234, + "f1_weighted": 0.69234, + "ap": 0.705714, + "ap_weighted": 0.705714 + }, + { + "accuracy": 0.714286, + "f1": 0.69234, + "f1_weighted": 0.69234, + "ap": 0.705714, + "ap_weighted": 0.705714 + }, + { + "accuracy": 0.714286, + "f1": 0.69234, + "f1_weighted": 0.69234, + "ap": 0.705714, + "ap_weighted": 0.705714 + }, + { + "accuracy": 0.714286, + "f1": 0.69234, + "f1_weighted": 0.69234, + "ap": 0.705714, + "ap_weighted": 0.705714 + }, + { + "accuracy": 0.714286, + "f1": 0.69234, + "f1_weighted": 0.69234, + "ap": 0.705714, + "ap_weighted": 0.705714 + }, + { + "accuracy": 0.714286, + "f1": 0.69234, + "f1_weighted": 0.69234, + "ap": 0.705714, + "ap_weighted": 0.705714 + } + ], + "main_score": 0.714286, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.1023259162902832, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADWarrantyDurationLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADWarrantyDurationLegalBenchClassification.json new file mode 100644 index 0000000000..c0074ea369 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CUADWarrantyDurationLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CUADWarrantyDurationLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.8375, + "f1": 0.837271, + "f1_weighted": 0.837271, + "ap": 0.791892, + "ap_weighted": 0.791892, + "scores_per_experiment": [ + { + "accuracy": 0.8375, + "f1": 0.837271, + "f1_weighted": 0.837271, + "ap": 0.791892, + "ap_weighted": 0.791892 + }, + { + "accuracy": 0.8375, + "f1": 0.837271, + "f1_weighted": 0.837271, + "ap": 0.791892, + "ap_weighted": 0.791892 + }, + { + "accuracy": 0.8375, + "f1": 0.837271, + "f1_weighted": 0.837271, + "ap": 0.791892, + "ap_weighted": 0.791892 + }, + { + "accuracy": 0.8375, + "f1": 0.837271, + "f1_weighted": 0.837271, + "ap": 0.791892, + "ap_weighted": 0.791892 + }, + { + "accuracy": 0.8375, + "f1": 0.837271, + "f1_weighted": 0.837271, + "ap": 0.791892, + "ap_weighted": 0.791892 + }, + { + "accuracy": 0.8375, + "f1": 0.837271, + "f1_weighted": 0.837271, + "ap": 0.791892, + "ap_weighted": 0.791892 + }, + { + "accuracy": 0.8375, + "f1": 0.837271, + "f1_weighted": 0.837271, + "ap": 0.791892, + "ap_weighted": 0.791892 + }, + { + "accuracy": 0.8375, + "f1": 0.837271, + "f1_weighted": 0.837271, + "ap": 0.791892, + "ap_weighted": 0.791892 + }, + { + "accuracy": 0.8375, + "f1": 0.837271, + "f1_weighted": 0.837271, + "ap": 0.791892, + "ap_weighted": 0.791892 + }, + { + "accuracy": 0.8375, + "f1": 0.837271, + "f1_weighted": 0.837271, + "ap": 0.791892, + "ap_weighted": 0.791892 + } + ], + "main_score": 0.8375, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.2492742538452148, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CanadaTaxCourtOutcomesLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CanadaTaxCourtOutcomesLegalBenchClassification.json new file mode 100644 index 0000000000..d32525d844 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CanadaTaxCourtOutcomesLegalBenchClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CanadaTaxCourtOutcomesLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.729508, + "f1": 0.636976, + "f1_weighted": 0.761715, + "scores_per_experiment": [ + { + "accuracy": 0.729508, + "f1": 0.636976, + "f1_weighted": 0.761715 + }, + { + "accuracy": 0.729508, + "f1": 0.636976, + "f1_weighted": 0.761715 + }, + { + "accuracy": 0.729508, + "f1": 0.636976, + "f1_weighted": 0.761715 + }, + { + "accuracy": 0.729508, + "f1": 0.636976, + "f1_weighted": 0.761715 + }, + { + "accuracy": 0.729508, + "f1": 0.636976, + "f1_weighted": 0.761715 + }, + { + "accuracy": 0.729508, + "f1": 0.636976, + "f1_weighted": 0.761715 + }, + { + "accuracy": 0.729508, + "f1": 0.636976, + "f1_weighted": 0.761715 + }, + { + "accuracy": 0.729508, + "f1": 0.636976, + "f1_weighted": 0.761715 + }, + { + "accuracy": 0.729508, + "f1": 0.636976, + "f1_weighted": 0.761715 + }, + { + "accuracy": 0.729508, + "f1": 0.636976, + "f1_weighted": 0.761715 + } + ], + "main_score": 0.729508, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.2476081848144531, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CataloniaTweetClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CataloniaTweetClassification.json new file mode 100644 index 0000000000..7730365665 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CataloniaTweetClassification.json @@ -0,0 +1,261 @@ +{ + "dataset_revision": "cf24d44e517efa534f048e5fc5981f399ed25bee", + "task_name": "CataloniaTweetClassification", + "mteb_version": "1.34.14", + "scores": { + "validation": [ + { + "accuracy": 0.473797, + "f1": 0.482774, + "f1_weighted": 0.468026, + "scores_per_experiment": [ + { + "accuracy": 0.49727, + "f1": 0.516229, + "f1_weighted": 0.499046 + }, + { + "accuracy": 0.519603, + "f1": 0.527311, + "f1_weighted": 0.51921 + }, + { + "accuracy": 0.426799, + "f1": 0.429527, + "f1_weighted": 0.417902 + }, + { + "accuracy": 0.51067, + "f1": 0.527111, + "f1_weighted": 0.510688 + }, + { + "accuracy": 0.473449, + "f1": 0.48694, + "f1_weighted": 0.466538 + }, + { + "accuracy": 0.503226, + "f1": 0.515765, + "f1_weighted": 0.500847 + }, + { + "accuracy": 0.451613, + "f1": 0.45816, + "f1_weighted": 0.446313 + }, + { + "accuracy": 0.422829, + "f1": 0.416012, + "f1_weighted": 0.411318 + }, + { + "accuracy": 0.435732, + "f1": 0.43888, + "f1_weighted": 0.415671 + }, + { + "accuracy": 0.496774, + "f1": 0.511802, + "f1_weighted": 0.492729 + } + ], + "main_score": 0.473797, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "accuracy": 0.438358, + "f1": 0.422724, + "f1_weighted": 0.434643, + "scores_per_experiment": [ + { + "accuracy": 0.453234, + "f1": 0.431262, + "f1_weighted": 0.444608 + }, + { + "accuracy": 0.376617, + "f1": 0.358101, + "f1_weighted": 0.370195 + }, + { + "accuracy": 0.472637, + "f1": 0.450508, + "f1_weighted": 0.459243 + }, + { + "accuracy": 0.471642, + "f1": 0.450115, + "f1_weighted": 0.470287 + }, + { + "accuracy": 0.441294, + "f1": 0.418459, + "f1_weighted": 0.440867 + }, + { + "accuracy": 0.40199, + "f1": 0.402371, + "f1_weighted": 0.401602 + }, + { + "accuracy": 0.430846, + "f1": 0.425853, + "f1_weighted": 0.43731 + }, + { + "accuracy": 0.422388, + "f1": 0.418985, + "f1_weighted": 0.422041 + }, + { + "accuracy": 0.503483, + "f1": 0.486454, + "f1_weighted": 0.504746 + }, + { + "accuracy": 0.409453, + "f1": 0.385137, + "f1_weighted": 0.395535 + } + ], + "main_score": 0.438358, + "hf_subset": "catalan", + "languages": [ + "cat-Latn" + ] + } + ], + "test": [ + { + "accuracy": 0.474256, + "f1": 0.482733, + "f1_weighted": 0.46694, + "scores_per_experiment": [ + { + "accuracy": 0.479663, + "f1": 0.496053, + "f1_weighted": 0.480396 + }, + { + "accuracy": 0.522817, + "f1": 0.530847, + "f1_weighted": 0.520975 + }, + { + "accuracy": 0.423115, + "f1": 0.424685, + "f1_weighted": 0.410255 + }, + { + "accuracy": 0.505456, + "f1": 0.521014, + "f1_weighted": 0.503325 + }, + { + "accuracy": 0.465774, + "f1": 0.480648, + "f1_weighted": 0.453204 + }, + { + "accuracy": 0.518849, + "f1": 0.528505, + "f1_weighted": 0.515725 + }, + { + "accuracy": 0.460317, + "f1": 0.468852, + "f1_weighted": 0.455407 + }, + { + "accuracy": 0.415179, + "f1": 0.409023, + "f1_weighted": 0.401542 + }, + { + "accuracy": 0.453373, + "f1": 0.456133, + "f1_weighted": 0.434137 + }, + { + "accuracy": 0.498016, + "f1": 0.51157, + "f1_weighted": 0.494438 + } + ], + "main_score": 0.474256, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "accuracy": 0.448308, + "f1": 0.433868, + "f1_weighted": 0.444773, + "scores_per_experiment": [ + { + "accuracy": 0.465672, + "f1": 0.439096, + "f1_weighted": 0.456398 + }, + { + "accuracy": 0.383582, + "f1": 0.37446, + "f1_weighted": 0.379511 + }, + { + "accuracy": 0.491045, + "f1": 0.466291, + "f1_weighted": 0.479192 + }, + { + "accuracy": 0.444776, + "f1": 0.430012, + "f1_weighted": 0.443691 + }, + { + "accuracy": 0.441791, + "f1": 0.422937, + "f1_weighted": 0.440468 + }, + { + "accuracy": 0.417413, + "f1": 0.416956, + "f1_weighted": 0.418157 + }, + { + "accuracy": 0.446269, + "f1": 0.439621, + "f1_weighted": 0.450919 + }, + { + "accuracy": 0.443284, + "f1": 0.439507, + "f1_weighted": 0.443291 + }, + { + "accuracy": 0.495025, + "f1": 0.484063, + "f1_weighted": 0.495903 + }, + { + "accuracy": 0.454229, + "f1": 0.425741, + "f1_weighted": 0.440201 + } + ], + "main_score": 0.448308, + "hf_subset": "catalan", + "languages": [ + "cat-Latn" + ] + } + ] + }, + "evaluation_time": 19.839929819107056, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ClimateFEVER.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ClimateFEVER.json deleted file mode 100644 index 4a9e37933c..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/ClimateFEVER.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "test": { - "evaluation_time": 20574.24, - "map_at_1": 0.10353, - "map_at_10": 0.1733, - "map_at_100": 0.1897, - "map_at_1000": 0.19148, - "map_at_3": 0.14292, - "map_at_5": 0.15786, - "ndcg_at_1": 0.23388, - "ndcg_at_10": 0.24884, - "ndcg_at_100": 0.31726, - "ndcg_at_1000": 0.35005, - "ndcg_at_3": 0.19708, - "ndcg_at_5": 0.21581, - "precision_at_1": 0.23388, - "precision_at_10": 0.07876, - "precision_at_100": 0.01506, - "precision_at_1000": 0.00212, - "precision_at_3": 0.14376, - "precision_at_5": 0.11388, - "recall_at_1": 0.10353, - "recall_at_10": 0.30528, - "recall_at_100": 0.54338, - "recall_at_1000": 0.72617, - "recall_at_3": 0.18038, - "recall_at_5": 0.23152 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "ClimateFEVER", - "dataset_revision": "392b78eb68c07badcd7c2cd8f39af108375dfcce" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIConfidentialityOfAgreementLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIConfidentialityOfAgreementLegalBenchClassification.json new file mode 100644 index 0000000000..ccc0e801fe --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIConfidentialityOfAgreementLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "ContractNLIConfidentialityOfAgreementLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.5, + "f1": 0.487109, + "f1_weighted": 0.487109, + "ap": 0.5, + "ap_weighted": 0.5, + "scores_per_experiment": [ + { + "accuracy": 0.5, + "f1": 0.487109, + "f1_weighted": 0.487109, + "ap": 0.5, + "ap_weighted": 0.5 + }, + { + "accuracy": 0.5, + "f1": 0.487109, + "f1_weighted": 0.487109, + "ap": 0.5, + "ap_weighted": 0.5 + }, + { + "accuracy": 0.5, + "f1": 0.487109, + "f1_weighted": 0.487109, + "ap": 0.5, + "ap_weighted": 0.5 + }, + { + "accuracy": 0.5, + "f1": 0.487109, + "f1_weighted": 0.487109, + "ap": 0.5, + "ap_weighted": 0.5 + }, + { + "accuracy": 0.5, + "f1": 0.487109, + "f1_weighted": 0.487109, + "ap": 0.5, + "ap_weighted": 0.5 + }, + { + "accuracy": 0.5, + "f1": 0.487109, + "f1_weighted": 0.487109, + "ap": 0.5, + "ap_weighted": 0.5 + }, + { + "accuracy": 0.5, + "f1": 0.487109, + "f1_weighted": 0.487109, + "ap": 0.5, + "ap_weighted": 0.5 + }, + { + "accuracy": 0.5, + "f1": 0.487109, + "f1_weighted": 0.487109, + "ap": 0.5, + "ap_weighted": 0.5 + }, + { + "accuracy": 0.5, + "f1": 0.487109, + "f1_weighted": 0.487109, + "ap": 0.5, + "ap_weighted": 0.5 + }, + { + "accuracy": 0.5, + "f1": 0.487109, + "f1_weighted": 0.487109, + "ap": 0.5, + "ap_weighted": 0.5 + } + ], + "main_score": 0.5, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.8664898872375488, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIExplicitIdentificationLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIExplicitIdentificationLegalBenchClassification.json new file mode 100644 index 0000000000..5d72a1b3c4 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIExplicitIdentificationLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "ContractNLIExplicitIdentificationLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.633028, + "f1": 0.530172, + "f1_weighted": 0.669329, + "ap": 0.207496, + "ap_weighted": 0.207496, + "scores_per_experiment": [ + { + "accuracy": 0.633028, + "f1": 0.530172, + "f1_weighted": 0.669329, + "ap": 0.207496, + "ap_weighted": 0.207496 + }, + { + "accuracy": 0.633028, + "f1": 0.530172, + "f1_weighted": 0.669329, + "ap": 0.207496, + "ap_weighted": 0.207496 + }, + { + "accuracy": 0.633028, + "f1": 0.530172, + "f1_weighted": 0.669329, + "ap": 0.207496, + "ap_weighted": 0.207496 + }, + { + "accuracy": 0.633028, + "f1": 0.530172, + "f1_weighted": 0.669329, + "ap": 0.207496, + "ap_weighted": 0.207496 + }, + { + "accuracy": 0.633028, + "f1": 0.530172, + "f1_weighted": 0.669329, + "ap": 0.207496, + "ap_weighted": 0.207496 + }, + { + "accuracy": 0.633028, + "f1": 0.530172, + "f1_weighted": 0.669329, + "ap": 0.207496, + "ap_weighted": 0.207496 + }, + { + "accuracy": 0.633028, + "f1": 0.530172, + "f1_weighted": 0.669329, + "ap": 0.207496, + "ap_weighted": 0.207496 + }, + { + "accuracy": 0.633028, + "f1": 0.530172, + "f1_weighted": 0.669329, + "ap": 0.207496, + "ap_weighted": 0.207496 + }, + { + "accuracy": 0.633028, + "f1": 0.530172, + "f1_weighted": 0.669329, + "ap": 0.207496, + "ap_weighted": 0.207496 + }, + { + "accuracy": 0.633028, + "f1": 0.530172, + "f1_weighted": 0.669329, + "ap": 0.207496, + "ap_weighted": 0.207496 + } + ], + "main_score": 0.633028, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.9137561321258545, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIInclusionOfVerballyConveyedInformationLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIInclusionOfVerballyConveyedInformationLegalBenchClassification.json new file mode 100644 index 0000000000..e7cdc67d1a --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIInclusionOfVerballyConveyedInformationLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "ContractNLIInclusionOfVerballyConveyedInformationLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.438849, + "f1": 0.430328, + "f1_weighted": 0.431832, + "ap": 0.462744, + "ap_weighted": 0.462744, + "scores_per_experiment": [ + { + "accuracy": 0.438849, + "f1": 0.430328, + "f1_weighted": 0.431832, + "ap": 0.462744, + "ap_weighted": 0.462744 + }, + { + "accuracy": 0.438849, + "f1": 0.430328, + "f1_weighted": 0.431832, + "ap": 0.462744, + "ap_weighted": 0.462744 + }, + { + "accuracy": 0.438849, + "f1": 0.430328, + "f1_weighted": 0.431832, + "ap": 0.462744, + "ap_weighted": 0.462744 + }, + { + "accuracy": 0.438849, + "f1": 0.430328, + "f1_weighted": 0.431832, + "ap": 0.462744, + "ap_weighted": 0.462744 + }, + { + "accuracy": 0.438849, + "f1": 0.430328, + "f1_weighted": 0.431832, + "ap": 0.462744, + "ap_weighted": 0.462744 + }, + { + "accuracy": 0.438849, + "f1": 0.430328, + "f1_weighted": 0.431832, + "ap": 0.462744, + "ap_weighted": 0.462744 + }, + { + "accuracy": 0.438849, + "f1": 0.430328, + "f1_weighted": 0.431832, + "ap": 0.462744, + "ap_weighted": 0.462744 + }, + { + "accuracy": 0.438849, + "f1": 0.430328, + "f1_weighted": 0.431832, + "ap": 0.462744, + "ap_weighted": 0.462744 + }, + { + "accuracy": 0.438849, + "f1": 0.430328, + "f1_weighted": 0.431832, + "ap": 0.462744, + "ap_weighted": 0.462744 + }, + { + "accuracy": 0.438849, + "f1": 0.430328, + "f1_weighted": 0.431832, + "ap": 0.462744, + "ap_weighted": 0.462744 + } + ], + "main_score": 0.438849, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.024176836013794, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLILimitedUseLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLILimitedUseLegalBenchClassification.json new file mode 100644 index 0000000000..bef88b945f --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLILimitedUseLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "ContractNLILimitedUseLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.586538, + "f1": 0.554226, + "f1_weighted": 0.546148, + "ap": 0.527442, + "ap_weighted": 0.527442, + "scores_per_experiment": [ + { + "accuracy": 0.586538, + "f1": 0.554226, + "f1_weighted": 0.546148, + "ap": 0.527442, + "ap_weighted": 0.527442 + }, + { + "accuracy": 0.586538, + "f1": 0.554226, + "f1_weighted": 0.546148, + "ap": 0.527442, + "ap_weighted": 0.527442 + }, + { + "accuracy": 0.586538, + "f1": 0.554226, + "f1_weighted": 0.546148, + "ap": 0.527442, + "ap_weighted": 0.527442 + }, + { + "accuracy": 0.586538, + "f1": 0.554226, + "f1_weighted": 0.546148, + "ap": 0.527442, + "ap_weighted": 0.527442 + }, + { + "accuracy": 0.586538, + "f1": 0.554226, + "f1_weighted": 0.546148, + "ap": 0.527442, + "ap_weighted": 0.527442 + }, + { + "accuracy": 0.586538, + "f1": 0.554226, + "f1_weighted": 0.546148, + "ap": 0.527442, + "ap_weighted": 0.527442 + }, + { + "accuracy": 0.586538, + "f1": 0.554226, + "f1_weighted": 0.546148, + "ap": 0.527442, + "ap_weighted": 0.527442 + }, + { + "accuracy": 0.586538, + "f1": 0.554226, + "f1_weighted": 0.546148, + "ap": 0.527442, + "ap_weighted": 0.527442 + }, + { + "accuracy": 0.586538, + "f1": 0.554226, + "f1_weighted": 0.546148, + "ap": 0.527442, + "ap_weighted": 0.527442 + }, + { + "accuracy": 0.586538, + "f1": 0.554226, + "f1_weighted": 0.546148, + "ap": 0.527442, + "ap_weighted": 0.527442 + } + ], + "main_score": 0.586538, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.160595417022705, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLINoLicensingLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLINoLicensingLegalBenchClassification.json new file mode 100644 index 0000000000..e1b9e5e1af --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLINoLicensingLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "ContractNLINoLicensingLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.851852, + "f1": 0.850737, + "f1_weighted": 0.850896, + "ap": 0.817729, + "ap_weighted": 0.817729, + "scores_per_experiment": [ + { + "accuracy": 0.851852, + "f1": 0.850737, + "f1_weighted": 0.850896, + "ap": 0.817729, + "ap_weighted": 0.817729 + }, + { + "accuracy": 0.851852, + "f1": 0.850737, + "f1_weighted": 0.850896, + "ap": 0.817729, + "ap_weighted": 0.817729 + }, + { + "accuracy": 0.851852, + "f1": 0.850737, + "f1_weighted": 0.850896, + "ap": 0.817729, + "ap_weighted": 0.817729 + }, + { + "accuracy": 0.851852, + "f1": 0.850737, + "f1_weighted": 0.850896, + "ap": 0.817729, + "ap_weighted": 0.817729 + }, + { + "accuracy": 0.851852, + "f1": 0.850737, + "f1_weighted": 0.850896, + "ap": 0.817729, + "ap_weighted": 0.817729 + }, + { + "accuracy": 0.851852, + "f1": 0.850737, + "f1_weighted": 0.850896, + "ap": 0.817729, + "ap_weighted": 0.817729 + }, + { + "accuracy": 0.851852, + "f1": 0.850737, + "f1_weighted": 0.850896, + "ap": 0.817729, + "ap_weighted": 0.817729 + }, + { + "accuracy": 0.851852, + "f1": 0.850737, + "f1_weighted": 0.850896, + "ap": 0.817729, + "ap_weighted": 0.817729 + }, + { + "accuracy": 0.851852, + "f1": 0.850737, + "f1_weighted": 0.850896, + "ap": 0.817729, + "ap_weighted": 0.817729 + }, + { + "accuracy": 0.851852, + "f1": 0.850737, + "f1_weighted": 0.850896, + "ap": 0.817729, + "ap_weighted": 0.817729 + } + ], + "main_score": 0.851852, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.9752199649810791, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLINoticeOnCompelledDisclosureLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLINoticeOnCompelledDisclosureLegalBenchClassification.json new file mode 100644 index 0000000000..d1b805a477 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLINoticeOnCompelledDisclosureLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "ContractNLINoticeOnCompelledDisclosureLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.880282, + "f1": 0.879799, + "f1_weighted": 0.879799, + "ap": 0.855747, + "ap_weighted": 0.855747, + "scores_per_experiment": [ + { + "accuracy": 0.880282, + "f1": 0.879799, + "f1_weighted": 0.879799, + "ap": 0.855747, + "ap_weighted": 0.855747 + }, + { + "accuracy": 0.880282, + "f1": 0.879799, + "f1_weighted": 0.879799, + "ap": 0.855747, + "ap_weighted": 0.855747 + }, + { + "accuracy": 0.880282, + "f1": 0.879799, + "f1_weighted": 0.879799, + "ap": 0.855747, + "ap_weighted": 0.855747 + }, + { + "accuracy": 0.880282, + "f1": 0.879799, + "f1_weighted": 0.879799, + "ap": 0.855747, + "ap_weighted": 0.855747 + }, + { + "accuracy": 0.880282, + "f1": 0.879799, + "f1_weighted": 0.879799, + "ap": 0.855747, + "ap_weighted": 0.855747 + }, + { + "accuracy": 0.880282, + "f1": 0.879799, + "f1_weighted": 0.879799, + "ap": 0.855747, + "ap_weighted": 0.855747 + }, + { + "accuracy": 0.880282, + "f1": 0.879799, + "f1_weighted": 0.879799, + "ap": 0.855747, + "ap_weighted": 0.855747 + }, + { + "accuracy": 0.880282, + "f1": 0.879799, + "f1_weighted": 0.879799, + "ap": 0.855747, + "ap_weighted": 0.855747 + }, + { + "accuracy": 0.880282, + "f1": 0.879799, + "f1_weighted": 0.879799, + "ap": 0.855747, + "ap_weighted": 0.855747 + }, + { + "accuracy": 0.880282, + "f1": 0.879799, + "f1_weighted": 0.879799, + "ap": 0.855747, + "ap_weighted": 0.855747 + } + ], + "main_score": 0.880282, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.0194270610809326, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIPermissibleAcquirementOfSimilarInformationLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIPermissibleAcquirementOfSimilarInformationLegalBenchClassification.json new file mode 100644 index 0000000000..f0f7c99db7 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIPermissibleAcquirementOfSimilarInformationLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "ContractNLIPermissibleAcquirementOfSimilarInformationLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.752809, + "f1": 0.742267, + "f1_weighted": 0.742267, + "ap": 0.67191, + "ap_weighted": 0.67191, + "scores_per_experiment": [ + { + "accuracy": 0.752809, + "f1": 0.742267, + "f1_weighted": 0.742267, + "ap": 0.67191, + "ap_weighted": 0.67191 + }, + { + "accuracy": 0.752809, + "f1": 0.742267, + "f1_weighted": 0.742267, + "ap": 0.67191, + "ap_weighted": 0.67191 + }, + { + "accuracy": 0.752809, + "f1": 0.742267, + "f1_weighted": 0.742267, + "ap": 0.67191, + "ap_weighted": 0.67191 + }, + { + "accuracy": 0.752809, + "f1": 0.742267, + "f1_weighted": 0.742267, + "ap": 0.67191, + "ap_weighted": 0.67191 + }, + { + "accuracy": 0.752809, + "f1": 0.742267, + "f1_weighted": 0.742267, + "ap": 0.67191, + "ap_weighted": 0.67191 + }, + { + "accuracy": 0.752809, + "f1": 0.742267, + "f1_weighted": 0.742267, + "ap": 0.67191, + "ap_weighted": 0.67191 + }, + { + "accuracy": 0.752809, + "f1": 0.742267, + "f1_weighted": 0.742267, + "ap": 0.67191, + "ap_weighted": 0.67191 + }, + { + "accuracy": 0.752809, + "f1": 0.742267, + "f1_weighted": 0.742267, + "ap": 0.67191, + "ap_weighted": 0.67191 + }, + { + "accuracy": 0.752809, + "f1": 0.742267, + "f1_weighted": 0.742267, + "ap": 0.67191, + "ap_weighted": 0.67191 + }, + { + "accuracy": 0.752809, + "f1": 0.742267, + "f1_weighted": 0.742267, + "ap": 0.67191, + "ap_weighted": 0.67191 + } + ], + "main_score": 0.752809, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.969775915145874, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIPermissibleCopyLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIPermissibleCopyLegalBenchClassification.json new file mode 100644 index 0000000000..4d439dc833 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIPermissibleCopyLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "ContractNLIPermissibleCopyLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.344828, + "f1": 0.327411, + "f1_weighted": 0.390858, + "ap": 0.179589, + "ap_weighted": 0.179589, + "scores_per_experiment": [ + { + "accuracy": 0.344828, + "f1": 0.327411, + "f1_weighted": 0.390858, + "ap": 0.179589, + "ap_weighted": 0.179589 + }, + { + "accuracy": 0.344828, + "f1": 0.327411, + "f1_weighted": 0.390858, + "ap": 0.179589, + "ap_weighted": 0.179589 + }, + { + "accuracy": 0.344828, + "f1": 0.327411, + "f1_weighted": 0.390858, + "ap": 0.179589, + "ap_weighted": 0.179589 + }, + { + "accuracy": 0.344828, + "f1": 0.327411, + "f1_weighted": 0.390858, + "ap": 0.179589, + "ap_weighted": 0.179589 + }, + { + "accuracy": 0.344828, + "f1": 0.327411, + "f1_weighted": 0.390858, + "ap": 0.179589, + "ap_weighted": 0.179589 + }, + { + "accuracy": 0.344828, + "f1": 0.327411, + "f1_weighted": 0.390858, + "ap": 0.179589, + "ap_weighted": 0.179589 + }, + { + "accuracy": 0.344828, + "f1": 0.327411, + "f1_weighted": 0.390858, + "ap": 0.179589, + "ap_weighted": 0.179589 + }, + { + "accuracy": 0.344828, + "f1": 0.327411, + "f1_weighted": 0.390858, + "ap": 0.179589, + "ap_weighted": 0.179589 + }, + { + "accuracy": 0.344828, + "f1": 0.327411, + "f1_weighted": 0.390858, + "ap": 0.179589, + "ap_weighted": 0.179589 + }, + { + "accuracy": 0.344828, + "f1": 0.327411, + "f1_weighted": 0.390858, + "ap": 0.179589, + "ap_weighted": 0.179589 + } + ], + "main_score": 0.344828, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.8335626125335693, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIPermissibleDevelopmentOfSimilarInformationLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIPermissibleDevelopmentOfSimilarInformationLegalBenchClassification.json new file mode 100644 index 0000000000..9f7cb60816 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIPermissibleDevelopmentOfSimilarInformationLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "ContractNLIPermissibleDevelopmentOfSimilarInformationLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.882353, + "f1": 0.882327, + "f1_weighted": 0.882327, + "ap": 0.8418, + "ap_weighted": 0.8418, + "scores_per_experiment": [ + { + "accuracy": 0.882353, + "f1": 0.882327, + "f1_weighted": 0.882327, + "ap": 0.8418, + "ap_weighted": 0.8418 + }, + { + "accuracy": 0.882353, + "f1": 0.882327, + "f1_weighted": 0.882327, + "ap": 0.8418, + "ap_weighted": 0.8418 + }, + { + "accuracy": 0.882353, + "f1": 0.882327, + "f1_weighted": 0.882327, + "ap": 0.8418, + "ap_weighted": 0.8418 + }, + { + "accuracy": 0.882353, + "f1": 0.882327, + "f1_weighted": 0.882327, + "ap": 0.8418, + "ap_weighted": 0.8418 + }, + { + "accuracy": 0.882353, + "f1": 0.882327, + "f1_weighted": 0.882327, + "ap": 0.8418, + "ap_weighted": 0.8418 + }, + { + "accuracy": 0.882353, + "f1": 0.882327, + "f1_weighted": 0.882327, + "ap": 0.8418, + "ap_weighted": 0.8418 + }, + { + "accuracy": 0.882353, + "f1": 0.882327, + "f1_weighted": 0.882327, + "ap": 0.8418, + "ap_weighted": 0.8418 + }, + { + "accuracy": 0.882353, + "f1": 0.882327, + "f1_weighted": 0.882327, + "ap": 0.8418, + "ap_weighted": 0.8418 + }, + { + "accuracy": 0.882353, + "f1": 0.882327, + "f1_weighted": 0.882327, + "ap": 0.8418, + "ap_weighted": 0.8418 + }, + { + "accuracy": 0.882353, + "f1": 0.882327, + "f1_weighted": 0.882327, + "ap": 0.8418, + "ap_weighted": 0.8418 + } + ], + "main_score": 0.882353, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.979377031326294, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIPermissiblePostAgreementPossessionLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIPermissiblePostAgreementPossessionLegalBenchClassification.json new file mode 100644 index 0000000000..8ad630dcb2 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIPermissiblePostAgreementPossessionLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "ContractNLIPermissiblePostAgreementPossessionLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.603604, + "f1": 0.588889, + "f1_weighted": 0.627427, + "ap": 0.349769, + "ap_weighted": 0.349769, + "scores_per_experiment": [ + { + "accuracy": 0.603604, + "f1": 0.588889, + "f1_weighted": 0.627427, + "ap": 0.349769, + "ap_weighted": 0.349769 + }, + { + "accuracy": 0.603604, + "f1": 0.588889, + "f1_weighted": 0.627427, + "ap": 0.349769, + "ap_weighted": 0.349769 + }, + { + "accuracy": 0.603604, + "f1": 0.588889, + "f1_weighted": 0.627427, + "ap": 0.349769, + "ap_weighted": 0.349769 + }, + { + "accuracy": 0.603604, + "f1": 0.588889, + "f1_weighted": 0.627427, + "ap": 0.349769, + "ap_weighted": 0.349769 + }, + { + "accuracy": 0.603604, + "f1": 0.588889, + "f1_weighted": 0.627427, + "ap": 0.349769, + "ap_weighted": 0.349769 + }, + { + "accuracy": 0.603604, + "f1": 0.588889, + "f1_weighted": 0.627427, + "ap": 0.349769, + "ap_weighted": 0.349769 + }, + { + "accuracy": 0.603604, + "f1": 0.588889, + "f1_weighted": 0.627427, + "ap": 0.349769, + "ap_weighted": 0.349769 + }, + { + "accuracy": 0.603604, + "f1": 0.588889, + "f1_weighted": 0.627427, + "ap": 0.349769, + "ap_weighted": 0.349769 + }, + { + "accuracy": 0.603604, + "f1": 0.588889, + "f1_weighted": 0.627427, + "ap": 0.349769, + "ap_weighted": 0.349769 + }, + { + "accuracy": 0.603604, + "f1": 0.588889, + "f1_weighted": 0.627427, + "ap": 0.349769, + "ap_weighted": 0.349769 + } + ], + "main_score": 0.603604, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.02829909324646, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIReturnOfConfidentialInformationLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIReturnOfConfidentialInformationLegalBenchClassification.json new file mode 100644 index 0000000000..e56334688a --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLIReturnOfConfidentialInformationLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "ContractNLIReturnOfConfidentialInformationLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.863636, + "f1": 0.863605, + "f1_weighted": 0.863668, + "ap": 0.80303, + "ap_weighted": 0.80303, + "scores_per_experiment": [ + { + "accuracy": 0.863636, + "f1": 0.863605, + "f1_weighted": 0.863668, + "ap": 0.80303, + "ap_weighted": 0.80303 + }, + { + "accuracy": 0.863636, + "f1": 0.863605, + "f1_weighted": 0.863668, + "ap": 0.80303, + "ap_weighted": 0.80303 + }, + { + "accuracy": 0.863636, + "f1": 0.863605, + "f1_weighted": 0.863668, + "ap": 0.80303, + "ap_weighted": 0.80303 + }, + { + "accuracy": 0.863636, + "f1": 0.863605, + "f1_weighted": 0.863668, + "ap": 0.80303, + "ap_weighted": 0.80303 + }, + { + "accuracy": 0.863636, + "f1": 0.863605, + "f1_weighted": 0.863668, + "ap": 0.80303, + "ap_weighted": 0.80303 + }, + { + "accuracy": 0.863636, + "f1": 0.863605, + "f1_weighted": 0.863668, + "ap": 0.80303, + "ap_weighted": 0.80303 + }, + { + "accuracy": 0.863636, + "f1": 0.863605, + "f1_weighted": 0.863668, + "ap": 0.80303, + "ap_weighted": 0.80303 + }, + { + "accuracy": 0.863636, + "f1": 0.863605, + "f1_weighted": 0.863668, + "ap": 0.80303, + "ap_weighted": 0.80303 + }, + { + "accuracy": 0.863636, + "f1": 0.863605, + "f1_weighted": 0.863668, + "ap": 0.80303, + "ap_weighted": 0.80303 + }, + { + "accuracy": 0.863636, + "f1": 0.863605, + "f1_weighted": 0.863668, + "ap": 0.80303, + "ap_weighted": 0.80303 + } + ], + "main_score": 0.863636, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.849682092666626, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLISharingWithEmployeesLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLISharingWithEmployeesLegalBenchClassification.json new file mode 100644 index 0000000000..ea37a975de --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLISharingWithEmployeesLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "ContractNLISharingWithEmployeesLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.711765, + "f1": 0.693964, + "f1_weighted": 0.696569, + "ap": 0.647329, + "ap_weighted": 0.647329, + "scores_per_experiment": [ + { + "accuracy": 0.711765, + "f1": 0.693964, + "f1_weighted": 0.696569, + "ap": 0.647329, + "ap_weighted": 0.647329 + }, + { + "accuracy": 0.711765, + "f1": 0.693964, + "f1_weighted": 0.696569, + "ap": 0.647329, + "ap_weighted": 0.647329 + }, + { + "accuracy": 0.711765, + "f1": 0.693964, + "f1_weighted": 0.696569, + "ap": 0.647329, + "ap_weighted": 0.647329 + }, + { + "accuracy": 0.711765, + "f1": 0.693964, + "f1_weighted": 0.696569, + "ap": 0.647329, + "ap_weighted": 0.647329 + }, + { + "accuracy": 0.711765, + "f1": 0.693964, + "f1_weighted": 0.696569, + "ap": 0.647329, + "ap_weighted": 0.647329 + }, + { + "accuracy": 0.711765, + "f1": 0.693964, + "f1_weighted": 0.696569, + "ap": 0.647329, + "ap_weighted": 0.647329 + }, + { + "accuracy": 0.711765, + "f1": 0.693964, + "f1_weighted": 0.696569, + "ap": 0.647329, + "ap_weighted": 0.647329 + }, + { + "accuracy": 0.711765, + "f1": 0.693964, + "f1_weighted": 0.696569, + "ap": 0.647329, + "ap_weighted": 0.647329 + }, + { + "accuracy": 0.711765, + "f1": 0.693964, + "f1_weighted": 0.696569, + "ap": 0.647329, + "ap_weighted": 0.647329 + }, + { + "accuracy": 0.711765, + "f1": 0.693964, + "f1_weighted": 0.696569, + "ap": 0.647329, + "ap_weighted": 0.647329 + } + ], + "main_score": 0.711765, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.0544118881225586, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLISharingWithThirdPartiesLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLISharingWithThirdPartiesLegalBenchClassification.json new file mode 100644 index 0000000000..d7babab29a --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLISharingWithThirdPartiesLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "ContractNLISharingWithThirdPartiesLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.6, + "f1": 0.553658, + "f1_weighted": 0.58402, + "ap": 0.42816, + "ap_weighted": 0.42816, + "scores_per_experiment": [ + { + "accuracy": 0.6, + "f1": 0.553658, + "f1_weighted": 0.58402, + "ap": 0.42816, + "ap_weighted": 0.42816 + }, + { + "accuracy": 0.6, + "f1": 0.553658, + "f1_weighted": 0.58402, + "ap": 0.42816, + "ap_weighted": 0.42816 + }, + { + "accuracy": 0.6, + "f1": 0.553658, + "f1_weighted": 0.58402, + "ap": 0.42816, + "ap_weighted": 0.42816 + }, + { + "accuracy": 0.6, + "f1": 0.553658, + "f1_weighted": 0.58402, + "ap": 0.42816, + "ap_weighted": 0.42816 + }, + { + "accuracy": 0.6, + "f1": 0.553658, + "f1_weighted": 0.58402, + "ap": 0.42816, + "ap_weighted": 0.42816 + }, + { + "accuracy": 0.6, + "f1": 0.553658, + "f1_weighted": 0.58402, + "ap": 0.42816, + "ap_weighted": 0.42816 + }, + { + "accuracy": 0.6, + "f1": 0.553658, + "f1_weighted": 0.58402, + "ap": 0.42816, + "ap_weighted": 0.42816 + }, + { + "accuracy": 0.6, + "f1": 0.553658, + "f1_weighted": 0.58402, + "ap": 0.42816, + "ap_weighted": 0.42816 + }, + { + "accuracy": 0.6, + "f1": 0.553658, + "f1_weighted": 0.58402, + "ap": 0.42816, + "ap_weighted": 0.42816 + }, + { + "accuracy": 0.6, + "f1": 0.553658, + "f1_weighted": 0.58402, + "ap": 0.42816, + "ap_weighted": 0.42816 + } + ], + "main_score": 0.6, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.038496494293213, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLISurvivalOfObligationsLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLISurvivalOfObligationsLegalBenchClassification.json new file mode 100644 index 0000000000..8a55774400 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/ContractNLISurvivalOfObligationsLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "ContractNLISurvivalOfObligationsLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.433121, + "f1": 0.427172, + "f1_weighted": 0.423825, + "ap": 0.444608, + "ap_weighted": 0.444608, + "scores_per_experiment": [ + { + "accuracy": 0.433121, + "f1": 0.427172, + "f1_weighted": 0.423825, + "ap": 0.444608, + "ap_weighted": 0.444608 + }, + { + "accuracy": 0.433121, + "f1": 0.427172, + "f1_weighted": 0.423825, + "ap": 0.444608, + "ap_weighted": 0.444608 + }, + { + "accuracy": 0.433121, + "f1": 0.427172, + "f1_weighted": 0.423825, + "ap": 0.444608, + "ap_weighted": 0.444608 + }, + { + "accuracy": 0.433121, + "f1": 0.427172, + "f1_weighted": 0.423825, + "ap": 0.444608, + "ap_weighted": 0.444608 + }, + { + "accuracy": 0.433121, + "f1": 0.427172, + "f1_weighted": 0.423825, + "ap": 0.444608, + "ap_weighted": 0.444608 + }, + { + "accuracy": 0.433121, + "f1": 0.427172, + "f1_weighted": 0.423825, + "ap": 0.444608, + "ap_weighted": 0.444608 + }, + { + "accuracy": 0.433121, + "f1": 0.427172, + "f1_weighted": 0.423825, + "ap": 0.444608, + "ap_weighted": 0.444608 + }, + { + "accuracy": 0.433121, + "f1": 0.427172, + "f1_weighted": 0.423825, + "ap": 0.444608, + "ap_weighted": 0.444608 + }, + { + "accuracy": 0.433121, + "f1": 0.427172, + "f1_weighted": 0.423825, + "ap": 0.444608, + "ap_weighted": 0.444608 + }, + { + "accuracy": 0.433121, + "f1": 0.427172, + "f1_weighted": 0.423825, + "ap": 0.444608, + "ap_weighted": 0.444608 + } + ], + "main_score": 0.433121, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.8785722255706787, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CorporateLobbyingLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CorporateLobbyingLegalBenchClassification.json new file mode 100644 index 0000000000..f6c50d1500 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CorporateLobbyingLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "CorporateLobbyingLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.704082, + "f1": 0.413174, + "f1_weighted": 0.581816, + "ap": 0.295918, + "ap_weighted": 0.295918, + "scores_per_experiment": [ + { + "accuracy": 0.704082, + "f1": 0.413174, + "f1_weighted": 0.581816, + "ap": 0.295918, + "ap_weighted": 0.295918 + }, + { + "accuracy": 0.704082, + "f1": 0.413174, + "f1_weighted": 0.581816, + "ap": 0.295918, + "ap_weighted": 0.295918 + }, + { + "accuracy": 0.704082, + "f1": 0.413174, + "f1_weighted": 0.581816, + "ap": 0.295918, + "ap_weighted": 0.295918 + }, + { + "accuracy": 0.704082, + "f1": 0.413174, + "f1_weighted": 0.581816, + "ap": 0.295918, + "ap_weighted": 0.295918 + }, + { + "accuracy": 0.704082, + "f1": 0.413174, + "f1_weighted": 0.581816, + "ap": 0.295918, + "ap_weighted": 0.295918 + }, + { + "accuracy": 0.704082, + "f1": 0.413174, + "f1_weighted": 0.581816, + "ap": 0.295918, + "ap_weighted": 0.295918 + }, + { + "accuracy": 0.704082, + "f1": 0.413174, + "f1_weighted": 0.581816, + "ap": 0.295918, + "ap_weighted": 0.295918 + }, + { + "accuracy": 0.704082, + "f1": 0.413174, + "f1_weighted": 0.581816, + "ap": 0.295918, + "ap_weighted": 0.295918 + }, + { + "accuracy": 0.704082, + "f1": 0.413174, + "f1_weighted": 0.581816, + "ap": 0.295918, + "ap_weighted": 0.295918 + }, + { + "accuracy": 0.704082, + "f1": 0.413174, + "f1_weighted": 0.581816, + "ap": 0.295918, + "ap_weighted": 0.295918 + } + ], + "main_score": 0.704082, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 9.164570569992065, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CyrillicTurkicLangClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CyrillicTurkicLangClassification.json new file mode 100644 index 0000000000..72dfdf7cd2 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CyrillicTurkicLangClassification.json @@ -0,0 +1,81 @@ +{ + "dataset_revision": "e42d330f33d65b7b72dfd408883daf1661f06f18", + "task_name": "CyrillicTurkicLangClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.263037, + "f1": 0.239769, + "f1_weighted": 0.239787, + "scores_per_experiment": [ + { + "accuracy": 0.285645, + "f1": 0.239434, + "f1_weighted": 0.239436 + }, + { + "accuracy": 0.288574, + "f1": 0.263192, + "f1_weighted": 0.263177 + }, + { + "accuracy": 0.214844, + "f1": 0.199465, + "f1_weighted": 0.199474 + }, + { + "accuracy": 0.287109, + "f1": 0.251243, + "f1_weighted": 0.251314 + }, + { + "accuracy": 0.233887, + "f1": 0.224413, + "f1_weighted": 0.224425 + }, + { + "accuracy": 0.289551, + "f1": 0.278048, + "f1_weighted": 0.278054 + }, + { + "accuracy": 0.239746, + "f1": 0.213142, + "f1_weighted": 0.213143 + }, + { + "accuracy": 0.247559, + "f1": 0.220121, + "f1_weighted": 0.220185 + }, + { + "accuracy": 0.266602, + "f1": 0.251114, + "f1_weighted": 0.251121 + }, + { + "accuracy": 0.276855, + "f1": 0.257522, + "f1_weighted": 0.257542 + } + ], + "main_score": 0.263037, + "hf_subset": "default", + "languages": [ + "bak-Cyrl", + "chv-Cyrl", + "tat-Cyrl", + "kir-Cyrl", + "rus-Cyrl", + "kaz-Cyrl", + "tyv-Cyrl", + "krc-Cyrl", + "sah-Cyrl" + ] + } + ] + }, + "evaluation_time": 6.5052409172058105, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CzechProductReviewSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CzechProductReviewSentimentClassification.json new file mode 100644 index 0000000000..8bd082db4a --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CzechProductReviewSentimentClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "2e6fedf42c9c104e83dfd95c3a453721e683e244", + "task_name": "CzechProductReviewSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.427197, + "f1": 0.422995, + "f1_weighted": 0.422977, + "scores_per_experiment": [ + { + "accuracy": 0.394043, + "f1": 0.387224, + "f1_weighted": 0.387272 + }, + { + "accuracy": 0.409668, + "f1": 0.406577, + "f1_weighted": 0.406538 + }, + { + "accuracy": 0.408691, + "f1": 0.40449, + "f1_weighted": 0.404447 + }, + { + "accuracy": 0.434082, + "f1": 0.421132, + "f1_weighted": 0.421092 + }, + { + "accuracy": 0.448242, + "f1": 0.44683, + "f1_weighted": 0.446802 + }, + { + "accuracy": 0.441406, + "f1": 0.441694, + "f1_weighted": 0.441678 + }, + { + "accuracy": 0.453613, + "f1": 0.454758, + "f1_weighted": 0.454758 + }, + { + "accuracy": 0.445312, + "f1": 0.441271, + "f1_weighted": 0.441264 + }, + { + "accuracy": 0.431641, + "f1": 0.430525, + "f1_weighted": 0.430519 + }, + { + "accuracy": 0.405273, + "f1": 0.395453, + "f1_weighted": 0.395403 + } + ], + "main_score": 0.427197, + "hf_subset": "default", + "languages": [ + "ces-Latn" + ] + } + ] + }, + "evaluation_time": 7.335152626037598, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CzechSoMeSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CzechSoMeSentimentClassification.json new file mode 100644 index 0000000000..ea61205135 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CzechSoMeSentimentClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "6ced1d87a030915822b087bf539e6d5c658f1988", + "task_name": "CzechSoMeSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.4543, + "f1": 0.452879, + "f1_weighted": 0.452938, + "scores_per_experiment": [ + { + "accuracy": 0.425, + "f1": 0.424043, + "f1_weighted": 0.424118 + }, + { + "accuracy": 0.482, + "f1": 0.482567, + "f1_weighted": 0.482632 + }, + { + "accuracy": 0.462, + "f1": 0.462239, + "f1_weighted": 0.462323 + }, + { + "accuracy": 0.418, + "f1": 0.41295, + "f1_weighted": 0.413019 + }, + { + "accuracy": 0.489, + "f1": 0.486109, + "f1_weighted": 0.486158 + }, + { + "accuracy": 0.45, + "f1": 0.455292, + "f1_weighted": 0.455381 + }, + { + "accuracy": 0.392, + "f1": 0.388882, + "f1_weighted": 0.388917 + }, + { + "accuracy": 0.487, + "f1": 0.485307, + "f1_weighted": 0.485348 + }, + { + "accuracy": 0.452, + "f1": 0.453438, + "f1_weighted": 0.453453 + }, + { + "accuracy": 0.486, + "f1": 0.477964, + "f1_weighted": 0.47803 + } + ], + "main_score": 0.4543, + "hf_subset": "default", + "languages": [ + "ces-Latn" + ] + } + ] + }, + "evaluation_time": 1.942051649093628, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/CzechSubjectivityClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/CzechSubjectivityClassification.json new file mode 100644 index 0000000000..f989b5fa96 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/CzechSubjectivityClassification.json @@ -0,0 +1,181 @@ +{ + "dataset_revision": "e387ddf167f3eba99936cff89909ed6264f17e1f", + "task_name": "CzechSubjectivityClassification", + "mteb_version": "1.34.14", + "scores": { + "validation": [ + { + "accuracy": 0.5816, + "f1": 0.575168, + "f1_weighted": 0.575168, + "ap": 0.549162, + "ap_weighted": 0.549162, + "scores_per_experiment": [ + { + "accuracy": 0.544, + "f1": 0.530098, + "f1_weighted": 0.530098, + "ap": 0.524951, + "ap_weighted": 0.524951 + }, + { + "accuracy": 0.6, + "f1": 0.580424, + "f1_weighted": 0.580424, + "ap": 0.567606, + "ap_weighted": 0.567606 + }, + { + "accuracy": 0.584, + "f1": 0.580449, + "f1_weighted": 0.580449, + "ap": 0.550647, + "ap_weighted": 0.550647 + }, + { + "accuracy": 0.608, + "f1": 0.607372, + "f1_weighted": 0.607372, + "ap": 0.5648, + "ap_weighted": 0.5648 + }, + { + "accuracy": 0.598, + "f1": 0.580185, + "f1_weighted": 0.580185, + "ap": 0.555802, + "ap_weighted": 0.555802 + }, + { + "accuracy": 0.512, + "f1": 0.511719, + "f1_weighted": 0.511719, + "ap": 0.506151, + "ap_weighted": 0.506151 + }, + { + "accuracy": 0.598, + "f1": 0.595539, + "f1_weighted": 0.595539, + "ap": 0.560379, + "ap_weighted": 0.560379 + }, + { + "accuracy": 0.636, + "f1": 0.635013, + "f1_weighted": 0.635013, + "ap": 0.584754, + "ap_weighted": 0.584754 + }, + { + "accuracy": 0.584, + "f1": 0.582067, + "f1_weighted": 0.582067, + "ap": 0.548211, + "ap_weighted": 0.548211 + }, + { + "accuracy": 0.552, + "f1": 0.548816, + "f1_weighted": 0.548816, + "ap": 0.528315, + "ap_weighted": 0.528315 + } + ], + "main_score": 0.5816, + "hf_subset": "default", + "languages": [ + "ces-Latn" + ] + } + ], + "test": [ + { + "accuracy": 0.58275, + "f1": 0.576307, + "f1_weighted": 0.576307, + "ap": 0.54992, + "ap_weighted": 0.54992, + "scores_per_experiment": [ + { + "accuracy": 0.5695, + "f1": 0.558549, + "f1_weighted": 0.558549, + "ap": 0.541801, + "ap_weighted": 0.541801 + }, + { + "accuracy": 0.5965, + "f1": 0.573651, + "f1_weighted": 0.573651, + "ap": 0.565591, + "ap_weighted": 0.565591 + }, + { + "accuracy": 0.604, + "f1": 0.600693, + "f1_weighted": 0.600693, + "ap": 0.565222, + "ap_weighted": 0.565222 + }, + { + "accuracy": 0.579, + "f1": 0.578423, + "f1_weighted": 0.578423, + "ap": 0.545311, + "ap_weighted": 0.545311 + }, + { + "accuracy": 0.587, + "f1": 0.5722, + "f1_weighted": 0.5722, + "ap": 0.549017, + "ap_weighted": 0.549017 + }, + { + "accuracy": 0.512, + "f1": 0.511218, + "f1_weighted": 0.511218, + "ap": 0.506157, + "ap_weighted": 0.506157 + }, + { + "accuracy": 0.5985, + "f1": 0.594726, + "f1_weighted": 0.594726, + "ap": 0.561273, + "ap_weighted": 0.561273 + }, + { + "accuracy": 0.6005, + "f1": 0.599396, + "f1_weighted": 0.599396, + "ap": 0.55939, + "ap_weighted": 0.55939 + }, + { + "accuracy": 0.614, + "f1": 0.612412, + "f1_weighted": 0.612412, + "ap": 0.568521, + "ap_weighted": 0.568521 + }, + { + "accuracy": 0.5665, + "f1": 0.561806, + "f1_weighted": 0.561806, + "ap": 0.536914, + "ap_weighted": 0.536914 + } + ], + "main_score": 0.58275, + "hf_subset": "default", + "languages": [ + "ces-Latn" + ] + } + ] + }, + "evaluation_time": 5.602463722229004, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/DBPedia.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/DBPedia.json deleted file mode 100644 index 590545efbd..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/DBPedia.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "test": { - "evaluation_time": 9432.05, - "map_at_1": 0.0789, - "map_at_10": 0.16181, - "map_at_100": 0.21782, - "map_at_1000": 0.22957, - "map_at_3": 0.12038, - "map_at_5": 0.14049, - "ndcg_at_1": 0.49, - "ndcg_at_10": 0.35244, - "ndcg_at_100": 0.38366, - "ndcg_at_1000": 0.44831, - "ndcg_at_3": 0.40516, - "ndcg_at_5": 0.37747, - "precision_at_1": 0.6, - "precision_at_10": 0.27125, - "precision_at_100": 0.0823, - "precision_at_1000": 0.01714, - "precision_at_3": 0.4375, - "precision_at_5": 0.36, - "recall_at_1": 0.0789, - "recall_at_10": 0.2103, - "recall_at_100": 0.43303, - "recall_at_1000": 0.63688, - "recall_at_3": 0.13335, - "recall_at_5": 0.16745 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "DBPedia", - "dataset_revision": "f097057d03ed98220bc7309ddb10b71a54d667d6" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/DBpediaClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/DBpediaClassification.json new file mode 100644 index 0000000000..4e59a65007 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/DBpediaClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "9abd46cf7fc8b4c64290f26993c540b92aa145ac", + "task_name": "DBpediaClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.85, + "f1": 0.842361, + "f1_weighted": 0.842441, + "scores_per_experiment": [ + { + "accuracy": 0.838867, + "f1": 0.830432, + "f1_weighted": 0.830532 + }, + { + "accuracy": 0.836426, + "f1": 0.82325, + "f1_weighted": 0.823363 + }, + { + "accuracy": 0.848145, + "f1": 0.842291, + "f1_weighted": 0.842368 + }, + { + "accuracy": 0.856445, + "f1": 0.85117, + "f1_weighted": 0.851246 + }, + { + "accuracy": 0.855469, + "f1": 0.848678, + "f1_weighted": 0.848787 + }, + { + "accuracy": 0.860352, + "f1": 0.854169, + "f1_weighted": 0.854244 + }, + { + "accuracy": 0.86377, + "f1": 0.858243, + "f1_weighted": 0.858308 + }, + { + "accuracy": 0.838379, + "f1": 0.830653, + "f1_weighted": 0.830715 + }, + { + "accuracy": 0.838867, + "f1": 0.830757, + "f1_weighted": 0.830787 + }, + { + "accuracy": 0.863281, + "f1": 0.853964, + "f1_weighted": 0.854064 + } + ], + "main_score": 0.85, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 6.806412935256958, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/DKHateClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/DKHateClassification.json new file mode 100644 index 0000000000..74aaa3eea2 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/DKHateClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "59d12749a3c91a186063c7d729ec392fda94681c", + "task_name": "DKHateClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.564134, + "f1": 0.457631, + "f1_weighted": 0.635802, + "ap": 0.13483, + "ap_weighted": 0.13483, + "scores_per_experiment": [ + { + "accuracy": 0.571429, + "f1": 0.466333, + "f1_weighted": 0.644132, + "ap": 0.136537, + "ap_weighted": 0.136537 + }, + { + "accuracy": 0.607903, + "f1": 0.478259, + "f1_weighted": 0.673515, + "ap": 0.133638, + "ap_weighted": 0.133638 + }, + { + "accuracy": 0.583587, + "f1": 0.458758, + "f1_weighted": 0.653901, + "ap": 0.12717, + "ap_weighted": 0.12717 + }, + { + "accuracy": 0.504559, + "f1": 0.428533, + "f1_weighted": 0.58502, + "ap": 0.131511, + "ap_weighted": 0.131511 + }, + { + "accuracy": 0.537994, + "f1": 0.465726, + "f1_weighted": 0.613248, + "ap": 0.154016, + "ap_weighted": 0.154016 + }, + { + "accuracy": 0.544073, + "f1": 0.451764, + "f1_weighted": 0.620655, + "ap": 0.134907, + "ap_weighted": 0.134907 + }, + { + "accuracy": 0.492401, + "f1": 0.42314, + "f1_weighted": 0.573206, + "ap": 0.132412, + "ap_weighted": 0.132412 + }, + { + "accuracy": 0.553191, + "f1": 0.427826, + "f1_weighted": 0.628899, + "ap": 0.117742, + "ap_weighted": 0.117742 + }, + { + "accuracy": 0.574468, + "f1": 0.482006, + "f1_weighted": 0.646309, + "ap": 0.15094, + "ap_weighted": 0.15094 + }, + { + "accuracy": 0.671733, + "f1": 0.493961, + "f1_weighted": 0.719138, + "ap": 0.129428, + "ap_weighted": 0.129428 + } + ], + "main_score": 0.564134, + "hf_subset": "default", + "languages": [ + "dan-Latn" + ] + } + ] + }, + "evaluation_time": 2.676509141921997, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/DalajClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/DalajClassification.json new file mode 100644 index 0000000000..8d8718dd51 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/DalajClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "7ebf0b4caa7b2ae39698a889de782c09e6f5ee56", + "task_name": "DalajClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.501351, + "f1": 0.496637, + "f1_weighted": 0.496637, + "ap": 0.500738, + "ap_weighted": 0.500738, + "scores_per_experiment": [ + { + "accuracy": 0.512387, + "f1": 0.505473, + "f1_weighted": 0.505473, + "ap": 0.506318, + "ap_weighted": 0.506318 + }, + { + "accuracy": 0.503378, + "f1": 0.493008, + "f1_weighted": 0.493008, + "ap": 0.501698, + "ap_weighted": 0.501698 + }, + { + "accuracy": 0.503378, + "f1": 0.502211, + "f1_weighted": 0.502211, + "ap": 0.501702, + "ap_weighted": 0.501702 + }, + { + "accuracy": 0.495495, + "f1": 0.49537, + "f1_weighted": 0.49537, + "ap": 0.497769, + "ap_weighted": 0.497769 + }, + { + "accuracy": 0.492117, + "f1": 0.488065, + "f1_weighted": 0.488065, + "ap": 0.496134, + "ap_weighted": 0.496134 + }, + { + "accuracy": 0.486486, + "f1": 0.482285, + "f1_weighted": 0.482285, + "ap": 0.493466, + "ap_weighted": 0.493466 + }, + { + "accuracy": 0.505631, + "f1": 0.501274, + "f1_weighted": 0.501274, + "ap": 0.502854, + "ap_weighted": 0.502854 + }, + { + "accuracy": 0.507883, + "f1": 0.503957, + "f1_weighted": 0.503957, + "ap": 0.504017, + "ap_weighted": 0.504017 + }, + { + "accuracy": 0.501126, + "f1": 0.49694, + "f1_weighted": 0.49694, + "ap": 0.500565, + "ap_weighted": 0.500565 + }, + { + "accuracy": 0.505631, + "f1": 0.497783, + "f1_weighted": 0.497783, + "ap": 0.502858, + "ap_weighted": 0.502858 + } + ], + "main_score": 0.501351, + "hf_subset": "default", + "languages": [ + "swe-Latn" + ] + } + ] + }, + "evaluation_time": 2.4790689945220947, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/DanishPoliticalCommentsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/DanishPoliticalCommentsClassification.json new file mode 100644 index 0000000000..4e45d0886b --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/DanishPoliticalCommentsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "edbb03726c04a0efab14fc8c3b8b79e4d420e5a1", + "task_name": "DanishPoliticalCommentsClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.268845, + "f1": 0.248598, + "f1_weighted": 0.2828, + "scores_per_experiment": [ + { + "accuracy": 0.283514, + "f1": 0.267711, + "f1_weighted": 0.295015 + }, + { + "accuracy": 0.268249, + "f1": 0.244564, + "f1_weighted": 0.284178 + }, + { + "accuracy": 0.27491, + "f1": 0.248334, + "f1_weighted": 0.298296 + }, + { + "accuracy": 0.262975, + "f1": 0.24745, + "f1_weighted": 0.279499 + }, + { + "accuracy": 0.2738, + "f1": 0.24618, + "f1_weighted": 0.285497 + }, + { + "accuracy": 0.238968, + "f1": 0.222284, + "f1_weighted": 0.25659 + }, + { + "accuracy": 0.26242, + "f1": 0.246995, + "f1_weighted": 0.26651 + }, + { + "accuracy": 0.2262, + "f1": 0.221629, + "f1_weighted": 0.232575 + }, + { + "accuracy": 0.324591, + "f1": 0.285048, + "f1_weighted": 0.339612 + }, + { + "accuracy": 0.272828, + "f1": 0.255787, + "f1_weighted": 0.290232 + } + ], + "main_score": 0.268845, + "hf_subset": "default", + "languages": [ + "dan-Latn" + ] + } + ] + }, + "evaluation_time": 9.704316139221191, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/DeepSentiPers.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/DeepSentiPers.json new file mode 100644 index 0000000000..c8cd0dd06e --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/DeepSentiPers.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "ee4f09f404051761cfe14d68127532c82de41cb3", + "task_name": "DeepSentiPers", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.350809, + "f1": 0.319652, + "f1_weighted": 0.363762, + "scores_per_experiment": [ + { + "accuracy": 0.299353, + "f1": 0.282298, + "f1_weighted": 0.291967 + }, + { + "accuracy": 0.449838, + "f1": 0.354309, + "f1_weighted": 0.445659 + }, + { + "accuracy": 0.320928, + "f1": 0.310987, + "f1_weighted": 0.351323 + }, + { + "accuracy": 0.271305, + "f1": 0.266331, + "f1_weighted": 0.280683 + }, + { + "accuracy": 0.416397, + "f1": 0.358545, + "f1_weighted": 0.424833 + }, + { + "accuracy": 0.297195, + "f1": 0.294713, + "f1_weighted": 0.321335 + }, + { + "accuracy": 0.324164, + "f1": 0.315995, + "f1_weighted": 0.357687 + }, + { + "accuracy": 0.290183, + "f1": 0.278941, + "f1_weighted": 0.290499 + }, + { + "accuracy": 0.451456, + "f1": 0.390755, + "f1_weighted": 0.465196 + }, + { + "accuracy": 0.387271, + "f1": 0.343651, + "f1_weighted": 0.408434 + } + ], + "main_score": 0.350809, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 3.783827781677246, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/DefinitionClassificationLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/DefinitionClassificationLegalBenchClassification.json new file mode 100644 index 0000000000..90dcb4d0e9 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/DefinitionClassificationLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "DefinitionClassificationLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.912491, + "f1": 0.912486, + "f1_weighted": 0.912508, + "ap": 0.891232, + "ap_weighted": 0.891232, + "scores_per_experiment": [ + { + "accuracy": 0.912491, + "f1": 0.912486, + "f1_weighted": 0.912508, + "ap": 0.891232, + "ap_weighted": 0.891232 + }, + { + "accuracy": 0.912491, + "f1": 0.912486, + "f1_weighted": 0.912508, + "ap": 0.891232, + "ap_weighted": 0.891232 + }, + { + "accuracy": 0.912491, + "f1": 0.912486, + "f1_weighted": 0.912508, + "ap": 0.891232, + "ap_weighted": 0.891232 + }, + { + "accuracy": 0.912491, + "f1": 0.912486, + "f1_weighted": 0.912508, + "ap": 0.891232, + "ap_weighted": 0.891232 + }, + { + "accuracy": 0.912491, + "f1": 0.912486, + "f1_weighted": 0.912508, + "ap": 0.891232, + "ap_weighted": 0.891232 + }, + { + "accuracy": 0.912491, + "f1": 0.912486, + "f1_weighted": 0.912508, + "ap": 0.891232, + "ap_weighted": 0.891232 + }, + { + "accuracy": 0.912491, + "f1": 0.912486, + "f1_weighted": 0.912508, + "ap": 0.891232, + "ap_weighted": 0.891232 + }, + { + "accuracy": 0.912491, + "f1": 0.912486, + "f1_weighted": 0.912508, + "ap": 0.891232, + "ap_weighted": 0.891232 + }, + { + "accuracy": 0.912491, + "f1": 0.912486, + "f1_weighted": 0.912508, + "ap": 0.891232, + "ap_weighted": 0.891232 + }, + { + "accuracy": 0.912491, + "f1": 0.912486, + "f1_weighted": 0.912508, + "ap": 0.891232, + "ap_weighted": 0.891232 + } + ], + "main_score": 0.912491, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.779165744781494, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/DigikalamagClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/DigikalamagClassification.json new file mode 100644 index 0000000000..d1aa603a1e --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/DigikalamagClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "969b335c9f50eda5c384460be4eb2b55505c2c53", + "task_name": "DigikalamagClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.457981, + "f1": 0.317822, + "f1_weighted": 0.48251, + "scores_per_experiment": [ + { + "accuracy": 0.495305, + "f1": 0.356013, + "f1_weighted": 0.525511 + }, + { + "accuracy": 0.46831, + "f1": 0.321488, + "f1_weighted": 0.491177 + }, + { + "accuracy": 0.437793, + "f1": 0.298106, + "f1_weighted": 0.46342 + }, + { + "accuracy": 0.430751, + "f1": 0.308859, + "f1_weighted": 0.463468 + }, + { + "accuracy": 0.510563, + "f1": 0.360022, + "f1_weighted": 0.530982 + }, + { + "accuracy": 0.431925, + "f1": 0.287058, + "f1_weighted": 0.45821 + }, + { + "accuracy": 0.453052, + "f1": 0.308961, + "f1_weighted": 0.482578 + }, + { + "accuracy": 0.476526, + "f1": 0.337335, + "f1_weighted": 0.505814 + }, + { + "accuracy": 0.449531, + "f1": 0.308534, + "f1_weighted": 0.464006 + }, + { + "accuracy": 0.426056, + "f1": 0.291842, + "f1_weighted": 0.439937 + } + ], + "main_score": 0.457981, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 48.27292323112488, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/Diversity1LegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/Diversity1LegalBenchClassification.json new file mode 100644 index 0000000000..911f45b4ff --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/Diversity1LegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "Diversity1LegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.763333, + "f1": 0.432892, + "f1_weighted": 0.660882, + "ap": 0.236667, + "ap_weighted": 0.236667, + "scores_per_experiment": [ + { + "accuracy": 0.763333, + "f1": 0.432892, + "f1_weighted": 0.660882, + "ap": 0.236667, + "ap_weighted": 0.236667 + }, + { + "accuracy": 0.763333, + "f1": 0.432892, + "f1_weighted": 0.660882, + "ap": 0.236667, + "ap_weighted": 0.236667 + }, + { + "accuracy": 0.763333, + "f1": 0.432892, + "f1_weighted": 0.660882, + "ap": 0.236667, + "ap_weighted": 0.236667 + }, + { + "accuracy": 0.763333, + "f1": 0.432892, + "f1_weighted": 0.660882, + "ap": 0.236667, + "ap_weighted": 0.236667 + }, + { + "accuracy": 0.763333, + "f1": 0.432892, + "f1_weighted": 0.660882, + "ap": 0.236667, + "ap_weighted": 0.236667 + }, + { + "accuracy": 0.763333, + "f1": 0.432892, + "f1_weighted": 0.660882, + "ap": 0.236667, + "ap_weighted": 0.236667 + }, + { + "accuracy": 0.763333, + "f1": 0.432892, + "f1_weighted": 0.660882, + "ap": 0.236667, + "ap_weighted": 0.236667 + }, + { + "accuracy": 0.763333, + "f1": 0.432892, + "f1_weighted": 0.660882, + "ap": 0.236667, + "ap_weighted": 0.236667 + }, + { + "accuracy": 0.763333, + "f1": 0.432892, + "f1_weighted": 0.660882, + "ap": 0.236667, + "ap_weighted": 0.236667 + }, + { + "accuracy": 0.763333, + "f1": 0.432892, + "f1_weighted": 0.660882, + "ap": 0.236667, + "ap_weighted": 0.236667 + } + ], + "main_score": 0.763333, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.7686808109283447, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/Diversity2LegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/Diversity2LegalBenchClassification.json new file mode 100644 index 0000000000..e2ac6af5ad --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/Diversity2LegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "Diversity2LegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.746667, + "f1": 0.427481, + "f1_weighted": 0.638372, + "ap": 0.253333, + "ap_weighted": 0.253333, + "scores_per_experiment": [ + { + "accuracy": 0.746667, + "f1": 0.427481, + "f1_weighted": 0.638372, + "ap": 0.253333, + "ap_weighted": 0.253333 + }, + { + "accuracy": 0.746667, + "f1": 0.427481, + "f1_weighted": 0.638372, + "ap": 0.253333, + "ap_weighted": 0.253333 + }, + { + "accuracy": 0.746667, + "f1": 0.427481, + "f1_weighted": 0.638372, + "ap": 0.253333, + "ap_weighted": 0.253333 + }, + { + "accuracy": 0.746667, + "f1": 0.427481, + "f1_weighted": 0.638372, + "ap": 0.253333, + "ap_weighted": 0.253333 + }, + { + "accuracy": 0.746667, + "f1": 0.427481, + "f1_weighted": 0.638372, + "ap": 0.253333, + "ap_weighted": 0.253333 + }, + { + "accuracy": 0.746667, + "f1": 0.427481, + "f1_weighted": 0.638372, + "ap": 0.253333, + "ap_weighted": 0.253333 + }, + { + "accuracy": 0.746667, + "f1": 0.427481, + "f1_weighted": 0.638372, + "ap": 0.253333, + "ap_weighted": 0.253333 + }, + { + "accuracy": 0.746667, + "f1": 0.427481, + "f1_weighted": 0.638372, + "ap": 0.253333, + "ap_weighted": 0.253333 + }, + { + "accuracy": 0.746667, + "f1": 0.427481, + "f1_weighted": 0.638372, + "ap": 0.253333, + "ap_weighted": 0.253333 + }, + { + "accuracy": 0.746667, + "f1": 0.427481, + "f1_weighted": 0.638372, + "ap": 0.253333, + "ap_weighted": 0.253333 + } + ], + "main_score": 0.746667, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.7874889373779297, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/Diversity3LegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/Diversity3LegalBenchClassification.json new file mode 100644 index 0000000000..213cc79b33 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/Diversity3LegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "Diversity3LegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.586667, + "f1": 0.369748, + "f1_weighted": 0.433838, + "ap": 0.586667, + "ap_weighted": 0.586667, + "scores_per_experiment": [ + { + "accuracy": 0.586667, + "f1": 0.369748, + "f1_weighted": 0.433838, + "ap": 0.586667, + "ap_weighted": 0.586667 + }, + { + "accuracy": 0.586667, + "f1": 0.369748, + "f1_weighted": 0.433838, + "ap": 0.586667, + "ap_weighted": 0.586667 + }, + { + "accuracy": 0.586667, + "f1": 0.369748, + "f1_weighted": 0.433838, + "ap": 0.586667, + "ap_weighted": 0.586667 + }, + { + "accuracy": 0.586667, + "f1": 0.369748, + "f1_weighted": 0.433838, + "ap": 0.586667, + "ap_weighted": 0.586667 + }, + { + "accuracy": 0.586667, + "f1": 0.369748, + "f1_weighted": 0.433838, + "ap": 0.586667, + "ap_weighted": 0.586667 + }, + { + "accuracy": 0.586667, + "f1": 0.369748, + "f1_weighted": 0.433838, + "ap": 0.586667, + "ap_weighted": 0.586667 + }, + { + "accuracy": 0.586667, + "f1": 0.369748, + "f1_weighted": 0.433838, + "ap": 0.586667, + "ap_weighted": 0.586667 + }, + { + "accuracy": 0.586667, + "f1": 0.369748, + "f1_weighted": 0.433838, + "ap": 0.586667, + "ap_weighted": 0.586667 + }, + { + "accuracy": 0.586667, + "f1": 0.369748, + "f1_weighted": 0.433838, + "ap": 0.586667, + "ap_weighted": 0.586667 + }, + { + "accuracy": 0.586667, + "f1": 0.369748, + "f1_weighted": 0.433838, + "ap": 0.586667, + "ap_weighted": 0.586667 + } + ], + "main_score": 0.586667, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.8595507144927979, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/Diversity4LegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/Diversity4LegalBenchClassification.json new file mode 100644 index 0000000000..795c693cbc --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/Diversity4LegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "Diversity4LegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.533333, + "f1": 0.347826, + "f1_weighted": 0.371014, + "ap": 0.533333, + "ap_weighted": 0.533333, + "scores_per_experiment": [ + { + "accuracy": 0.533333, + "f1": 0.347826, + "f1_weighted": 0.371014, + "ap": 0.533333, + "ap_weighted": 0.533333 + }, + { + "accuracy": 0.533333, + "f1": 0.347826, + "f1_weighted": 0.371014, + "ap": 0.533333, + "ap_weighted": 0.533333 + }, + { + "accuracy": 0.533333, + "f1": 0.347826, + "f1_weighted": 0.371014, + "ap": 0.533333, + "ap_weighted": 0.533333 + }, + { + "accuracy": 0.533333, + "f1": 0.347826, + "f1_weighted": 0.371014, + "ap": 0.533333, + "ap_weighted": 0.533333 + }, + { + "accuracy": 0.533333, + "f1": 0.347826, + "f1_weighted": 0.371014, + "ap": 0.533333, + "ap_weighted": 0.533333 + }, + { + "accuracy": 0.533333, + "f1": 0.347826, + "f1_weighted": 0.371014, + "ap": 0.533333, + "ap_weighted": 0.533333 + }, + { + "accuracy": 0.533333, + "f1": 0.347826, + "f1_weighted": 0.371014, + "ap": 0.533333, + "ap_weighted": 0.533333 + }, + { + "accuracy": 0.533333, + "f1": 0.347826, + "f1_weighted": 0.371014, + "ap": 0.533333, + "ap_weighted": 0.533333 + }, + { + "accuracy": 0.533333, + "f1": 0.347826, + "f1_weighted": 0.371014, + "ap": 0.533333, + "ap_weighted": 0.533333 + }, + { + "accuracy": 0.533333, + "f1": 0.347826, + "f1_weighted": 0.371014, + "ap": 0.533333, + "ap_weighted": 0.533333 + } + ], + "main_score": 0.533333, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.8241691589355469, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/Diversity5LegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/Diversity5LegalBenchClassification.json new file mode 100644 index 0000000000..d9c5410220 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/Diversity5LegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "Diversity5LegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.573333, + "f1": 0.364407, + "f1_weighted": 0.417853, + "ap": 0.573333, + "ap_weighted": 0.573333, + "scores_per_experiment": [ + { + "accuracy": 0.573333, + "f1": 0.364407, + "f1_weighted": 0.417853, + "ap": 0.573333, + "ap_weighted": 0.573333 + }, + { + "accuracy": 0.573333, + "f1": 0.364407, + "f1_weighted": 0.417853, + "ap": 0.573333, + "ap_weighted": 0.573333 + }, + { + "accuracy": 0.573333, + "f1": 0.364407, + "f1_weighted": 0.417853, + "ap": 0.573333, + "ap_weighted": 0.573333 + }, + { + "accuracy": 0.573333, + "f1": 0.364407, + "f1_weighted": 0.417853, + "ap": 0.573333, + "ap_weighted": 0.573333 + }, + { + "accuracy": 0.573333, + "f1": 0.364407, + "f1_weighted": 0.417853, + "ap": 0.573333, + "ap_weighted": 0.573333 + }, + { + "accuracy": 0.573333, + "f1": 0.364407, + "f1_weighted": 0.417853, + "ap": 0.573333, + "ap_weighted": 0.573333 + }, + { + "accuracy": 0.573333, + "f1": 0.364407, + "f1_weighted": 0.417853, + "ap": 0.573333, + "ap_weighted": 0.573333 + }, + { + "accuracy": 0.573333, + "f1": 0.364407, + "f1_weighted": 0.417853, + "ap": 0.573333, + "ap_weighted": 0.573333 + }, + { + "accuracy": 0.573333, + "f1": 0.364407, + "f1_weighted": 0.417853, + "ap": 0.573333, + "ap_weighted": 0.573333 + }, + { + "accuracy": 0.573333, + "f1": 0.364407, + "f1_weighted": 0.417853, + "ap": 0.573333, + "ap_weighted": 0.573333 + } + ], + "main_score": 0.573333, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.8662729263305664, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/Diversity6LegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/Diversity6LegalBenchClassification.json new file mode 100644 index 0000000000..45b7002298 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/Diversity6LegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "Diversity6LegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.513333, + "f1": 0.512792, + "f1_weighted": 0.513766, + "ap": 0.536728, + "ap_weighted": 0.536728, + "scores_per_experiment": [ + { + "accuracy": 0.513333, + "f1": 0.512792, + "f1_weighted": 0.513766, + "ap": 0.536728, + "ap_weighted": 0.536728 + }, + { + "accuracy": 0.513333, + "f1": 0.512792, + "f1_weighted": 0.513766, + "ap": 0.536728, + "ap_weighted": 0.536728 + }, + { + "accuracy": 0.513333, + "f1": 0.512792, + "f1_weighted": 0.513766, + "ap": 0.536728, + "ap_weighted": 0.536728 + }, + { + "accuracy": 0.513333, + "f1": 0.512792, + "f1_weighted": 0.513766, + "ap": 0.536728, + "ap_weighted": 0.536728 + }, + { + "accuracy": 0.513333, + "f1": 0.512792, + "f1_weighted": 0.513766, + "ap": 0.536728, + "ap_weighted": 0.536728 + }, + { + "accuracy": 0.513333, + "f1": 0.512792, + "f1_weighted": 0.513766, + "ap": 0.536728, + "ap_weighted": 0.536728 + }, + { + "accuracy": 0.513333, + "f1": 0.512792, + "f1_weighted": 0.513766, + "ap": 0.536728, + "ap_weighted": 0.536728 + }, + { + "accuracy": 0.513333, + "f1": 0.512792, + "f1_weighted": 0.513766, + "ap": 0.536728, + "ap_weighted": 0.536728 + }, + { + "accuracy": 0.513333, + "f1": 0.512792, + "f1_weighted": 0.513766, + "ap": 0.536728, + "ap_weighted": 0.536728 + }, + { + "accuracy": 0.513333, + "f1": 0.512792, + "f1_weighted": 0.513766, + "ap": 0.536728, + "ap_weighted": 0.536728 + } + ], + "main_score": 0.513333, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.9909365177154541, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/DutchBookReviewSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/DutchBookReviewSentimentClassification.json new file mode 100644 index 0000000000..a48d35f03e --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/DutchBookReviewSentimentClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "3f756ab4572e071eb53e887ab629f19fa747d39e", + "task_name": "DutchBookReviewSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.516862, + "f1": 0.509559, + "f1_weighted": 0.509559, + "ap": 0.509408, + "ap_weighted": 0.509408, + "scores_per_experiment": [ + { + "accuracy": 0.531475, + "f1": 0.522434, + "f1_weighted": 0.522434, + "ap": 0.516514, + "ap_weighted": 0.516514 + }, + { + "accuracy": 0.477068, + "f1": 0.459677, + "f1_weighted": 0.459677, + "ap": 0.488921, + "ap_weighted": 0.488921 + }, + { + "accuracy": 0.547212, + "f1": 0.545225, + "f1_weighted": 0.545225, + "ap": 0.525575, + "ap_weighted": 0.525575 + }, + { + "accuracy": 0.482464, + "f1": 0.471731, + "f1_weighted": 0.471731, + "ap": 0.491471, + "ap_weighted": 0.491471 + }, + { + "accuracy": 0.53732, + "f1": 0.534883, + "f1_weighted": 0.534883, + "ap": 0.520289, + "ap_weighted": 0.520289 + }, + { + "accuracy": 0.469874, + "f1": 0.464966, + "f1_weighted": 0.464966, + "ap": 0.485699, + "ap_weighted": 0.485699 + }, + { + "accuracy": 0.540018, + "f1": 0.536454, + "f1_weighted": 0.536454, + "ap": 0.521951, + "ap_weighted": 0.521951 + }, + { + "accuracy": 0.513939, + "f1": 0.50736, + "f1_weighted": 0.50736, + "ap": 0.507127, + "ap_weighted": 0.507127 + }, + { + "accuracy": 0.535971, + "f1": 0.530224, + "f1_weighted": 0.530224, + "ap": 0.519045, + "ap_weighted": 0.519045 + }, + { + "accuracy": 0.533273, + "f1": 0.522635, + "f1_weighted": 0.522635, + "ap": 0.517489, + "ap_weighted": 0.517489 + } + ], + "main_score": 0.516862, + "hf_subset": "default", + "languages": [ + "nld-Latn" + ] + } + ] + }, + "evaluation_time": 19.16693687438965, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/EmotionClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/EmotionClassification.json index adcf0267e0..b3dc6d09d0 100644 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/EmotionClassification.json +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/EmotionClassification.json @@ -1,13 +1,73 @@ { - "test": { - "accuracy": 0.422, - "accuracy_stderr": 0.02340939982143925, - "evaluation_time": 24.36, - "f1": 0.38911819699966416, - "f1_stderr": 0.013880285697045311, - "main_score": 0.422 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "EmotionClassification", - "dataset_revision": "829147f8f75a25f005913200eb5ed41fae320aa1" + "dataset_revision": "4f58c6b202a23cf9a4da393831edf4f9183cad37", + "task_name": "EmotionClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.45445, + "f1": 0.412797, + "f1_weighted": 0.477425, + "scores_per_experiment": [ + { + "accuracy": 0.478, + "f1": 0.430477, + "f1_weighted": 0.502964 + }, + { + "accuracy": 0.4315, + "f1": 0.400387, + "f1_weighted": 0.449821 + }, + { + "accuracy": 0.457, + "f1": 0.411731, + "f1_weighted": 0.479735 + }, + { + "accuracy": 0.4335, + "f1": 0.39866, + "f1_weighted": 0.462326 + }, + { + "accuracy": 0.4675, + "f1": 0.430657, + "f1_weighted": 0.489519 + }, + { + "accuracy": 0.4645, + "f1": 0.415549, + "f1_weighted": 0.486253 + }, + { + "accuracy": 0.4605, + "f1": 0.406992, + "f1_weighted": 0.484194 + }, + { + "accuracy": 0.429, + "f1": 0.396873, + "f1_weighted": 0.448628 + }, + { + "accuracy": 0.46, + "f1": 0.419229, + "f1_weighted": 0.485097 + }, + { + "accuracy": 0.463, + "f1": 0.417413, + "f1_weighted": 0.485714 + } + ], + "main_score": 0.45445, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.068608283996582, + "kg_co2_emissions": null } \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/EstonianValenceClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/EstonianValenceClassification.json new file mode 100644 index 0000000000..dea4fd0b06 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/EstonianValenceClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "9157397f05a127b3ac93b93dd88abf1bdf710c22", + "task_name": "EstonianValenceClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.29401, + "f1": 0.283836, + "f1_weighted": 0.301906, + "scores_per_experiment": [ + { + "accuracy": 0.297066, + "f1": 0.282935, + "f1_weighted": 0.31239 + }, + { + "accuracy": 0.288509, + "f1": 0.280337, + "f1_weighted": 0.300069 + }, + { + "accuracy": 0.282396, + "f1": 0.275366, + "f1_weighted": 0.291539 + }, + { + "accuracy": 0.300733, + "f1": 0.26934, + "f1_weighted": 0.310972 + }, + { + "accuracy": 0.257946, + "f1": 0.25952, + "f1_weighted": 0.251004 + }, + { + "accuracy": 0.304401, + "f1": 0.29796, + "f1_weighted": 0.312744 + }, + { + "accuracy": 0.270171, + "f1": 0.269505, + "f1_weighted": 0.269918 + }, + { + "accuracy": 0.323961, + "f1": 0.313079, + "f1_weighted": 0.330017 + }, + { + "accuracy": 0.305623, + "f1": 0.294012, + "f1_weighted": 0.31917 + }, + { + "accuracy": 0.309291, + "f1": 0.296302, + "f1_weighted": 0.321234 + } + ], + "main_score": 0.29401, + "hf_subset": "default", + "languages": [ + "est-Latn" + ] + } + ] + }, + "evaluation_time": 4.949531078338623, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/FEVER.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/FEVER.json deleted file mode 100644 index b74bc665d8..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/FEVER.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "mteb_version": "0.0.2", - "test": { - "evaluation_time": 20370.0, - "map_at_1": 0.50058, - "map_at_10": 0.62452, - "map_at_100": 0.62925, - "map_at_1000": 0.62942, - "map_at_3": 0.59885, - "map_at_5": 0.6157, - "ndcg_at_1": 0.5399, - "ndcg_at_10": 0.68932, - "ndcg_at_100": 0.70956, - "ndcg_at_1000": 0.71321, - "ndcg_at_3": 0.64005, - "ndcg_at_5": 0.66906, - "precision_at_1": 0.5399, - "precision_at_10": 0.09272, - "precision_at_100": 0.01036, - "precision_at_1000": 0.00108, - "precision_at_3": 0.25933, - "precision_at_5": 0.17144, - "recall_at_1": 0.50058, - "recall_at_10": 0.84868, - "recall_at_100": 0.93618, - "recall_at_1000": 0.96163, - "recall_at_3": 0.71655, - "recall_at_5": 0.78757 - }, - "mteb_dataset_name": "FEVER", - "dataset_revision": "1429cf27e393599b8b359b9b72c666f96b2525f9" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/FiQA2018.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/FiQA2018.json deleted file mode 100644 index e27e88e5c0..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/FiQA2018.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "test": { - "evaluation_time": 326.1, - "map_at_1": 0.16, - "map_at_10": 0.27545, - "map_at_100": 0.29213, - "map_at_1000": 0.29382, - "map_at_3": 0.2376, - "map_at_5": 0.25638, - "ndcg_at_1": 0.33488, - "ndcg_at_10": 0.35147, - "ndcg_at_100": 0.41804, - "ndcg_at_1000": 0.44824, - "ndcg_at_3": 0.31489, - "ndcg_at_5": 0.32377, - "precision_at_1": 0.33488, - "precision_at_10": 0.10046, - "precision_at_100": 0.01699, - "precision_at_1000": 0.00224, - "precision_at_3": 0.21399, - "precision_at_5": 0.15679, - "recall_at_1": 0.16, - "recall_at_10": 0.42647, - "recall_at_100": 0.67993, - "recall_at_1000": 0.85966, - "recall_at_3": 0.29087, - "recall_at_5": 0.34244 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "FiQA2018", - "dataset_revision": "41b686a7f28c59bcaaa5791efd47c67c8ebe28be" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/FilipinoShopeeReviewsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/FilipinoShopeeReviewsClassification.json new file mode 100644 index 0000000000..91cd85525e --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/FilipinoShopeeReviewsClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "d096f402fdc76886458c0cfb5dedc829bea2b935", + "task_name": "FilipinoShopeeReviewsClassification", + "mteb_version": "1.34.14", + "scores": { + "validation": [ + { + "accuracy": 0.277295, + "f1": 0.274044, + "f1_weighted": 0.274048, + "scores_per_experiment": [ + { + "accuracy": 0.286621, + "f1": 0.286941, + "f1_weighted": 0.286935 + }, + { + "accuracy": 0.278809, + "f1": 0.273151, + "f1_weighted": 0.273166 + }, + { + "accuracy": 0.271973, + "f1": 0.270545, + "f1_weighted": 0.270527 + }, + { + "accuracy": 0.306152, + "f1": 0.307712, + "f1_weighted": 0.307722 + }, + { + "accuracy": 0.237305, + "f1": 0.23314, + "f1_weighted": 0.233147 + }, + { + "accuracy": 0.27832, + "f1": 0.268314, + "f1_weighted": 0.268327 + }, + { + "accuracy": 0.23584, + "f1": 0.237872, + "f1_weighted": 0.237845 + }, + { + "accuracy": 0.298828, + "f1": 0.296278, + "f1_weighted": 0.296293 + }, + { + "accuracy": 0.296387, + "f1": 0.293076, + "f1_weighted": 0.293078 + }, + { + "accuracy": 0.282715, + "f1": 0.27341, + "f1_weighted": 0.273442 + } + ], + "main_score": 0.277295, + "hf_subset": "default", + "languages": [ + "fil-Latn" + ] + } + ], + "test": [ + { + "accuracy": 0.276172, + "f1": 0.272648, + "f1_weighted": 0.272651, + "scores_per_experiment": [ + { + "accuracy": 0.269043, + "f1": 0.268645, + "f1_weighted": 0.268631 + }, + { + "accuracy": 0.289551, + "f1": 0.284254, + "f1_weighted": 0.284277 + }, + { + "accuracy": 0.254883, + "f1": 0.25322, + "f1_weighted": 0.253197 + }, + { + "accuracy": 0.302246, + "f1": 0.306318, + "f1_weighted": 0.306303 + }, + { + "accuracy": 0.269531, + "f1": 0.266646, + "f1_weighted": 0.266638 + }, + { + "accuracy": 0.251953, + "f1": 0.237969, + "f1_weighted": 0.237974 + }, + { + "accuracy": 0.240234, + "f1": 0.239836, + "f1_weighted": 0.239846 + }, + { + "accuracy": 0.305664, + "f1": 0.302558, + "f1_weighted": 0.302571 + }, + { + "accuracy": 0.290527, + "f1": 0.286397, + "f1_weighted": 0.28641 + }, + { + "accuracy": 0.288086, + "f1": 0.280636, + "f1_weighted": 0.280662 + } + ], + "main_score": 0.276172, + "hf_subset": "default", + "languages": [ + "fil-Latn" + ] + } + ] + }, + "evaluation_time": 20.292845487594604, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/FinToxicityClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/FinToxicityClassification.json new file mode 100644 index 0000000000..4c3c48fb20 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/FinToxicityClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "6e7340e6be87124f319e25290778760c14df64d3", + "task_name": "FinToxicityClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.48877, + "f1": 0.410755, + "f1_weighted": 0.577817, + "ap": 0.113181, + "ap_weighted": 0.113181, + "scores_per_experiment": [ + { + "accuracy": 0.462402, + "f1": 0.378882, + "f1_weighted": 0.563272, + "ap": 0.092056, + "ap_weighted": 0.092056 + }, + { + "accuracy": 0.569824, + "f1": 0.468836, + "f1_weighted": 0.656337, + "ap": 0.128582, + "ap_weighted": 0.128582 + }, + { + "accuracy": 0.317871, + "f1": 0.298887, + "f1_weighted": 0.392286, + "ap": 0.101935, + "ap_weighted": 0.101935 + }, + { + "accuracy": 0.613281, + "f1": 0.496598, + "f1_weighted": 0.692806, + "ap": 0.137096, + "ap_weighted": 0.137096 + }, + { + "accuracy": 0.559082, + "f1": 0.450317, + "f1_weighted": 0.648267, + "ap": 0.112608, + "ap_weighted": 0.112608 + }, + { + "accuracy": 0.532715, + "f1": 0.445301, + "f1_weighted": 0.623569, + "ap": 0.122615, + "ap_weighted": 0.122615 + }, + { + "accuracy": 0.505859, + "f1": 0.415828, + "f1_weighted": 0.601489, + "ap": 0.104356, + "ap_weighted": 0.104356 + }, + { + "accuracy": 0.533691, + "f1": 0.448981, + "f1_weighted": 0.623888, + "ap": 0.1271, + "ap_weighted": 0.1271 + }, + { + "accuracy": 0.356445, + "f1": 0.322253, + "f1_weighted": 0.445493, + "ap": 0.096391, + "ap_weighted": 0.096391 + }, + { + "accuracy": 0.436523, + "f1": 0.381668, + "f1_weighted": 0.530767, + "ap": 0.109069, + "ap_weighted": 0.109069 + } + ], + "main_score": 0.410755, + "hf_subset": "default", + "languages": [ + "fin-Latn" + ] + } + ] + }, + "evaluation_time": 9.099807500839233, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/FinancialPhrasebankClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/FinancialPhrasebankClassification.json new file mode 100644 index 0000000000..409b309025 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/FinancialPhrasebankClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "1484d06fe7af23030c7c977b12556108d1f67039", + "task_name": "FinancialPhrasebankClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.672747, + "f1": 0.585938, + "f1_weighted": 0.680434, + "scores_per_experiment": [ + { + "accuracy": 0.750883, + "f1": 0.670479, + "f1_weighted": 0.750276 + }, + { + "accuracy": 0.692138, + "f1": 0.583879, + "f1_weighted": 0.68801 + }, + { + "accuracy": 0.585689, + "f1": 0.502953, + "f1_weighted": 0.608519 + }, + { + "accuracy": 0.638693, + "f1": 0.53833, + "f1_weighted": 0.643489 + }, + { + "accuracy": 0.718198, + "f1": 0.617096, + "f1_weighted": 0.713894 + }, + { + "accuracy": 0.623233, + "f1": 0.562607, + "f1_weighted": 0.648867 + }, + { + "accuracy": 0.669611, + "f1": 0.608254, + "f1_weighted": 0.676166 + }, + { + "accuracy": 0.653269, + "f1": 0.569527, + "f1_weighted": 0.670011 + }, + { + "accuracy": 0.708039, + "f1": 0.603206, + "f1_weighted": 0.705037 + }, + { + "accuracy": 0.687721, + "f1": 0.603053, + "f1_weighted": 0.700069 + } + ], + "main_score": 0.672747, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.7945451736450195, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/FrenchBookReviews.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/FrenchBookReviews.json new file mode 100644 index 0000000000..68ec841d2b --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/FrenchBookReviews.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "534725e03fec6f560dbe8166e8ae3825314a6290", + "task_name": "FrenchBookReviews", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.350781, + "f1": 0.301095, + "f1_weighted": 0.39311, + "scores_per_experiment": [ + { + "accuracy": 0.328613, + "f1": 0.287632, + "f1_weighted": 0.378834 + }, + { + "accuracy": 0.32373, + "f1": 0.292879, + "f1_weighted": 0.37226 + }, + { + "accuracy": 0.384766, + "f1": 0.326154, + "f1_weighted": 0.427479 + }, + { + "accuracy": 0.297852, + "f1": 0.276871, + "f1_weighted": 0.326395 + }, + { + "accuracy": 0.400391, + "f1": 0.330388, + "f1_weighted": 0.443067 + }, + { + "accuracy": 0.408203, + "f1": 0.331759, + "f1_weighted": 0.448833 + }, + { + "accuracy": 0.396973, + "f1": 0.315153, + "f1_weighted": 0.439923 + }, + { + "accuracy": 0.376465, + "f1": 0.303699, + "f1_weighted": 0.422612 + }, + { + "accuracy": 0.278809, + "f1": 0.26821, + "f1_weighted": 0.312227 + }, + { + "accuracy": 0.312012, + "f1": 0.27821, + "f1_weighted": 0.359475 + } + ], + "main_score": 0.350781, + "hf_subset": "default", + "languages": [ + "fra-Latn" + ] + } + ] + }, + "evaluation_time": 6.226322650909424, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/FrenkEnClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/FrenkEnClassification.json new file mode 100644 index 0000000000..3436e4d9ba --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/FrenkEnClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "52483dba0ff23291271ee9249839865e3c3e7e50", + "task_name": "FrenkEnClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.599522, + "f1": 0.586361, + "f1_weighted": 0.595856, + "ap": 0.44932, + "ap_weighted": 0.44932, + "scores_per_experiment": [ + { + "accuracy": 0.575837, + "f1": 0.57576, + "f1_weighted": 0.574391, + "ap": 0.446187, + "ap_weighted": 0.446187 + }, + { + "accuracy": 0.528031, + "f1": 0.495891, + "f1_weighted": 0.526372, + "ap": 0.378414, + "ap_weighted": 0.378414 + }, + { + "accuracy": 0.637549, + "f1": 0.564758, + "f1_weighted": 0.607381, + "ap": 0.428962, + "ap_weighted": 0.428962 + }, + { + "accuracy": 0.652325, + "f1": 0.646435, + "f1_weighted": 0.657363, + "ap": 0.485496, + "ap_weighted": 0.485496 + }, + { + "accuracy": 0.588874, + "f1": 0.588566, + "f1_weighted": 0.585868, + "ap": 0.457967, + "ap_weighted": 0.457967 + }, + { + "accuracy": 0.627553, + "f1": 0.626876, + "f1_weighted": 0.630682, + "ap": 0.477572, + "ap_weighted": 0.477572 + }, + { + "accuracy": 0.64146, + "f1": 0.629345, + "f1_weighted": 0.645392, + "ap": 0.467785, + "ap_weighted": 0.467785 + }, + { + "accuracy": 0.592786, + "f1": 0.592578, + "f1_weighted": 0.590373, + "ap": 0.460177, + "ap_weighted": 0.460177 + }, + { + "accuracy": 0.558453, + "f1": 0.553878, + "f1_weighted": 0.543061, + "ap": 0.447265, + "ap_weighted": 0.447265 + }, + { + "accuracy": 0.592351, + "f1": 0.589523, + "f1_weighted": 0.597682, + "ap": 0.443373, + "ap_weighted": 0.443373 + } + ], + "main_score": 0.599522, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.9862966537475586, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/FrenkHrClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/FrenkHrClassification.json new file mode 100644 index 0000000000..bde81fb86c --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/FrenkHrClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "e7fc9f3d8d6c5640a26679d8a50b1666b02cc41f", + "task_name": "FrenkHrClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.5689, + "f1": 0.556831, + "f1_weighted": 0.560899, + "ap": 0.60121, + "ap_weighted": 0.60121, + "scores_per_experiment": [ + { + "accuracy": 0.559226, + "f1": 0.556855, + "f1_weighted": 0.560878, + "ap": 0.593895, + "ap_weighted": 0.593895 + }, + { + "accuracy": 0.588957, + "f1": 0.579608, + "f1_weighted": 0.587389, + "ap": 0.60576, + "ap_weighted": 0.60576 + }, + { + "accuracy": 0.611609, + "f1": 0.58832, + "f1_weighted": 0.600473, + "ap": 0.61226, + "ap_weighted": 0.61226 + }, + { + "accuracy": 0.596508, + "f1": 0.585058, + "f1_weighted": 0.593613, + "ap": 0.608999, + "ap_weighted": 0.608999 + }, + { + "accuracy": 0.560642, + "f1": 0.555404, + "f1_weighted": 0.549414, + "ap": 0.612004, + "ap_weighted": 0.612004 + }, + { + "accuracy": 0.602643, + "f1": 0.560948, + "f1_weighted": 0.577741, + "ap": 0.601205, + "ap_weighted": 0.601205 + }, + { + "accuracy": 0.396885, + "f1": 0.385615, + "f1_weighted": 0.375287, + "ap": 0.531841, + "ap_weighted": 0.531841 + }, + { + "accuracy": 0.628598, + "f1": 0.618304, + "f1_weighted": 0.626084, + "ap": 0.629941, + "ap_weighted": 0.629941 + }, + { + "accuracy": 0.591317, + "f1": 0.588379, + "f1_weighted": 0.592695, + "ap": 0.612999, + "ap_weighted": 0.612999 + }, + { + "accuracy": 0.552619, + "f1": 0.549823, + "f1_weighted": 0.54542, + "ap": 0.603196, + "ap_weighted": 0.603196 + } + ], + "main_score": 0.5689, + "hf_subset": "default", + "languages": [ + "hrv-Latn" + ] + } + ] + }, + "evaluation_time": 3.7134809494018555, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/FrenkSlClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/FrenkSlClassification.json new file mode 100644 index 0000000000..48cb668fa0 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/FrenkSlClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "37c8b42c63d4eb75f549679158a85eb5bd984caa", + "task_name": "FrenkSlClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.568457, + "f1": 0.563259, + "f1_weighted": 0.563748, + "ap": 0.550423, + "ap_weighted": 0.550423, + "scores_per_experiment": [ + { + "accuracy": 0.555664, + "f1": 0.553637, + "f1_weighted": 0.554342, + "ap": 0.541497, + "ap_weighted": 0.541497 + }, + { + "accuracy": 0.609375, + "f1": 0.60375, + "f1_weighted": 0.604857, + "ap": 0.574331, + "ap_weighted": 0.574331 + }, + { + "accuracy": 0.58252, + "f1": 0.575301, + "f1_weighted": 0.576599, + "ap": 0.556609, + "ap_weighted": 0.556609 + }, + { + "accuracy": 0.605957, + "f1": 0.603319, + "f1_weighted": 0.604078, + "ap": 0.57321, + "ap_weighted": 0.57321 + }, + { + "accuracy": 0.549316, + "f1": 0.548385, + "f1_weighted": 0.547904, + "ap": 0.539877, + "ap_weighted": 0.539877 + }, + { + "accuracy": 0.592285, + "f1": 0.590288, + "f1_weighted": 0.589617, + "ap": 0.569176, + "ap_weighted": 0.569176 + }, + { + "accuracy": 0.508301, + "f1": 0.508063, + "f1_weighted": 0.50781, + "ap": 0.516351, + "ap_weighted": 0.516351 + }, + { + "accuracy": 0.557129, + "f1": 0.556984, + "f1_weighted": 0.556797, + "ap": 0.544107, + "ap_weighted": 0.544107 + }, + { + "accuracy": 0.561035, + "f1": 0.560859, + "f1_weighted": 0.560653, + "ap": 0.54659, + "ap_weighted": 0.54659 + }, + { + "accuracy": 0.562988, + "f1": 0.531999, + "f1_weighted": 0.534822, + "ap": 0.542481, + "ap_weighted": 0.542481 + } + ], + "main_score": 0.568457, + "hf_subset": "default", + "languages": [ + "slv-Latn" + ] + } + ] + }, + "evaluation_time": 4.501609802246094, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/FunctionOfDecisionSectionLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/FunctionOfDecisionSectionLegalBenchClassification.json new file mode 100644 index 0000000000..eda446ef3f --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/FunctionOfDecisionSectionLegalBenchClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "FunctionOfDecisionSectionLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.220708, + "f1": 0.190366, + "f1_weighted": 0.19176, + "scores_per_experiment": [ + { + "accuracy": 0.220708, + "f1": 0.190366, + "f1_weighted": 0.19176 + }, + { + "accuracy": 0.220708, + "f1": 0.190366, + "f1_weighted": 0.19176 + }, + { + "accuracy": 0.220708, + "f1": 0.190366, + "f1_weighted": 0.19176 + }, + { + "accuracy": 0.220708, + "f1": 0.190366, + "f1_weighted": 0.19176 + }, + { + "accuracy": 0.220708, + "f1": 0.190366, + "f1_weighted": 0.19176 + }, + { + "accuracy": 0.220708, + "f1": 0.190366, + "f1_weighted": 0.19176 + }, + { + "accuracy": 0.220708, + "f1": 0.190366, + "f1_weighted": 0.19176 + }, + { + "accuracy": 0.220708, + "f1": 0.190366, + "f1_weighted": 0.19176 + }, + { + "accuracy": 0.220708, + "f1": 0.190366, + "f1_weighted": 0.19176 + }, + { + "accuracy": 0.220708, + "f1": 0.190366, + "f1_weighted": 0.19176 + } + ], + "main_score": 0.220708, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.8300673961639404, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/GeoreviewClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/GeoreviewClassification.json new file mode 100644 index 0000000000..b092d10503 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/GeoreviewClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "3765c0d1de6b7d264bc459433c45e5a75513839c", + "task_name": "GeoreviewClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.268164, + "f1": 0.257715, + "f1_weighted": 0.257729, + "scores_per_experiment": [ + { + "accuracy": 0.243652, + "f1": 0.245236, + "f1_weighted": 0.245242 + }, + { + "accuracy": 0.296875, + "f1": 0.289492, + "f1_weighted": 0.289476 + }, + { + "accuracy": 0.292969, + "f1": 0.275874, + "f1_weighted": 0.27591 + }, + { + "accuracy": 0.242188, + "f1": 0.237392, + "f1_weighted": 0.23745 + }, + { + "accuracy": 0.260254, + "f1": 0.250523, + "f1_weighted": 0.250536 + }, + { + "accuracy": 0.282715, + "f1": 0.269505, + "f1_weighted": 0.269514 + }, + { + "accuracy": 0.251953, + "f1": 0.244692, + "f1_weighted": 0.244732 + }, + { + "accuracy": 0.251953, + "f1": 0.231647, + "f1_weighted": 0.231658 + }, + { + "accuracy": 0.262207, + "f1": 0.25013, + "f1_weighted": 0.250113 + }, + { + "accuracy": 0.296875, + "f1": 0.282657, + "f1_weighted": 0.282655 + } + ], + "main_score": 0.268164, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 14.619102239608765, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/GermanPoliticiansTwitterSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/GermanPoliticiansTwitterSentimentClassification.json new file mode 100644 index 0000000000..33853c201e --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/GermanPoliticiansTwitterSentimentClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "65343b17f5a76227ab2e15b9424dfab6466ffcb1", + "task_name": "GermanPoliticiansTwitterSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.389916, + "f1": 0.388609, + "f1_weighted": 0.385467, + "scores_per_experiment": [ + { + "accuracy": 0.411765, + "f1": 0.399056, + "f1_weighted": 0.407058 + }, + { + "accuracy": 0.394958, + "f1": 0.396244, + "f1_weighted": 0.383468 + }, + { + "accuracy": 0.403361, + "f1": 0.398922, + "f1_weighted": 0.403611 + }, + { + "accuracy": 0.453782, + "f1": 0.452119, + "f1_weighted": 0.454315 + }, + { + "accuracy": 0.364146, + "f1": 0.366546, + "f1_weighted": 0.361832 + }, + { + "accuracy": 0.411765, + "f1": 0.412432, + "f1_weighted": 0.404302 + }, + { + "accuracy": 0.355742, + "f1": 0.350441, + "f1_weighted": 0.337275 + }, + { + "accuracy": 0.35014, + "f1": 0.356439, + "f1_weighted": 0.348963 + }, + { + "accuracy": 0.338936, + "f1": 0.339475, + "f1_weighted": 0.339236 + }, + { + "accuracy": 0.414566, + "f1": 0.414415, + "f1_weighted": 0.414608 + } + ], + "main_score": 0.389916, + "hf_subset": "default", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 1.7920775413513184, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/GreekLegalCodeClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/GreekLegalCodeClassification.json new file mode 100644 index 0000000000..ba72337ea7 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/GreekLegalCodeClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "de0fdb34424f07d1ac6f0ede23ee0ed44bd9f5d1", + "task_name": "GreekLegalCodeClassification", + "mteb_version": "1.34.14", + "scores": { + "validation": [ + { + "accuracy": 0.028174, + "f1": 0.014065, + "f1_weighted": 0.016716, + "scores_per_experiment": [ + { + "accuracy": 0.023926, + "f1": 0.011915, + "f1_weighted": 0.016104 + }, + { + "accuracy": 0.029785, + "f1": 0.014776, + "f1_weighted": 0.018863 + }, + { + "accuracy": 0.026367, + "f1": 0.015441, + "f1_weighted": 0.014556 + }, + { + "accuracy": 0.019531, + "f1": 0.011665, + "f1_weighted": 0.011764 + }, + { + "accuracy": 0.023926, + "f1": 0.011622, + "f1_weighted": 0.011574 + }, + { + "accuracy": 0.033203, + "f1": 0.016842, + "f1_weighted": 0.021502 + }, + { + "accuracy": 0.033691, + "f1": 0.01278, + "f1_weighted": 0.021029 + }, + { + "accuracy": 0.033203, + "f1": 0.01815, + "f1_weighted": 0.020902 + }, + { + "accuracy": 0.029297, + "f1": 0.013857, + "f1_weighted": 0.016355 + }, + { + "accuracy": 0.028809, + "f1": 0.0136, + "f1_weighted": 0.014509 + } + ], + "main_score": 0.028174, + "hf_subset": "default", + "languages": [ + "ell-Grek" + ] + } + ], + "test": [ + { + "accuracy": 0.028271, + "f1": 0.015988, + "f1_weighted": 0.016241, + "scores_per_experiment": [ + { + "accuracy": 0.028809, + "f1": 0.017289, + "f1_weighted": 0.016183 + }, + { + "accuracy": 0.03418, + "f1": 0.020147, + "f1_weighted": 0.024854 + }, + { + "accuracy": 0.02832, + "f1": 0.01716, + "f1_weighted": 0.016353 + }, + { + "accuracy": 0.021484, + "f1": 0.015055, + "f1_weighted": 0.011895 + }, + { + "accuracy": 0.02832, + "f1": 0.019094, + "f1_weighted": 0.01852 + }, + { + "accuracy": 0.029297, + "f1": 0.015368, + "f1_weighted": 0.015046 + }, + { + "accuracy": 0.026855, + "f1": 0.01468, + "f1_weighted": 0.016757 + }, + { + "accuracy": 0.026855, + "f1": 0.013694, + "f1_weighted": 0.017093 + }, + { + "accuracy": 0.02832, + "f1": 0.010147, + "f1_weighted": 0.008866 + }, + { + "accuracy": 0.030273, + "f1": 0.017248, + "f1_weighted": 0.016848 + } + ], + "main_score": 0.028271, + "hf_subset": "default", + "languages": [ + "ell-Grek" + ] + } + ] + }, + "evaluation_time": 734.3054769039154, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/GujaratiNewsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/GujaratiNewsClassification.json new file mode 100644 index 0000000000..118838c385 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/GujaratiNewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "1a5f2fa2914bfeff4fcdc6fff4194fa8ec8fa19e", + "task_name": "GujaratiNewsClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.47959, + "f1": 0.461575, + "f1_weighted": 0.474341, + "scores_per_experiment": [ + { + "accuracy": 0.489378, + "f1": 0.469489, + "f1_weighted": 0.50379 + }, + { + "accuracy": 0.41047, + "f1": 0.379825, + "f1_weighted": 0.393233 + }, + { + "accuracy": 0.479514, + "f1": 0.443582, + "f1_weighted": 0.477526 + }, + { + "accuracy": 0.474203, + "f1": 0.456011, + "f1_weighted": 0.473574 + }, + { + "accuracy": 0.477997, + "f1": 0.47818, + "f1_weighted": 0.476099 + }, + { + "accuracy": 0.497724, + "f1": 0.495312, + "f1_weighted": 0.492586 + }, + { + "accuracy": 0.526555, + "f1": 0.518598, + "f1_weighted": 0.526548 + }, + { + "accuracy": 0.599393, + "f1": 0.57769, + "f1_weighted": 0.593396 + }, + { + "accuracy": 0.465099, + "f1": 0.417654, + "f1_weighted": 0.431705 + }, + { + "accuracy": 0.375569, + "f1": 0.379408, + "f1_weighted": 0.374949 + } + ], + "main_score": 0.47959, + "hf_subset": "default", + "languages": [ + "guj-Gujr" + ] + } + ] + }, + "evaluation_time": 1.7617313861846924, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/HateSpeechPortugueseClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/HateSpeechPortugueseClassification.json new file mode 100644 index 0000000000..505fd9baac --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/HateSpeechPortugueseClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "b0f431acbf8d3865cb7c7b3effb2a9771a618ebc", + "task_name": "HateSpeechPortugueseClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.535889, + "f1": 0.513134, + "f1_weighted": 0.548282, + "ap": 0.332784, + "ap_weighted": 0.332784, + "scores_per_experiment": [ + { + "accuracy": 0.507812, + "f1": 0.498162, + "f1_weighted": 0.523851, + "ap": 0.328697, + "ap_weighted": 0.328697 + }, + { + "accuracy": 0.556641, + "f1": 0.544969, + "f1_weighted": 0.571871, + "ap": 0.35477, + "ap_weighted": 0.35477 + }, + { + "accuracy": 0.475586, + "f1": 0.475573, + "f1_weighted": 0.476519, + "ap": 0.338436, + "ap_weighted": 0.338436 + }, + { + "accuracy": 0.598145, + "f1": 0.542009, + "f1_weighted": 0.601198, + "ap": 0.337202, + "ap_weighted": 0.337202 + }, + { + "accuracy": 0.532715, + "f1": 0.515596, + "f1_weighted": 0.549211, + "ap": 0.332735, + "ap_weighted": 0.332735 + }, + { + "accuracy": 0.541016, + "f1": 0.503638, + "f1_weighted": 0.553918, + "ap": 0.319474, + "ap_weighted": 0.319474 + }, + { + "accuracy": 0.530273, + "f1": 0.507974, + "f1_weighted": 0.54664, + "ap": 0.325999, + "ap_weighted": 0.325999 + }, + { + "accuracy": 0.527344, + "f1": 0.50359, + "f1_weighted": 0.543675, + "ap": 0.323162, + "ap_weighted": 0.323162 + }, + { + "accuracy": 0.574219, + "f1": 0.534823, + "f1_weighted": 0.584795, + "ap": 0.335349, + "ap_weighted": 0.335349 + }, + { + "accuracy": 0.515137, + "f1": 0.505005, + "f1_weighted": 0.531147, + "ap": 0.332016, + "ap_weighted": 0.332016 + } + ], + "main_score": 0.535889, + "hf_subset": "default", + "languages": [ + "por-Latn" + ] + } + ] + }, + "evaluation_time": 3.3274154663085938, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/HeadlineClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/HeadlineClassification.json new file mode 100644 index 0000000000..9c1a6ea6d2 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/HeadlineClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb", + "task_name": "HeadlineClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.289355, + "f1": 0.276813, + "f1_weighted": 0.276808, + "scores_per_experiment": [ + { + "accuracy": 0.321777, + "f1": 0.312746, + "f1_weighted": 0.31278 + }, + { + "accuracy": 0.285156, + "f1": 0.277236, + "f1_weighted": 0.277234 + }, + { + "accuracy": 0.290527, + "f1": 0.291562, + "f1_weighted": 0.29151 + }, + { + "accuracy": 0.314453, + "f1": 0.308133, + "f1_weighted": 0.308116 + }, + { + "accuracy": 0.29834, + "f1": 0.285979, + "f1_weighted": 0.285995 + }, + { + "accuracy": 0.288574, + "f1": 0.26258, + "f1_weighted": 0.262651 + }, + { + "accuracy": 0.289551, + "f1": 0.271399, + "f1_weighted": 0.271374 + }, + { + "accuracy": 0.252441, + "f1": 0.235026, + "f1_weighted": 0.234989 + }, + { + "accuracy": 0.278809, + "f1": 0.267072, + "f1_weighted": 0.267024 + }, + { + "accuracy": 0.273926, + "f1": 0.256395, + "f1_weighted": 0.256412 + } + ], + "main_score": 0.289355, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 3.986896514892578, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/HebrewSentimentAnalysis.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/HebrewSentimentAnalysis.json new file mode 100644 index 0000000000..349a98a1f3 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/HebrewSentimentAnalysis.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "952c9525954c1dac50d5f95945eb5585bb6464e7", + "task_name": "HebrewSentimentAnalysis", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.623877, + "f1": 0.478222, + "f1_weighted": 0.631682, + "scores_per_experiment": [ + { + "accuracy": 0.616699, + "f1": 0.51384, + "f1_weighted": 0.629787 + }, + { + "accuracy": 0.679199, + "f1": 0.544231, + "f1_weighted": 0.685964 + }, + { + "accuracy": 0.596191, + "f1": 0.460811, + "f1_weighted": 0.62252 + }, + { + "accuracy": 0.674805, + "f1": 0.52561, + "f1_weighted": 0.681964 + }, + { + "accuracy": 0.614258, + "f1": 0.40817, + "f1_weighted": 0.604112 + }, + { + "accuracy": 0.663086, + "f1": 0.500639, + "f1_weighted": 0.674955 + }, + { + "accuracy": 0.628418, + "f1": 0.494223, + "f1_weighted": 0.64084 + }, + { + "accuracy": 0.472168, + "f1": 0.384072, + "f1_weighted": 0.47892 + }, + { + "accuracy": 0.681641, + "f1": 0.469136, + "f1_weighted": 0.662841 + }, + { + "accuracy": 0.612305, + "f1": 0.481492, + "f1_weighted": 0.634917 + } + ], + "main_score": 0.623877, + "hf_subset": "default", + "languages": [ + "heb-Hebr" + ] + } + ] + }, + "evaluation_time": 5.036404371261597, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/HinDialectClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/HinDialectClassification.json new file mode 100644 index 0000000000..d3aaaf992b --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/HinDialectClassification.json @@ -0,0 +1,93 @@ +{ + "dataset_revision": "944a44cf93932ce62b51e7c07d44d8cc03d6bcae", + "task_name": "HinDialectClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.433767, + "f1": 0.202958, + "f1_weighted": 0.425647, + "scores_per_experiment": [ + { + "accuracy": 0.417535, + "f1": 0.194567, + "f1_weighted": 0.418051 + }, + { + "accuracy": 0.40191, + "f1": 0.18722, + "f1_weighted": 0.391748 + }, + { + "accuracy": 0.395833, + "f1": 0.196945, + "f1_weighted": 0.373519 + }, + { + "accuracy": 0.471354, + "f1": 0.231774, + "f1_weighted": 0.470802 + }, + { + "accuracy": 0.402778, + "f1": 0.197833, + "f1_weighted": 0.381631 + }, + { + "accuracy": 0.432292, + "f1": 0.188736, + "f1_weighted": 0.423034 + }, + { + "accuracy": 0.465278, + "f1": 0.232484, + "f1_weighted": 0.488746 + }, + { + "accuracy": 0.505208, + "f1": 0.226189, + "f1_weighted": 0.484688 + }, + { + "accuracy": 0.399306, + "f1": 0.189563, + "f1_weighted": 0.393981 + }, + { + "accuracy": 0.446181, + "f1": 0.18427, + "f1_weighted": 0.430266 + } + ], + "main_score": 0.202958, + "hf_subset": "default", + "languages": [ + "pan-Guru", + "bgc-Deva", + "mag-Deva", + "bns-Deva", + "kfg-Deva", + "noe-Deva", + "bhb-Deva", + "bho-Deva", + "gbm-Deva", + "mup-Deva", + "anp-Deva", + "hne-Deva", + "bra-Deva", + "raj-Deva", + "awa-Deva", + "guj-Gujr", + "ben-Beng", + "bhd-Deva", + "kfy-Deva", + "mar-Deva", + "bjj-Deva" + ] + } + ] + }, + "evaluation_time": 14.571729898452759, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/HindiDiscourseClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/HindiDiscourseClassification.json new file mode 100644 index 0000000000..46bf0c30b8 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/HindiDiscourseClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "218ce687943a0da435d6d62751a4ab216be6cd40", + "task_name": "HindiDiscourseClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.256055, + "f1": 0.289003, + "f1_weighted": 0.285097, + "scores_per_experiment": [ + { + "accuracy": 0.208496, + "f1": 0.27728, + "f1_weighted": 0.237996 + }, + { + "accuracy": 0.208496, + "f1": 0.282149, + "f1_weighted": 0.259848 + }, + { + "accuracy": 0.249023, + "f1": 0.288585, + "f1_weighted": 0.296655 + }, + { + "accuracy": 0.230957, + "f1": 0.255432, + "f1_weighted": 0.232969 + }, + { + "accuracy": 0.246094, + "f1": 0.293116, + "f1_weighted": 0.298225 + }, + { + "accuracy": 0.344238, + "f1": 0.331883, + "f1_weighted": 0.370085 + }, + { + "accuracy": 0.262207, + "f1": 0.299546, + "f1_weighted": 0.308848 + }, + { + "accuracy": 0.257324, + "f1": 0.278829, + "f1_weighted": 0.282181 + }, + { + "accuracy": 0.350098, + "f1": 0.306437, + "f1_weighted": 0.332437 + }, + { + "accuracy": 0.203613, + "f1": 0.276773, + "f1_weighted": 0.231725 + } + ], + "main_score": 0.256055, + "hf_subset": "default", + "languages": [ + "hin-Deva" + ] + } + ] + }, + "evaluation_time": 3.256507635116577, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/HotelReviewSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/HotelReviewSentimentClassification.json new file mode 100644 index 0000000000..95d510ac9e --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/HotelReviewSentimentClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "b108d2c32ee4e1f4176ea233e1a5ac17bceb9ef9", + "task_name": "HotelReviewSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.289697, + "f1": 0.265239, + "f1_weighted": 0.275475, + "scores_per_experiment": [ + { + "accuracy": 0.333008, + "f1": 0.313894, + "f1_weighted": 0.33201 + }, + { + "accuracy": 0.269043, + "f1": 0.261497, + "f1_weighted": 0.248058 + }, + { + "accuracy": 0.304688, + "f1": 0.274272, + "f1_weighted": 0.303404 + }, + { + "accuracy": 0.296875, + "f1": 0.286173, + "f1_weighted": 0.305289 + }, + { + "accuracy": 0.262207, + "f1": 0.244119, + "f1_weighted": 0.251815 + }, + { + "accuracy": 0.23291, + "f1": 0.225267, + "f1_weighted": 0.211781 + }, + { + "accuracy": 0.264648, + "f1": 0.253382, + "f1_weighted": 0.242983 + }, + { + "accuracy": 0.235352, + "f1": 0.222001, + "f1_weighted": 0.203703 + }, + { + "accuracy": 0.344238, + "f1": 0.275481, + "f1_weighted": 0.32077 + }, + { + "accuracy": 0.354004, + "f1": 0.296308, + "f1_weighted": 0.334935 + } + ], + "main_score": 0.289697, + "hf_subset": "default", + "languages": [ + "ara-Arab" + ] + } + ] + }, + "evaluation_time": 4.650956392288208, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/HotpotQA.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/HotpotQA.json deleted file mode 100644 index 9a2a12a14b..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/HotpotQA.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "mteb_version": "0.0.2", - "test": { - "evaluation_time": 11375.95, - "map_at_1": 0.32829, - "map_at_10": 0.45778, - "map_at_100": 0.46582, - "map_at_1000": 0.46652, - "map_at_3": 0.43055, - "map_at_5": 0.44695, - "ndcg_at_1": 0.65658, - "ndcg_at_10": 0.54926, - "ndcg_at_100": 0.57989, - "ndcg_at_1000": 0.59496, - "ndcg_at_3": 0.50671, - "ndcg_at_5": 0.52933, - "precision_at_1": 0.65658, - "precision_at_10": 0.11333, - "precision_at_100": 0.01374, - "precision_at_1000": 0.00158, - "precision_at_3": 0.31456, - "precision_at_5": 0.20659, - "recall_at_1": 0.32829, - "recall_at_10": 0.56664, - "recall_at_100": 0.68683, - "recall_at_1000": 0.78751, - "recall_at_3": 0.47184, - "recall_at_5": 0.51648 - }, - "mteb_dataset_name": "HotpotQA", - "dataset_revision": "766870b35a1b9ca65e67a0d1913899973551fc6c" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/IFlyTek.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/IFlyTek.json new file mode 100644 index 0000000000..84c09b7743 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/IFlyTek.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "421605374b29664c5fc098418fe20ada9bd55f8a", + "task_name": "IFlyTek", + "mteb_version": "1.34.14", + "scores": { + "validation": [ + { + "accuracy": 0.080569, + "f1": 0.050147, + "f1_weighted": 0.077037, + "scores_per_experiment": [ + { + "accuracy": 0.089265, + "f1": 0.048073, + "f1_weighted": 0.090683 + }, + { + "accuracy": 0.083494, + "f1": 0.058368, + "f1_weighted": 0.077214 + }, + { + "accuracy": 0.091189, + "f1": 0.052031, + "f1_weighted": 0.086617 + }, + { + "accuracy": 0.077722, + "f1": 0.049995, + "f1_weighted": 0.070155 + }, + { + "accuracy": 0.061177, + "f1": 0.042266, + "f1_weighted": 0.060517 + } + ], + "main_score": 0.080569, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 40.8479585647583, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ImdbClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ImdbClassification.json index 314a5e3e1c..ced31fb8e5 100644 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/ImdbClassification.json +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/ImdbClassification.json @@ -1,15 +1,95 @@ { - "test": { - "accuracy": 0.659944, - "accuracy_stderr": 0.02785758754809897, - "ap": 0.6082565191656919, - "ap_stderr": 0.022986389600399903, - "evaluation_time": 2046.55, - "f1": 0.6561758506396675, - "f1_stderr": 0.03106996054448068, - "main_score": 0.659944 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "ImdbClassification", - "dataset_revision": "8d743909f834c38949e8323a8a6ce8721ea6c7f4" + "dataset_revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7", + "task_name": "ImdbClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.66628, + "f1": 0.663776, + "f1_weighted": 0.663776, + "ap": 0.610259, + "ap_weighted": 0.610259, + "scores_per_experiment": [ + { + "accuracy": 0.67872, + "f1": 0.67603, + "f1_weighted": 0.67603, + "ap": 0.628419, + "ap_weighted": 0.628419 + }, + { + "accuracy": 0.69988, + "f1": 0.699844, + "f1_weighted": 0.699844, + "ap": 0.640791, + "ap_weighted": 0.640791 + }, + { + "accuracy": 0.67984, + "f1": 0.675814, + "f1_weighted": 0.675814, + "ap": 0.616368, + "ap_weighted": 0.616368 + }, + { + "accuracy": 0.68376, + "f1": 0.682301, + "f1_weighted": 0.682301, + "ap": 0.621618, + "ap_weighted": 0.621618 + }, + { + "accuracy": 0.6972, + "f1": 0.697149, + "f1_weighted": 0.697149, + "ap": 0.638523, + "ap_weighted": 0.638523 + }, + { + "accuracy": 0.6608, + "f1": 0.650038, + "f1_weighted": 0.650038, + "ap": 0.599543, + "ap_weighted": 0.599543 + }, + { + "accuracy": 0.6374, + "f1": 0.637296, + "f1_weighted": 0.637296, + "ap": 0.58824, + "ap_weighted": 0.58824 + }, + { + "accuracy": 0.62068, + "f1": 0.620024, + "f1_weighted": 0.620024, + "ap": 0.573786, + "ap_weighted": 0.573786 + }, + { + "accuracy": 0.65108, + "f1": 0.645992, + "f1_weighted": 0.645992, + "ap": 0.593951, + "ap_weighted": 0.593951 + }, + { + "accuracy": 0.65344, + "f1": 0.653272, + "f1_weighted": 0.653272, + "ap": 0.601347, + "ap_weighted": 0.601347 + } + ], + "main_score": 0.66628, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 159.46412014961243, + "kg_co2_emissions": null } \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/InappropriatenessClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/InappropriatenessClassification.json new file mode 100644 index 0000000000..d0f8b38baa --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/InappropriatenessClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285", + "task_name": "InappropriatenessClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.519482, + "f1": 0.509718, + "f1_weighted": 0.509718, + "ap": 0.510485, + "ap_weighted": 0.510485, + "scores_per_experiment": [ + { + "accuracy": 0.51709, + "f1": 0.489879, + "f1_weighted": 0.489879, + "ap": 0.509088, + "ap_weighted": 0.509088 + }, + { + "accuracy": 0.567871, + "f1": 0.567462, + "f1_weighted": 0.567462, + "ap": 0.538275, + "ap_weighted": 0.538275 + }, + { + "accuracy": 0.499512, + "f1": 0.498142, + "f1_weighted": 0.498142, + "ap": 0.499756, + "ap_weighted": 0.499756 + }, + { + "accuracy": 0.509766, + "f1": 0.508774, + "f1_weighted": 0.508774, + "ap": 0.50497, + "ap_weighted": 0.50497 + }, + { + "accuracy": 0.512695, + "f1": 0.490222, + "f1_weighted": 0.490222, + "ap": 0.506461, + "ap_weighted": 0.506461 + }, + { + "accuracy": 0.508301, + "f1": 0.500923, + "f1_weighted": 0.500923, + "ap": 0.504206, + "ap_weighted": 0.504206 + }, + { + "accuracy": 0.530273, + "f1": 0.508611, + "f1_weighted": 0.508611, + "ap": 0.516717, + "ap_weighted": 0.516717 + }, + { + "accuracy": 0.522949, + "f1": 0.511239, + "f1_weighted": 0.511239, + "ap": 0.511877, + "ap_weighted": 0.511877 + }, + { + "accuracy": 0.515625, + "f1": 0.511239, + "f1_weighted": 0.511239, + "ap": 0.508018, + "ap_weighted": 0.508018 + }, + { + "accuracy": 0.510742, + "f1": 0.510686, + "f1_weighted": 0.510686, + "ap": 0.505484, + "ap_weighted": 0.505484 + } + ], + "main_score": 0.519482, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 5.120187044143677, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/IndicLangClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/IndicLangClassification.json new file mode 100644 index 0000000000..1931b3decf --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/IndicLangClassification.json @@ -0,0 +1,96 @@ +{ + "dataset_revision": "c54a95d9b9d62c891a03bd5da60715df7176b097", + "task_name": "IndicLangClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.141281, + "f1": 0.102306, + "f1_weighted": 0.106028, + "scores_per_experiment": [ + { + "accuracy": 0.15264, + "f1": 0.109399, + "f1_weighted": 0.113768 + }, + { + "accuracy": 0.120126, + "f1": 0.084536, + "f1_weighted": 0.083062 + }, + { + "accuracy": 0.132915, + "f1": 0.104738, + "f1_weighted": 0.114612 + }, + { + "accuracy": 0.136926, + "f1": 0.099534, + "f1_weighted": 0.100681 + }, + { + "accuracy": 0.160431, + "f1": 0.112932, + "f1_weighted": 0.119425 + }, + { + "accuracy": 0.124499, + "f1": 0.092991, + "f1_weighted": 0.094788 + }, + { + "accuracy": 0.14429, + "f1": 0.103918, + "f1_weighted": 0.104884 + }, + { + "accuracy": 0.143731, + "f1": 0.101637, + "f1_weighted": 0.103077 + }, + { + "accuracy": 0.159872, + "f1": 0.120678, + "f1_weighted": 0.130325 + }, + { + "accuracy": 0.137386, + "f1": 0.092696, + "f1_weighted": 0.09566 + } + ], + "main_score": 0.141281, + "hf_subset": "default", + "languages": [ + "asm-Beng", + "brx-Deva", + "ben-Beng", + "doi-Deva", + "gom-Deva", + "guj-Gujr", + "hin-Deva", + "kan-Knda", + "kas-Arab", + "kas-Deva", + "mai-Deva", + "mal-Mlym", + "mar-Deva", + "mni-Beng", + "mni-Mtei", + "npi-Deva", + "ory-Orya", + "pan-Guru", + "san-Deva", + "sat-Olck", + "snd-Arab", + "tam-Taml", + "tel-Telu", + "urd-Arab" + ] + } + ] + }, + "evaluation_time": 30.469869375228882, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/IndicNLPNewsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/IndicNLPNewsClassification.json new file mode 100644 index 0000000000..57887b7545 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/IndicNLPNewsClassification.json @@ -0,0 +1,507 @@ +{ + "dataset_revision": "3f23bd4a622a462adfb6989419cfadf7dc778f25", + "task_name": "IndicNLPNewsClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.614697, + "f1": 0.595977, + "f1_weighted": 0.598931, + "scores_per_experiment": [ + { + "accuracy": 0.59668, + "f1": 0.578234, + "f1_weighted": 0.581212 + }, + { + "accuracy": 0.576172, + "f1": 0.513973, + "f1_weighted": 0.519825 + }, + { + "accuracy": 0.681152, + "f1": 0.656941, + "f1_weighted": 0.660599 + }, + { + "accuracy": 0.630859, + "f1": 0.631152, + "f1_weighted": 0.63327 + }, + { + "accuracy": 0.650391, + "f1": 0.6465, + "f1_weighted": 0.649054 + }, + { + "accuracy": 0.604004, + "f1": 0.599167, + "f1_weighted": 0.600006 + }, + { + "accuracy": 0.543457, + "f1": 0.527447, + "f1_weighted": 0.528759 + }, + { + "accuracy": 0.609375, + "f1": 0.55999, + "f1_weighted": 0.565632 + }, + { + "accuracy": 0.629395, + "f1": 0.627331, + "f1_weighted": 0.628991 + }, + { + "accuracy": 0.625488, + "f1": 0.619039, + "f1_weighted": 0.621959 + } + ], + "main_score": 0.614697, + "hf_subset": "gu", + "languages": [ + "guj-Gujr" + ] + }, + { + "accuracy": 0.637549, + "f1": 0.627721, + "f1_weighted": 0.628852, + "scores_per_experiment": [ + { + "accuracy": 0.62207, + "f1": 0.616069, + "f1_weighted": 0.616232 + }, + { + "accuracy": 0.644531, + "f1": 0.636828, + "f1_weighted": 0.636993 + }, + { + "accuracy": 0.545898, + "f1": 0.539133, + "f1_weighted": 0.542722 + }, + { + "accuracy": 0.603516, + "f1": 0.604585, + "f1_weighted": 0.60247 + }, + { + "accuracy": 0.656738, + "f1": 0.63032, + "f1_weighted": 0.632985 + }, + { + "accuracy": 0.679199, + "f1": 0.66673, + "f1_weighted": 0.668733 + }, + { + "accuracy": 0.68457, + "f1": 0.68769, + "f1_weighted": 0.688059 + }, + { + "accuracy": 0.617676, + "f1": 0.614731, + "f1_weighted": 0.614524 + }, + { + "accuracy": 0.614746, + "f1": 0.582621, + "f1_weighted": 0.585788 + }, + { + "accuracy": 0.706543, + "f1": 0.698504, + "f1_weighted": 0.700015 + } + ], + "main_score": 0.637549, + "hf_subset": "kn", + "languages": [ + "kan-Knda" + ] + }, + { + "accuracy": 0.412207, + "f1": 0.370662, + "f1_weighted": 0.372693, + "scores_per_experiment": [ + { + "accuracy": 0.420898, + "f1": 0.373566, + "f1_weighted": 0.37549 + }, + { + "accuracy": 0.432617, + "f1": 0.367921, + "f1_weighted": 0.369189 + }, + { + "accuracy": 0.45752, + "f1": 0.433229, + "f1_weighted": 0.434955 + }, + { + "accuracy": 0.417969, + "f1": 0.376723, + "f1_weighted": 0.379691 + }, + { + "accuracy": 0.458984, + "f1": 0.409957, + "f1_weighted": 0.410881 + }, + { + "accuracy": 0.387695, + "f1": 0.341947, + "f1_weighted": 0.345217 + }, + { + "accuracy": 0.397949, + "f1": 0.343421, + "f1_weighted": 0.347818 + }, + { + "accuracy": 0.374023, + "f1": 0.325941, + "f1_weighted": 0.329394 + }, + { + "accuracy": 0.457031, + "f1": 0.422132, + "f1_weighted": 0.423641 + }, + { + "accuracy": 0.317383, + "f1": 0.311782, + "f1_weighted": 0.310649 + } + ], + "main_score": 0.412207, + "hf_subset": "mal", + "languages": [ + "mal-Mlym" + ] + }, + { + "accuracy": 0.476318, + "f1": 0.450683, + "f1_weighted": 0.450422, + "scores_per_experiment": [ + { + "accuracy": 0.479004, + "f1": 0.448305, + "f1_weighted": 0.44801 + }, + { + "accuracy": 0.462891, + "f1": 0.458338, + "f1_weighted": 0.457185 + }, + { + "accuracy": 0.422852, + "f1": 0.369206, + "f1_weighted": 0.369398 + }, + { + "accuracy": 0.483887, + "f1": 0.457739, + "f1_weighted": 0.45908 + }, + { + "accuracy": 0.472656, + "f1": 0.446563, + "f1_weighted": 0.445131 + }, + { + "accuracy": 0.466309, + "f1": 0.442381, + "f1_weighted": 0.443729 + }, + { + "accuracy": 0.484863, + "f1": 0.461554, + "f1_weighted": 0.459934 + }, + { + "accuracy": 0.513184, + "f1": 0.488714, + "f1_weighted": 0.49039 + }, + { + "accuracy": 0.472656, + "f1": 0.449521, + "f1_weighted": 0.448241 + }, + { + "accuracy": 0.504883, + "f1": 0.484508, + "f1_weighted": 0.483122 + } + ], + "main_score": 0.476318, + "hf_subset": "mr", + "languages": [ + "mar-Deva" + ] + }, + { + "accuracy": 0.609766, + "f1": 0.600951, + "f1_weighted": 0.60423, + "scores_per_experiment": [ + { + "accuracy": 0.637207, + "f1": 0.634796, + "f1_weighted": 0.637935 + }, + { + "accuracy": 0.550781, + "f1": 0.548659, + "f1_weighted": 0.549656 + }, + { + "accuracy": 0.645996, + "f1": 0.636345, + "f1_weighted": 0.639719 + }, + { + "accuracy": 0.5625, + "f1": 0.557453, + "f1_weighted": 0.560993 + }, + { + "accuracy": 0.635742, + "f1": 0.636249, + "f1_weighted": 0.6386 + }, + { + "accuracy": 0.64209, + "f1": 0.629594, + "f1_weighted": 0.632782 + }, + { + "accuracy": 0.57666, + "f1": 0.570565, + "f1_weighted": 0.574944 + }, + { + "accuracy": 0.615234, + "f1": 0.605039, + "f1_weighted": 0.60824 + }, + { + "accuracy": 0.609375, + "f1": 0.595433, + "f1_weighted": 0.59923 + }, + { + "accuracy": 0.62207, + "f1": 0.595377, + "f1_weighted": 0.600202 + } + ], + "main_score": 0.609766, + "hf_subset": "tel", + "languages": [ + "tel-Telu" + ] + }, + { + "accuracy": 0.406836, + "f1": 0.358907, + "f1_weighted": 0.360709, + "scores_per_experiment": [ + { + "accuracy": 0.399902, + "f1": 0.381662, + "f1_weighted": 0.389034 + }, + { + "accuracy": 0.447754, + "f1": 0.403855, + "f1_weighted": 0.409031 + }, + { + "accuracy": 0.276855, + "f1": 0.273018, + "f1_weighted": 0.269139 + }, + { + "accuracy": 0.456055, + "f1": 0.392834, + "f1_weighted": 0.393454 + }, + { + "accuracy": 0.401367, + "f1": 0.37988, + "f1_weighted": 0.387643 + }, + { + "accuracy": 0.333008, + "f1": 0.233122, + "f1_weighted": 0.220475 + }, + { + "accuracy": 0.436035, + "f1": 0.390424, + "f1_weighted": 0.394263 + }, + { + "accuracy": 0.463867, + "f1": 0.418462, + "f1_weighted": 0.420144 + }, + { + "accuracy": 0.388672, + "f1": 0.319535, + "f1_weighted": 0.326486 + }, + { + "accuracy": 0.464844, + "f1": 0.396276, + "f1_weighted": 0.397421 + } + ], + "main_score": 0.406836, + "hf_subset": "ori", + "languages": [ + "ori-Orya" + ] + }, + { + "accuracy": 0.493429, + "f1": 0.48573, + "f1_weighted": 0.481573, + "scores_per_experiment": [ + { + "accuracy": 0.509615, + "f1": 0.499332, + "f1_weighted": 0.497464 + }, + { + "accuracy": 0.440705, + "f1": 0.438897, + "f1_weighted": 0.435971 + }, + { + "accuracy": 0.525641, + "f1": 0.525017, + "f1_weighted": 0.517322 + }, + { + "accuracy": 0.426282, + "f1": 0.38985, + "f1_weighted": 0.384624 + }, + { + "accuracy": 0.464744, + "f1": 0.439813, + "f1_weighted": 0.43544 + }, + { + "accuracy": 0.514423, + "f1": 0.512721, + "f1_weighted": 0.509531 + }, + { + "accuracy": 0.485577, + "f1": 0.484083, + "f1_weighted": 0.481307 + }, + { + "accuracy": 0.511218, + "f1": 0.510499, + "f1_weighted": 0.50522 + }, + { + "accuracy": 0.541667, + "f1": 0.548435, + "f1_weighted": 0.544923 + }, + { + "accuracy": 0.514423, + "f1": 0.508652, + "f1_weighted": 0.503928 + } + ], + "main_score": 0.493429, + "hf_subset": "pa", + "languages": [ + "pan-Guru" + ] + }, + { + "accuracy": 0.511475, + "f1": 0.488407, + "f1_weighted": 0.486837, + "scores_per_experiment": [ + { + "accuracy": 0.51709, + "f1": 0.472467, + "f1_weighted": 0.47694 + }, + { + "accuracy": 0.524414, + "f1": 0.50043, + "f1_weighted": 0.496354 + }, + { + "accuracy": 0.518066, + "f1": 0.493023, + "f1_weighted": 0.488344 + }, + { + "accuracy": 0.491211, + "f1": 0.449359, + "f1_weighted": 0.442894 + }, + { + "accuracy": 0.515137, + "f1": 0.515508, + "f1_weighted": 0.51429 + }, + { + "accuracy": 0.470703, + "f1": 0.472605, + "f1_weighted": 0.469661 + }, + { + "accuracy": 0.539551, + "f1": 0.527279, + "f1_weighted": 0.52467 + }, + { + "accuracy": 0.499512, + "f1": 0.480998, + "f1_weighted": 0.476651 + }, + { + "accuracy": 0.522949, + "f1": 0.469502, + "f1_weighted": 0.47458 + }, + { + "accuracy": 0.516113, + "f1": 0.502896, + "f1_weighted": 0.503992 + } + ], + "main_score": 0.511475, + "hf_subset": "ta", + "languages": [ + "tam-Taml" + ] + } + ] + }, + "evaluation_time": 146.84807062149048, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/IndicSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/IndicSentimentClassification.json new file mode 100644 index 0000000000..4772493ab2 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/IndicSentimentClassification.json @@ -0,0 +1,1103 @@ +{ + "dataset_revision": "3389cc78b2ffcbd33639e91dfc57e6b6b6496241", + "task_name": "IndicSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.511824, + "f1": 0.477737, + "f1_weighted": 0.476546, + "ap": 0.515264, + "ap_weighted": 0.515264, + "scores_per_experiment": [ + { + "accuracy": 0.558116, + "f1": 0.53865, + "f1_weighted": 0.537321, + "ap": 0.543962, + "ap_weighted": 0.543962 + }, + { + "accuracy": 0.488978, + "f1": 0.446657, + "f1_weighted": 0.444511, + "ap": 0.503605, + "ap_weighted": 0.503605 + }, + { + "accuracy": 0.472946, + "f1": 0.392075, + "f1_weighted": 0.388965, + "ap": 0.497928, + "ap_weighted": 0.497928 + }, + { + "accuracy": 0.508016, + "f1": 0.467591, + "f1_weighted": 0.465533, + "ap": 0.513329, + "ap_weighted": 0.513329 + }, + { + "accuracy": 0.519038, + "f1": 0.481055, + "f1_weighted": 0.483025, + "ap": 0.514837, + "ap_weighted": 0.514837 + }, + { + "accuracy": 0.512024, + "f1": 0.510537, + "f1_weighted": 0.510159, + "ap": 0.513649, + "ap_weighted": 0.513649 + }, + { + "accuracy": 0.546092, + "f1": 0.542453, + "f1_weighted": 0.543026, + "ap": 0.531194, + "ap_weighted": 0.531194 + }, + { + "accuracy": 0.48998, + "f1": 0.413201, + "f1_weighted": 0.410223, + "ap": 0.504679, + "ap_weighted": 0.504679 + }, + { + "accuracy": 0.533066, + "f1": 0.530196, + "f1_weighted": 0.529681, + "ap": 0.52554, + "ap_weighted": 0.52554 + }, + { + "accuracy": 0.48998, + "f1": 0.454952, + "f1_weighted": 0.453014, + "ap": 0.503915, + "ap_weighted": 0.503915 + } + ], + "main_score": 0.511824, + "hf_subset": "bd", + "languages": [ + "brx-Deva" + ] + }, + { + "accuracy": 0.508517, + "f1": 0.49011, + "f1_weighted": 0.489252, + "ap": 0.513074, + "ap_weighted": 0.513074, + "scores_per_experiment": [ + { + "accuracy": 0.54008, + "f1": 0.527475, + "f1_weighted": 0.526393, + "ap": 0.530947, + "ap_weighted": 0.530947 + }, + { + "accuracy": 0.523046, + "f1": 0.509475, + "f1_weighted": 0.50833, + "ap": 0.520733, + "ap_weighted": 0.520733 + }, + { + "accuracy": 0.48497, + "f1": 0.484141, + "f1_weighted": 0.483851, + "ap": 0.500054, + "ap_weighted": 0.500054 + }, + { + "accuracy": 0.51503, + "f1": 0.495159, + "f1_weighted": 0.493754, + "ap": 0.516507, + "ap_weighted": 0.516507 + }, + { + "accuracy": 0.507014, + "f1": 0.506568, + "f1_weighted": 0.50636, + "ap": 0.510842, + "ap_weighted": 0.510842 + }, + { + "accuracy": 0.493988, + "f1": 0.493133, + "f1_weighted": 0.493425, + "ap": 0.503808, + "ap_weighted": 0.503808 + }, + { + "accuracy": 0.568136, + "f1": 0.558559, + "f1_weighted": 0.559471, + "ap": 0.54347, + "ap_weighted": 0.54347 + }, + { + "accuracy": 0.485972, + "f1": 0.383428, + "f1_weighted": 0.3799, + "ap": 0.5033, + "ap_weighted": 0.5033 + }, + { + "accuracy": 0.46493, + "f1": 0.459282, + "f1_weighted": 0.458506, + "ap": 0.491664, + "ap_weighted": 0.491664 + }, + { + "accuracy": 0.502004, + "f1": 0.483884, + "f1_weighted": 0.482527, + "ap": 0.509416, + "ap_weighted": 0.509416 + } + ], + "main_score": 0.508517, + "hf_subset": "as", + "languages": [ + "asm-Beng" + ] + }, + { + "accuracy": 0.521343, + "f1": 0.483989, + "f1_weighted": 0.482634, + "ap": 0.520309, + "ap_weighted": 0.520309, + "scores_per_experiment": [ + { + "accuracy": 0.537074, + "f1": 0.534515, + "f1_weighted": 0.534031, + "ap": 0.527839, + "ap_weighted": 0.527839 + }, + { + "accuracy": 0.504008, + "f1": 0.459968, + "f1_weighted": 0.457805, + "ap": 0.511227, + "ap_weighted": 0.511227 + }, + { + "accuracy": 0.506012, + "f1": 0.406299, + "f1_weighted": 0.402885, + "ap": 0.513778, + "ap_weighted": 0.513778 + }, + { + "accuracy": 0.511022, + "f1": 0.454205, + "f1_weighted": 0.451734, + "ap": 0.515546, + "ap_weighted": 0.515546 + }, + { + "accuracy": 0.536072, + "f1": 0.50981, + "f1_weighted": 0.511402, + "ap": 0.524214, + "ap_weighted": 0.524214 + }, + { + "accuracy": 0.47996, + "f1": 0.463849, + "f1_weighted": 0.462545, + "ap": 0.498732, + "ap_weighted": 0.498732 + }, + { + "accuracy": 0.56513, + "f1": 0.565018, + "f1_weighted": 0.565116, + "ap": 0.543609, + "ap_weighted": 0.543609 + }, + { + "accuracy": 0.5, + "f1": 0.411509, + "f1_weighted": 0.408307, + "ap": 0.509927, + "ap_weighted": 0.509927 + }, + { + "accuracy": 0.51002, + "f1": 0.471015, + "f1_weighted": 0.469, + "ap": 0.51441, + "ap_weighted": 0.51441 + }, + { + "accuracy": 0.564128, + "f1": 0.563707, + "f1_weighted": 0.563517, + "ap": 0.543808, + "ap_weighted": 0.543808 + } + ], + "main_score": 0.521343, + "hf_subset": "ml", + "languages": [ + "mal-Mlym" + ] + }, + { + "accuracy": 0.524148, + "f1": 0.483408, + "f1_weighted": 0.482454, + "ap": 0.521316, + "ap_weighted": 0.521316, + "scores_per_experiment": [ + { + "accuracy": 0.548096, + "f1": 0.538624, + "f1_weighted": 0.537696, + "ap": 0.53567, + "ap_weighted": 0.53567 + }, + { + "accuracy": 0.490982, + "f1": 0.441155, + "f1_weighted": 0.438814, + "ap": 0.504705, + "ap_weighted": 0.504705 + }, + { + "accuracy": 0.501002, + "f1": 0.392356, + "f1_weighted": 0.388752, + "ap": 0.510878, + "ap_weighted": 0.510878 + }, + { + "accuracy": 0.517034, + "f1": 0.492835, + "f1_weighted": 0.491281, + "ap": 0.517845, + "ap_weighted": 0.517845 + }, + { + "accuracy": 0.51503, + "f1": 0.451352, + "f1_weighted": 0.453974, + "ap": 0.512252, + "ap_weighted": 0.512252 + }, + { + "accuracy": 0.536072, + "f1": 0.528917, + "f1_weighted": 0.528102, + "ap": 0.527879, + "ap_weighted": 0.527879 + }, + { + "accuracy": 0.537074, + "f1": 0.510144, + "f1_weighted": 0.511755, + "ap": 0.524734, + "ap_weighted": 0.524734 + }, + { + "accuracy": 0.504008, + "f1": 0.405062, + "f1_weighted": 0.401658, + "ap": 0.512487, + "ap_weighted": 0.512487 + }, + { + "accuracy": 0.529058, + "f1": 0.510958, + "f1_weighted": 0.509638, + "ap": 0.524606, + "ap_weighted": 0.524606 + }, + { + "accuracy": 0.563126, + "f1": 0.562677, + "f1_weighted": 0.562873, + "ap": 0.542102, + "ap_weighted": 0.542102 + } + ], + "main_score": 0.524148, + "hf_subset": "mr", + "languages": [ + "mar-Deva" + ] + }, + { + "accuracy": 0.516533, + "f1": 0.484546, + "f1_weighted": 0.483263, + "ap": 0.517779, + "ap_weighted": 0.517779, + "scores_per_experiment": [ + { + "accuracy": 0.554108, + "f1": 0.552847, + "f1_weighted": 0.55318, + "ap": 0.536326, + "ap_weighted": 0.536326 + }, + { + "accuracy": 0.496994, + "f1": 0.460369, + "f1_weighted": 0.458396, + "ap": 0.507389, + "ap_weighted": 0.507389 + }, + { + "accuracy": 0.498998, + "f1": 0.398462, + "f1_weighted": 0.395012, + "ap": 0.509566, + "ap_weighted": 0.509566 + }, + { + "accuracy": 0.497996, + "f1": 0.436789, + "f1_weighted": 0.434184, + "ap": 0.508396, + "ap_weighted": 0.508396 + }, + { + "accuracy": 0.541082, + "f1": 0.54061, + "f1_weighted": 0.540816, + "ap": 0.528937, + "ap_weighted": 0.528937 + }, + { + "accuracy": 0.457916, + "f1": 0.448114, + "f1_weighted": 0.447083, + "ap": 0.489162, + "ap_weighted": 0.489162 + }, + { + "accuracy": 0.568136, + "f1": 0.567406, + "f1_weighted": 0.567157, + "ap": 0.546586, + "ap_weighted": 0.546586 + }, + { + "accuracy": 0.498998, + "f1": 0.406495, + "f1_weighted": 0.403208, + "ap": 0.509439, + "ap_weighted": 0.509439 + }, + { + "accuracy": 0.514028, + "f1": 0.499329, + "f1_weighted": 0.498126, + "ap": 0.515698, + "ap_weighted": 0.515698 + }, + { + "accuracy": 0.537074, + "f1": 0.535041, + "f1_weighted": 0.535472, + "ap": 0.526295, + "ap_weighted": 0.526295 + } + ], + "main_score": 0.516533, + "hf_subset": "ta", + "languages": [ + "tam-Taml" + ] + }, + { + "accuracy": 0.516032, + "f1": 0.508832, + "f1_weighted": 0.50872, + "ap": 0.516134, + "ap_weighted": 0.516134, + "scores_per_experiment": [ + { + "accuracy": 0.532064, + "f1": 0.525315, + "f1_weighted": 0.524521, + "ap": 0.525442, + "ap_weighted": 0.525442 + }, + { + "accuracy": 0.509018, + "f1": 0.486723, + "f1_weighted": 0.485222, + "ap": 0.513285, + "ap_weighted": 0.513285 + }, + { + "accuracy": 0.480962, + "f1": 0.465451, + "f1_weighted": 0.466729, + "ap": 0.496688, + "ap_weighted": 0.496688 + }, + { + "accuracy": 0.539078, + "f1": 0.539012, + "f1_weighted": 0.539089, + "ap": 0.528004, + "ap_weighted": 0.528004 + }, + { + "accuracy": 0.47495, + "f1": 0.466985, + "f1_weighted": 0.467899, + "ap": 0.494251, + "ap_weighted": 0.494251 + }, + { + "accuracy": 0.49499, + "f1": 0.492493, + "f1_weighted": 0.491994, + "ap": 0.505068, + "ap_weighted": 0.505068 + }, + { + "accuracy": 0.547094, + "f1": 0.540673, + "f1_weighted": 0.541435, + "ap": 0.531454, + "ap_weighted": 0.531454 + }, + { + "accuracy": 0.527054, + "f1": 0.525665, + "f1_weighted": 0.525305, + "ap": 0.521845, + "ap_weighted": 0.521845 + }, + { + "accuracy": 0.50501, + "f1": 0.495922, + "f1_weighted": 0.494973, + "ap": 0.510577, + "ap_weighted": 0.510577 + }, + { + "accuracy": 0.5501, + "f1": 0.550078, + "f1_weighted": 0.550034, + "ap": 0.534729, + "ap_weighted": 0.534729 + } + ], + "main_score": 0.516032, + "hf_subset": "bn", + "languages": [ + "ben-Beng" + ] + }, + { + "accuracy": 0.509519, + "f1": 0.493037, + "f1_weighted": 0.492166, + "ap": 0.513586, + "ap_weighted": 0.513586, + "scores_per_experiment": [ + { + "accuracy": 0.552104, + "f1": 0.545422, + "f1_weighted": 0.544649, + "ap": 0.53782, + "ap_weighted": 0.53782 + }, + { + "accuracy": 0.487976, + "f1": 0.46354, + "f1_weighted": 0.461933, + "ap": 0.502689, + "ap_weighted": 0.502689 + }, + { + "accuracy": 0.475952, + "f1": 0.456458, + "f1_weighted": 0.455014, + "ap": 0.497104, + "ap_weighted": 0.497104 + }, + { + "accuracy": 0.516032, + "f1": 0.514863, + "f1_weighted": 0.514528, + "ap": 0.515738, + "ap_weighted": 0.515738 + }, + { + "accuracy": 0.481964, + "f1": 0.479757, + "f1_weighted": 0.480232, + "ap": 0.497902, + "ap_weighted": 0.497902 + }, + { + "accuracy": 0.514028, + "f1": 0.513285, + "f1_weighted": 0.513018, + "ap": 0.514585, + "ap_weighted": 0.514585 + }, + { + "accuracy": 0.55511, + "f1": 0.550414, + "f1_weighted": 0.551059, + "ap": 0.536297, + "ap_weighted": 0.536297 + }, + { + "accuracy": 0.49499, + "f1": 0.421361, + "f1_weighted": 0.418466, + "ap": 0.50706, + "ap_weighted": 0.50706 + }, + { + "accuracy": 0.48998, + "f1": 0.476002, + "f1_weighted": 0.474801, + "ap": 0.503286, + "ap_weighted": 0.503286 + }, + { + "accuracy": 0.527054, + "f1": 0.509268, + "f1_weighted": 0.507957, + "ap": 0.523375, + "ap_weighted": 0.523375 + } + ], + "main_score": 0.509519, + "hf_subset": "pa", + "languages": [ + "pan-Guru" + ] + }, + { + "accuracy": 0.520842, + "f1": 0.510871, + "f1_weighted": 0.511101, + "ap": 0.518244, + "ap_weighted": 0.518244, + "scores_per_experiment": [ + { + "accuracy": 0.532064, + "f1": 0.524847, + "f1_weighted": 0.524025, + "ap": 0.525492, + "ap_weighted": 0.525492 + }, + { + "accuracy": 0.508016, + "f1": 0.507873, + "f1_weighted": 0.507756, + "ap": 0.511263, + "ap_weighted": 0.511263 + }, + { + "accuracy": 0.47495, + "f1": 0.468728, + "f1_weighted": 0.469535, + "ap": 0.494354, + "ap_weighted": 0.494354 + }, + { + "accuracy": 0.528056, + "f1": 0.527949, + "f1_weighted": 0.528049, + "ap": 0.521744, + "ap_weighted": 0.521744 + }, + { + "accuracy": 0.51002, + "f1": 0.455038, + "f1_weighted": 0.457466, + "ap": 0.509865, + "ap_weighted": 0.509865 + }, + { + "accuracy": 0.495992, + "f1": 0.495809, + "f1_weighted": 0.495944, + "ap": 0.504942, + "ap_weighted": 0.504942 + }, + { + "accuracy": 0.546092, + "f1": 0.531351, + "f1_weighted": 0.532517, + "ap": 0.53027, + "ap_weighted": 0.53027 + }, + { + "accuracy": 0.537074, + "f1": 0.536001, + "f1_weighted": 0.535688, + "ap": 0.527525, + "ap_weighted": 0.527525 + }, + { + "accuracy": 0.52505, + "f1": 0.51016, + "f1_weighted": 0.508962, + "ap": 0.521989, + "ap_weighted": 0.521989 + }, + { + "accuracy": 0.551102, + "f1": 0.550956, + "f1_weighted": 0.55107, + "ap": 0.534992, + "ap_weighted": 0.534992 + } + ], + "main_score": 0.520842, + "hf_subset": "hi", + "languages": [ + "hin-Deva" + ] + }, + { + "accuracy": 0.532565, + "f1": 0.524644, + "f1_weighted": 0.52453, + "ap": 0.524924, + "ap_weighted": 0.524924, + "scores_per_experiment": [ + { + "accuracy": 0.53507, + "f1": 0.531592, + "f1_weighted": 0.531026, + "ap": 0.526807, + "ap_weighted": 0.526807 + }, + { + "accuracy": 0.534068, + "f1": 0.515574, + "f1_weighted": 0.514246, + "ap": 0.527725, + "ap_weighted": 0.527725 + }, + { + "accuracy": 0.54008, + "f1": 0.531764, + "f1_weighted": 0.53264, + "ap": 0.527324, + "ap_weighted": 0.527324 + }, + { + "accuracy": 0.519038, + "f1": 0.51903, + "f1_weighted": 0.519057, + "ap": 0.516915, + "ap_weighted": 0.516915 + }, + { + "accuracy": 0.541082, + "f1": 0.511906, + "f1_weighted": 0.51358, + "ap": 0.526842, + "ap_weighted": 0.526842 + }, + { + "accuracy": 0.492986, + "f1": 0.48742, + "f1_weighted": 0.48667, + "ap": 0.504325, + "ap_weighted": 0.504325 + }, + { + "accuracy": 0.558116, + "f1": 0.557134, + "f1_weighted": 0.557427, + "ap": 0.538815, + "ap_weighted": 0.538815 + }, + { + "accuracy": 0.52505, + "f1": 0.524039, + "f1_weighted": 0.523731, + "ap": 0.520646, + "ap_weighted": 0.520646 + }, + { + "accuracy": 0.544088, + "f1": 0.531906, + "f1_weighted": 0.530847, + "ap": 0.533446, + "ap_weighted": 0.533446 + }, + { + "accuracy": 0.536072, + "f1": 0.536072, + "f1_weighted": 0.536078, + "ap": 0.526397, + "ap_weighted": 0.526397 + } + ], + "main_score": 0.532565, + "hf_subset": "kn", + "languages": [ + "kan-Knda" + ] + }, + { + "accuracy": 0.51022, + "f1": 0.475161, + "f1_weighted": 0.473913, + "ap": 0.514617, + "ap_weighted": 0.514617, + "scores_per_experiment": [ + { + "accuracy": 0.546092, + "f1": 0.542616, + "f1_weighted": 0.543176, + "ap": 0.531216, + "ap_weighted": 0.531216 + }, + { + "accuracy": 0.524048, + "f1": 0.521198, + "f1_weighted": 0.52068, + "ap": 0.520383, + "ap_weighted": 0.520383 + }, + { + "accuracy": 0.492986, + "f1": 0.373392, + "f1_weighted": 0.369552, + "ap": 0.506626, + "ap_weighted": 0.506626 + }, + { + "accuracy": 0.517034, + "f1": 0.504293, + "f1_weighted": 0.503178, + "ap": 0.517262, + "ap_weighted": 0.517262 + }, + { + "accuracy": 0.536072, + "f1": 0.508839, + "f1_weighted": 0.510461, + "ap": 0.524179, + "ap_weighted": 0.524179 + }, + { + "accuracy": 0.44489, + "f1": 0.422145, + "f1_weighted": 0.420536, + "ap": 0.485466, + "ap_weighted": 0.485466 + }, + { + "accuracy": 0.546092, + "f1": 0.53203, + "f1_weighted": 0.530892, + "ap": 0.534972, + "ap_weighted": 0.534972 + }, + { + "accuracy": 0.493988, + "f1": 0.399997, + "f1_weighted": 0.396666, + "ap": 0.506834, + "ap_weighted": 0.506834 + }, + { + "accuracy": 0.476954, + "f1": 0.430156, + "f1_weighted": 0.427865, + "ap": 0.498406, + "ap_weighted": 0.498406 + }, + { + "accuracy": 0.524048, + "f1": 0.516947, + "f1_weighted": 0.516126, + "ap": 0.520824, + "ap_weighted": 0.520824 + } + ], + "main_score": 0.51022, + "hf_subset": "te", + "languages": [ + "tel-Telu" + ] + }, + { + "accuracy": 0.508417, + "f1": 0.475052, + "f1_weighted": 0.474158, + "ap": 0.512571, + "ap_weighted": 0.512571, + "scores_per_experiment": [ + { + "accuracy": 0.546092, + "f1": 0.538858, + "f1_weighted": 0.538048, + "ap": 0.53407, + "ap_weighted": 0.53407 + }, + { + "accuracy": 0.488978, + "f1": 0.463864, + "f1_weighted": 0.462236, + "ap": 0.503181, + "ap_weighted": 0.503181 + }, + { + "accuracy": 0.48998, + "f1": 0.367618, + "f1_weighted": 0.363715, + "ap": 0.505269, + "ap_weighted": 0.505269 + }, + { + "accuracy": 0.516032, + "f1": 0.50209, + "f1_weighted": 0.503259, + "ap": 0.514047, + "ap_weighted": 0.514047 + }, + { + "accuracy": 0.511022, + "f1": 0.486522, + "f1_weighted": 0.488095, + "ap": 0.511087, + "ap_weighted": 0.511087 + }, + { + "accuracy": 0.492986, + "f1": 0.492935, + "f1_weighted": 0.493006, + "ap": 0.503535, + "ap_weighted": 0.503535 + }, + { + "accuracy": 0.547094, + "f1": 0.540223, + "f1_weighted": 0.541012, + "ap": 0.53141, + "ap_weighted": 0.53141 + }, + { + "accuracy": 0.486974, + "f1": 0.394509, + "f1_weighted": 0.39119, + "ap": 0.503559, + "ap_weighted": 0.503559 + }, + { + "accuracy": 0.497996, + "f1": 0.473085, + "f1_weighted": 0.471477, + "ap": 0.507589, + "ap_weighted": 0.507589 + }, + { + "accuracy": 0.507014, + "f1": 0.490816, + "f1_weighted": 0.489542, + "ap": 0.511966, + "ap_weighted": 0.511966 + } + ], + "main_score": 0.508417, + "hf_subset": "or", + "languages": [ + "ory-Orya" + ] + }, + { + "accuracy": 0.512525, + "f1": 0.47476, + "f1_weighted": 0.473978, + "ap": 0.514421, + "ap_weighted": 0.514421, + "scores_per_experiment": [ + { + "accuracy": 0.532064, + "f1": 0.521101, + "f1_weighted": 0.522117, + "ap": 0.522725, + "ap_weighted": 0.522725 + }, + { + "accuracy": 0.491984, + "f1": 0.455298, + "f1_weighted": 0.453315, + "ap": 0.504914, + "ap_weighted": 0.504914 + }, + { + "accuracy": 0.495992, + "f1": 0.369509, + "f1_weighted": 0.365547, + "ap": 0.508262, + "ap_weighted": 0.508262 + }, + { + "accuracy": 0.513026, + "f1": 0.512586, + "f1_weighted": 0.512791, + "ap": 0.513518, + "ap_weighted": 0.513518 + }, + { + "accuracy": 0.506012, + "f1": 0.467312, + "f1_weighted": 0.469326, + "ap": 0.508182, + "ap_weighted": 0.508182 + }, + { + "accuracy": 0.509018, + "f1": 0.506728, + "f1_weighted": 0.506257, + "ap": 0.512168, + "ap_weighted": 0.512168 + }, + { + "accuracy": 0.533066, + "f1": 0.512286, + "f1_weighted": 0.513699, + "ap": 0.522794, + "ap_weighted": 0.522794 + }, + { + "accuracy": 0.498998, + "f1": 0.397273, + "f1_weighted": 0.3938, + "ap": 0.509585, + "ap_weighted": 0.509585 + }, + { + "accuracy": 0.498998, + "f1": 0.464298, + "f1_weighted": 0.462385, + "ap": 0.508362, + "ap_weighted": 0.508362 + }, + { + "accuracy": 0.546092, + "f1": 0.541205, + "f1_weighted": 0.540541, + "ap": 0.533704, + "ap_weighted": 0.533704 + } + ], + "main_score": 0.512525, + "hf_subset": "gu", + "languages": [ + "guj-Gujr" + ] + }, + { + "accuracy": 0.518537, + "f1": 0.503131, + "f1_weighted": 0.503412, + "ap": 0.51727, + "ap_weighted": 0.51727, + "scores_per_experiment": [ + { + "accuracy": 0.504008, + "f1": 0.486358, + "f1_weighted": 0.487694, + "ap": 0.507769, + "ap_weighted": 0.507769 + }, + { + "accuracy": 0.509018, + "f1": 0.465089, + "f1_weighted": 0.462939, + "ap": 0.513995, + "ap_weighted": 0.513995 + }, + { + "accuracy": 0.521042, + "f1": 0.487307, + "f1_weighted": 0.489152, + "ap": 0.515988, + "ap_weighted": 0.515988 + }, + { + "accuracy": 0.543086, + "f1": 0.54307, + "f1_weighted": 0.543031, + "ap": 0.530544, + "ap_weighted": 0.530544 + }, + { + "accuracy": 0.548096, + "f1": 0.518098, + "f1_weighted": 0.519785, + "ap": 0.530688, + "ap_weighted": 0.530688 + }, + { + "accuracy": 0.460922, + "f1": 0.457435, + "f1_weighted": 0.456825, + "ap": 0.489798, + "ap_weighted": 0.489798 + }, + { + "accuracy": 0.54008, + "f1": 0.53251, + "f1_weighted": 0.533344, + "ap": 0.527383, + "ap_weighted": 0.527383 + }, + { + "accuracy": 0.548096, + "f1": 0.536926, + "f1_weighted": 0.537935, + "ap": 0.531632, + "ap_weighted": 0.531632 + }, + { + "accuracy": 0.46493, + "f1": 0.462579, + "f1_weighted": 0.462081, + "ap": 0.491331, + "ap_weighted": 0.491331 + }, + { + "accuracy": 0.546092, + "f1": 0.541942, + "f1_weighted": 0.54133, + "ap": 0.533577, + "ap_weighted": 0.533577 + } + ], + "main_score": 0.518537, + "hf_subset": "ur", + "languages": [ + "urd-Arab" + ] + } + ] + }, + "evaluation_time": 21.06170082092285, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/IndonesianIdClickbaitClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/IndonesianIdClickbaitClassification.json new file mode 100644 index 0000000000..e760262f8e --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/IndonesianIdClickbaitClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "9fa4d0824015fe537ae2c8166781f5c79873da2c", + "task_name": "IndonesianIdClickbaitClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.546533, + "f1": 0.539231, + "f1_weighted": 0.54634, + "ap": 0.445509, + "ap_weighted": 0.445509, + "scores_per_experiment": [ + { + "accuracy": 0.597168, + "f1": 0.589439, + "f1_weighted": 0.598516, + "ap": 0.473875, + "ap_weighted": 0.473875 + }, + { + "accuracy": 0.508301, + "f1": 0.505959, + "f1_weighted": 0.51144, + "ap": 0.424649, + "ap_weighted": 0.424649 + }, + { + "accuracy": 0.49707, + "f1": 0.496838, + "f1_weighted": 0.49858, + "ap": 0.422775, + "ap_weighted": 0.422775 + }, + { + "accuracy": 0.534668, + "f1": 0.533706, + "f1_weighted": 0.530295, + "ap": 0.449794, + "ap_weighted": 0.449794 + }, + { + "accuracy": 0.588379, + "f1": 0.583075, + "f1_weighted": 0.590652, + "ap": 0.469844, + "ap_weighted": 0.469844 + }, + { + "accuracy": 0.470703, + "f1": 0.470436, + "f1_weighted": 0.472353, + "ap": 0.409891, + "ap_weighted": 0.409891 + }, + { + "accuracy": 0.568359, + "f1": 0.557832, + "f1_weighted": 0.568826, + "ap": 0.452133, + "ap_weighted": 0.452133 + }, + { + "accuracy": 0.602051, + "f1": 0.599322, + "f1_weighted": 0.60465, + "ap": 0.482688, + "ap_weighted": 0.482688 + }, + { + "accuracy": 0.544922, + "f1": 0.525922, + "f1_weighted": 0.541215, + "ap": 0.433287, + "ap_weighted": 0.433287 + }, + { + "accuracy": 0.553711, + "f1": 0.529782, + "f1_weighted": 0.546874, + "ap": 0.436151, + "ap_weighted": 0.436151 + } + ], + "main_score": 0.539231, + "hf_subset": "default", + "languages": [ + "ind-Latn" + ] + } + ] + }, + "evaluation_time": 2.4304685592651367, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/IndonesianMongabayConservationClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/IndonesianMongabayConservationClassification.json new file mode 100644 index 0000000000..a6c5411a10 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/IndonesianMongabayConservationClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "c9e9f2c09836bfec57c543ab65983f3398e9657a", + "task_name": "IndonesianMongabayConservationClassification", + "mteb_version": "1.34.14", + "scores": { + "validation": [ + { + "accuracy": 0.307927, + "f1": 0.301251, + "f1_weighted": 0.303515, + "scores_per_experiment": [ + { + "accuracy": 0.306911, + "f1": 0.296447, + "f1_weighted": 0.285944 + }, + { + "accuracy": 0.304878, + "f1": 0.300435, + "f1_weighted": 0.295835 + }, + { + "accuracy": 0.337398, + "f1": 0.335161, + "f1_weighted": 0.337178 + }, + { + "accuracy": 0.29065, + "f1": 0.28587, + "f1_weighted": 0.294439 + }, + { + "accuracy": 0.351626, + "f1": 0.350763, + "f1_weighted": 0.35197 + }, + { + "accuracy": 0.231707, + "f1": 0.229022, + "f1_weighted": 0.230761 + }, + { + "accuracy": 0.317073, + "f1": 0.317581, + "f1_weighted": 0.319318 + }, + { + "accuracy": 0.325203, + "f1": 0.321552, + "f1_weighted": 0.32702 + }, + { + "accuracy": 0.337398, + "f1": 0.317426, + "f1_weighted": 0.331065 + }, + { + "accuracy": 0.276423, + "f1": 0.258251, + "f1_weighted": 0.261617 + } + ], + "main_score": 0.301251, + "hf_subset": "default", + "languages": [ + "ind-Latn" + ] + } + ], + "test": [ + { + "accuracy": 0.306858, + "f1": 0.303629, + "f1_weighted": 0.303461, + "scores_per_experiment": [ + { + "accuracy": 0.288639, + "f1": 0.283941, + "f1_weighted": 0.269926 + }, + { + "accuracy": 0.295803, + "f1": 0.29472, + "f1_weighted": 0.289174 + }, + { + "accuracy": 0.318321, + "f1": 0.315935, + "f1_weighted": 0.31697 + }, + { + "accuracy": 0.287615, + "f1": 0.2832, + "f1_weighted": 0.291684 + }, + { + "accuracy": 0.336745, + "f1": 0.33633, + "f1_weighted": 0.340971 + }, + { + "accuracy": 0.298874, + "f1": 0.293298, + "f1_weighted": 0.299897 + }, + { + "accuracy": 0.291709, + "f1": 0.292668, + "f1_weighted": 0.289431 + }, + { + "accuracy": 0.341863, + "f1": 0.339392, + "f1_weighted": 0.345041 + }, + { + "accuracy": 0.32651, + "f1": 0.321138, + "f1_weighted": 0.321324 + }, + { + "accuracy": 0.282497, + "f1": 0.275671, + "f1_weighted": 0.270196 + } + ], + "main_score": 0.303629, + "hf_subset": "default", + "languages": [ + "ind-Latn" + ] + } + ] + }, + "evaluation_time": 20.44131302833557, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/InsurancePolicyInterpretationLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/InsurancePolicyInterpretationLegalBenchClassification.json new file mode 100644 index 0000000000..c9335a9514 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/InsurancePolicyInterpretationLegalBenchClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "InsurancePolicyInterpretationLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.203008, + "f1": 0.1125, + "f1_weighted": 0.068515, + "scores_per_experiment": [ + { + "accuracy": 0.203008, + "f1": 0.1125, + "f1_weighted": 0.068515 + }, + { + "accuracy": 0.203008, + "f1": 0.1125, + "f1_weighted": 0.068515 + }, + { + "accuracy": 0.203008, + "f1": 0.1125, + "f1_weighted": 0.068515 + }, + { + "accuracy": 0.203008, + "f1": 0.1125, + "f1_weighted": 0.068515 + }, + { + "accuracy": 0.203008, + "f1": 0.1125, + "f1_weighted": 0.068515 + }, + { + "accuracy": 0.203008, + "f1": 0.1125, + "f1_weighted": 0.068515 + }, + { + "accuracy": 0.203008, + "f1": 0.1125, + "f1_weighted": 0.068515 + }, + { + "accuracy": 0.203008, + "f1": 0.1125, + "f1_weighted": 0.068515 + }, + { + "accuracy": 0.203008, + "f1": 0.1125, + "f1_weighted": 0.068515 + }, + { + "accuracy": 0.203008, + "f1": 0.1125, + "f1_weighted": 0.068515 + } + ], + "main_score": 0.203008, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.776421308517456, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/InternationalCitizenshipQuestionsLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/InternationalCitizenshipQuestionsLegalBenchClassification.json new file mode 100644 index 0000000000..ad8361883a --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/InternationalCitizenshipQuestionsLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "InternationalCitizenshipQuestionsLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.62207, + "f1": 0.571754, + "f1_weighted": 0.59469, + "ap": 0.47864, + "ap_weighted": 0.47864, + "scores_per_experiment": [ + { + "accuracy": 0.62207, + "f1": 0.571754, + "f1_weighted": 0.59469, + "ap": 0.47864, + "ap_weighted": 0.47864 + }, + { + "accuracy": 0.62207, + "f1": 0.571754, + "f1_weighted": 0.59469, + "ap": 0.47864, + "ap_weighted": 0.47864 + }, + { + "accuracy": 0.62207, + "f1": 0.571754, + "f1_weighted": 0.59469, + "ap": 0.47864, + "ap_weighted": 0.47864 + }, + { + "accuracy": 0.62207, + "f1": 0.571754, + "f1_weighted": 0.59469, + "ap": 0.47864, + "ap_weighted": 0.47864 + }, + { + "accuracy": 0.62207, + "f1": 0.571754, + "f1_weighted": 0.59469, + "ap": 0.47864, + "ap_weighted": 0.47864 + }, + { + "accuracy": 0.62207, + "f1": 0.571754, + "f1_weighted": 0.59469, + "ap": 0.47864, + "ap_weighted": 0.47864 + }, + { + "accuracy": 0.62207, + "f1": 0.571754, + "f1_weighted": 0.59469, + "ap": 0.47864, + "ap_weighted": 0.47864 + }, + { + "accuracy": 0.62207, + "f1": 0.571754, + "f1_weighted": 0.59469, + "ap": 0.47864, + "ap_weighted": 0.47864 + }, + { + "accuracy": 0.62207, + "f1": 0.571754, + "f1_weighted": 0.59469, + "ap": 0.47864, + "ap_weighted": 0.47864 + }, + { + "accuracy": 0.62207, + "f1": 0.571754, + "f1_weighted": 0.59469, + "ap": 0.47864, + "ap_weighted": 0.47864 + } + ], + "main_score": 0.62207, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.7848105430603027, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/IsiZuluNewsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/IsiZuluNewsClassification.json new file mode 100644 index 0000000000..84191a8d93 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/IsiZuluNewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "55caf0e52693a1ea63b15a4980a73fc137fb862b", + "task_name": "IsiZuluNewsClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.233777, + "f1": 0.205107, + "f1_weighted": 0.245409, + "scores_per_experiment": [ + { + "accuracy": 0.256649, + "f1": 0.198774, + "f1_weighted": 0.274189 + }, + { + "accuracy": 0.226064, + "f1": 0.209544, + "f1_weighted": 0.233374 + }, + { + "accuracy": 0.198138, + "f1": 0.193716, + "f1_weighted": 0.198173 + }, + { + "accuracy": 0.255319, + "f1": 0.220314, + "f1_weighted": 0.271952 + }, + { + "accuracy": 0.235372, + "f1": 0.206689, + "f1_weighted": 0.260794 + }, + { + "accuracy": 0.243351, + "f1": 0.217504, + "f1_weighted": 0.254854 + }, + { + "accuracy": 0.212766, + "f1": 0.226922, + "f1_weighted": 0.216883 + }, + { + "accuracy": 0.226064, + "f1": 0.177684, + "f1_weighted": 0.241224 + }, + { + "accuracy": 0.226064, + "f1": 0.184012, + "f1_weighted": 0.229165 + }, + { + "accuracy": 0.257979, + "f1": 0.215905, + "f1_weighted": 0.27348 + } + ], + "main_score": 0.233777, + "hf_subset": "default", + "languages": [ + "zul-Latn" + ] + } + ] + }, + "evaluation_time": 2.002426862716675, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ItaCaseholdClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ItaCaseholdClassification.json new file mode 100644 index 0000000000..686b730b62 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/ItaCaseholdClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "fafcfc4fee815f7017848e54b26c47ece8ff1626", + "task_name": "ItaCaseholdClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.547511, + "f1": 0.20056, + "f1_weighted": 0.509654, + "scores_per_experiment": [ + { + "accuracy": 0.542986, + "f1": 0.199023, + "f1_weighted": 0.507746 + }, + { + "accuracy": 0.515837, + "f1": 0.198131, + "f1_weighted": 0.484179 + }, + { + "accuracy": 0.561086, + "f1": 0.19819, + "f1_weighted": 0.517024 + }, + { + "accuracy": 0.529412, + "f1": 0.19388, + "f1_weighted": 0.507586 + }, + { + "accuracy": 0.538462, + "f1": 0.201279, + "f1_weighted": 0.50332 + }, + { + "accuracy": 0.58371, + "f1": 0.202916, + "f1_weighted": 0.532633 + }, + { + "accuracy": 0.511312, + "f1": 0.195636, + "f1_weighted": 0.472857 + }, + { + "accuracy": 0.552036, + "f1": 0.207869, + "f1_weighted": 0.512025 + }, + { + "accuracy": 0.561086, + "f1": 0.208772, + "f1_weighted": 0.529573 + }, + { + "accuracy": 0.579186, + "f1": 0.199905, + "f1_weighted": 0.529598 + } + ], + "main_score": 0.547511, + "hf_subset": "default", + "languages": [ + "ita-Latn" + ] + } + ] + }, + "evaluation_time": 63.106486082077026, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/Itacola.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/Itacola.json new file mode 100644 index 0000000000..783c624e3a --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/Itacola.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "f8f98e5c4d3059cf1a00c8eb3d70aa271423f636", + "task_name": "Itacola", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.509744, + "f1": 0.43236, + "f1_weighted": 0.565171, + "ap": 0.842376, + "ap_weighted": 0.842376, + "scores_per_experiment": [ + { + "accuracy": 0.584615, + "f1": 0.47218, + "f1_weighted": 0.638834, + "ap": 0.843492, + "ap_weighted": 0.843492 + }, + { + "accuracy": 0.619487, + "f1": 0.48223, + "f1_weighted": 0.664602, + "ap": 0.842682, + "ap_weighted": 0.842682 + }, + { + "accuracy": 0.328205, + "f1": 0.321349, + "f1_weighted": 0.368014, + "ap": 0.838822, + "ap_weighted": 0.838822 + }, + { + "accuracy": 0.578462, + "f1": 0.475492, + "f1_weighted": 0.634475, + "ap": 0.846769, + "ap_weighted": 0.846769 + }, + { + "accuracy": 0.531282, + "f1": 0.437562, + "f1_weighted": 0.594625, + "ap": 0.835167, + "ap_weighted": 0.835167 + }, + { + "accuracy": 0.547692, + "f1": 0.450529, + "f1_weighted": 0.608597, + "ap": 0.839078, + "ap_weighted": 0.839078 + }, + { + "accuracy": 0.362051, + "f1": 0.338757, + "f1_weighted": 0.423661, + "ap": 0.826775, + "ap_weighted": 0.826775 + }, + { + "accuracy": 0.644103, + "f1": 0.523838, + "f1_weighted": 0.687545, + "ap": 0.859728, + "ap_weighted": 0.859728 + }, + { + "accuracy": 0.447179, + "f1": 0.403442, + "f1_weighted": 0.513945, + "ap": 0.840035, + "ap_weighted": 0.840035 + }, + { + "accuracy": 0.454359, + "f1": 0.418221, + "f1_weighted": 0.517414, + "ap": 0.851213, + "ap_weighted": 0.851213 + } + ], + "main_score": 0.509744, + "hf_subset": "default", + "languages": [ + "ita-Latn" + ] + } + ] + }, + "evaluation_time": 1.2928755283355713, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/JCrewBlockerLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/JCrewBlockerLegalBenchClassification.json new file mode 100644 index 0000000000..f40acec208 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/JCrewBlockerLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "JCrewBlockerLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.833333, + "f1": 0.712596, + "f1_weighted": 0.836783, + "ap": 0.900673, + "ap_weighted": 0.900673, + "scores_per_experiment": [ + { + "accuracy": 0.833333, + "f1": 0.712596, + "f1_weighted": 0.836783, + "ap": 0.900673, + "ap_weighted": 0.900673 + }, + { + "accuracy": 0.833333, + "f1": 0.712596, + "f1_weighted": 0.836783, + "ap": 0.900673, + "ap_weighted": 0.900673 + }, + { + "accuracy": 0.833333, + "f1": 0.712596, + "f1_weighted": 0.836783, + "ap": 0.900673, + "ap_weighted": 0.900673 + }, + { + "accuracy": 0.833333, + "f1": 0.712596, + "f1_weighted": 0.836783, + "ap": 0.900673, + "ap_weighted": 0.900673 + }, + { + "accuracy": 0.833333, + "f1": 0.712596, + "f1_weighted": 0.836783, + "ap": 0.900673, + "ap_weighted": 0.900673 + }, + { + "accuracy": 0.833333, + "f1": 0.712596, + "f1_weighted": 0.836783, + "ap": 0.900673, + "ap_weighted": 0.900673 + }, + { + "accuracy": 0.833333, + "f1": 0.712596, + "f1_weighted": 0.836783, + "ap": 0.900673, + "ap_weighted": 0.900673 + }, + { + "accuracy": 0.833333, + "f1": 0.712596, + "f1_weighted": 0.836783, + "ap": 0.900673, + "ap_weighted": 0.900673 + }, + { + "accuracy": 0.833333, + "f1": 0.712596, + "f1_weighted": 0.836783, + "ap": 0.900673, + "ap_weighted": 0.900673 + }, + { + "accuracy": 0.833333, + "f1": 0.712596, + "f1_weighted": 0.836783, + "ap": 0.900673, + "ap_weighted": 0.900673 + } + ], + "main_score": 0.833333, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.3791935443878174, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/JDReview.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/JDReview.json new file mode 100644 index 0000000000..3015db5140 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/JDReview.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "b7c64bd89eb87f8ded463478346f76731f07bf8b", + "task_name": "JDReview", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.543527, + "f1": 0.469201, + "f1_weighted": 0.59113, + "ap": 0.190846, + "ap_weighted": 0.190846, + "scores_per_experiment": [ + { + "accuracy": 0.564728, + "f1": 0.482004, + "f1_weighted": 0.61211, + "ap": 0.191676, + "ap_weighted": 0.191676 + }, + { + "accuracy": 0.393996, + "f1": 0.374543, + "f1_weighted": 0.443872, + "ap": 0.176864, + "ap_weighted": 0.176864 + }, + { + "accuracy": 0.58349, + "f1": 0.49511, + "f1_weighted": 0.627878, + "ap": 0.19587, + "ap_weighted": 0.19587 + }, + { + "accuracy": 0.574109, + "f1": 0.461192, + "f1_weighted": 0.616221, + "ap": 0.17867, + "ap_weighted": 0.17867 + }, + { + "accuracy": 0.532833, + "f1": 0.469626, + "f1_weighted": 0.584704, + "ap": 0.192943, + "ap_weighted": 0.192943 + }, + { + "accuracy": 0.532833, + "f1": 0.469626, + "f1_weighted": 0.584704, + "ap": 0.192943, + "ap_weighted": 0.192943 + }, + { + "accuracy": 0.547842, + "f1": 0.478793, + "f1_weighted": 0.598028, + "ap": 0.194861, + "ap_weighted": 0.194861 + }, + { + "accuracy": 0.58349, + "f1": 0.49511, + "f1_weighted": 0.627878, + "ap": 0.19587, + "ap_weighted": 0.19587 + }, + { + "accuracy": 0.598499, + "f1": 0.501617, + "f1_weighted": 0.639726, + "ap": 0.19648, + "ap_weighted": 0.19648 + }, + { + "accuracy": 0.523452, + "f1": 0.464385, + "f1_weighted": 0.576179, + "ap": 0.192285, + "ap_weighted": 0.192285 + } + ], + "main_score": 0.543527, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 3.815296173095703, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/JavaneseIMDBClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/JavaneseIMDBClassification.json new file mode 100644 index 0000000000..d68bdaa5bd --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/JavaneseIMDBClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "11bef3dfce0ce107eb5e276373dcd28759ce85ee", + "task_name": "JavaneseIMDBClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.518652, + "f1": 0.515651, + "f1_weighted": 0.515651, + "ap": 0.510223, + "ap_weighted": 0.510223, + "scores_per_experiment": [ + { + "accuracy": 0.535645, + "f1": 0.520754, + "f1_weighted": 0.520754, + "ap": 0.519785, + "ap_weighted": 0.519785 + }, + { + "accuracy": 0.491211, + "f1": 0.491054, + "f1_weighted": 0.491054, + "ap": 0.49568, + "ap_weighted": 0.49568 + }, + { + "accuracy": 0.51709, + "f1": 0.515865, + "f1_weighted": 0.515865, + "ap": 0.50881, + "ap_weighted": 0.50881 + }, + { + "accuracy": 0.503906, + "f1": 0.503902, + "f1_weighted": 0.503902, + "ap": 0.501968, + "ap_weighted": 0.501968 + }, + { + "accuracy": 0.549316, + "f1": 0.545445, + "f1_weighted": 0.545445, + "ap": 0.527641, + "ap_weighted": 0.527641 + }, + { + "accuracy": 0.534668, + "f1": 0.530051, + "f1_weighted": 0.530051, + "ap": 0.518833, + "ap_weighted": 0.518833 + }, + { + "accuracy": 0.535645, + "f1": 0.534643, + "f1_weighted": 0.534643, + "ap": 0.519223, + "ap_weighted": 0.519223 + }, + { + "accuracy": 0.524902, + "f1": 0.523744, + "f1_weighted": 0.523744, + "ap": 0.513139, + "ap_weighted": 0.513139 + }, + { + "accuracy": 0.491211, + "f1": 0.490394, + "f1_weighted": 0.490394, + "ap": 0.495677, + "ap_weighted": 0.495677 + }, + { + "accuracy": 0.50293, + "f1": 0.500662, + "f1_weighted": 0.500662, + "ap": 0.501472, + "ap_weighted": 0.501472 + } + ], + "main_score": 0.518652, + "hf_subset": "default", + "languages": [ + "jav-Latn" + ] + } + ] + }, + "evaluation_time": 11.121540546417236, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/KLUE-TC.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/KLUE-TC.json new file mode 100644 index 0000000000..67c50d98c6 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/KLUE-TC.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "349481ec73fff722f88e0453ca05c77a447d967c", + "task_name": "KLUE-TC", + "mteb_version": "1.34.14", + "scores": { + "validation": [ + { + "accuracy": 0.185498, + "f1": 0.158558, + "f1_weighted": 0.161765, + "scores_per_experiment": [ + { + "accuracy": 0.16748, + "f1": 0.1567, + "f1_weighted": 0.166847 + }, + { + "accuracy": 0.216309, + "f1": 0.176048, + "f1_weighted": 0.163297 + }, + { + "accuracy": 0.182129, + "f1": 0.146299, + "f1_weighted": 0.153626 + }, + { + "accuracy": 0.206055, + "f1": 0.188878, + "f1_weighted": 0.190814 + }, + { + "accuracy": 0.16748, + "f1": 0.14023, + "f1_weighted": 0.113718 + }, + { + "accuracy": 0.211914, + "f1": 0.193712, + "f1_weighted": 0.214119 + }, + { + "accuracy": 0.17627, + "f1": 0.129358, + "f1_weighted": 0.159261 + }, + { + "accuracy": 0.189941, + "f1": 0.16931, + "f1_weighted": 0.18571 + }, + { + "accuracy": 0.164551, + "f1": 0.137911, + "f1_weighted": 0.123438 + }, + { + "accuracy": 0.172852, + "f1": 0.147139, + "f1_weighted": 0.146823 + } + ], + "main_score": 0.185498, + "hf_subset": "default", + "languages": [ + "kor-Hang" + ] + } + ] + }, + "evaluation_time": 3.672860622406006, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/KannadaNewsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/KannadaNewsClassification.json new file mode 100644 index 0000000000..bc67f9d27b --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/KannadaNewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "a470711069906ac0a559defec3b89cb3725601bd", + "task_name": "KannadaNewsClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.452734, + "f1": 0.403645, + "f1_weighted": 0.460505, + "scores_per_experiment": [ + { + "accuracy": 0.385742, + "f1": 0.345987, + "f1_weighted": 0.358567 + }, + { + "accuracy": 0.534668, + "f1": 0.463158, + "f1_weighted": 0.533199 + }, + { + "accuracy": 0.329102, + "f1": 0.310499, + "f1_weighted": 0.338898 + }, + { + "accuracy": 0.483887, + "f1": 0.425156, + "f1_weighted": 0.504326 + }, + { + "accuracy": 0.48291, + "f1": 0.399447, + "f1_weighted": 0.49352 + }, + { + "accuracy": 0.381836, + "f1": 0.378409, + "f1_weighted": 0.410578 + }, + { + "accuracy": 0.468262, + "f1": 0.402695, + "f1_weighted": 0.463859 + }, + { + "accuracy": 0.32959, + "f1": 0.336927, + "f1_weighted": 0.364241 + }, + { + "accuracy": 0.567871, + "f1": 0.488821, + "f1_weighted": 0.576717 + }, + { + "accuracy": 0.563477, + "f1": 0.485347, + "f1_weighted": 0.561149 + } + ], + "main_score": 0.452734, + "hf_subset": "default", + "languages": [ + "kan-Knda" + ] + } + ] + }, + "evaluation_time": 2.334988832473755, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/KinopoiskClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/KinopoiskClassification.json new file mode 100644 index 0000000000..6d818ddad6 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/KinopoiskClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "5911f26666ac11af46cb9c6849d0dc80a378af24", + "task_name": "KinopoiskClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.353667, + "f1": 0.31251, + "f1_weighted": 0.31251, + "scores_per_experiment": [ + { + "accuracy": 0.374, + "f1": 0.358793, + "f1_weighted": 0.358793 + }, + { + "accuracy": 0.371333, + "f1": 0.326162, + "f1_weighted": 0.326162 + }, + { + "accuracy": 0.361333, + "f1": 0.353086, + "f1_weighted": 0.353086 + }, + { + "accuracy": 0.354667, + "f1": 0.331357, + "f1_weighted": 0.331357 + }, + { + "accuracy": 0.340667, + "f1": 0.256433, + "f1_weighted": 0.256433 + }, + { + "accuracy": 0.322667, + "f1": 0.30711, + "f1_weighted": 0.30711 + }, + { + "accuracy": 0.354, + "f1": 0.250921, + "f1_weighted": 0.250921 + }, + { + "accuracy": 0.35, + "f1": 0.289296, + "f1_weighted": 0.289296 + }, + { + "accuracy": 0.346667, + "f1": 0.309302, + "f1_weighted": 0.309302 + }, + { + "accuracy": 0.361333, + "f1": 0.342634, + "f1_weighted": 0.342634 + } + ], + "main_score": 0.353667, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 19.538918495178223, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/KorHateClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/KorHateClassification.json new file mode 100644 index 0000000000..5871b58c97 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/KorHateClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "bd1a7370caf712125fac1fda375834ca8ddefaca", + "task_name": "KorHateClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.377441, + "f1": 0.352052, + "f1_weighted": 0.369397, + "scores_per_experiment": [ + { + "accuracy": 0.384277, + "f1": 0.371891, + "f1_weighted": 0.389182 + }, + { + "accuracy": 0.346191, + "f1": 0.292051, + "f1_weighted": 0.314754 + }, + { + "accuracy": 0.430664, + "f1": 0.375197, + "f1_weighted": 0.405349 + }, + { + "accuracy": 0.330566, + "f1": 0.328739, + "f1_weighted": 0.332185 + }, + { + "accuracy": 0.318848, + "f1": 0.29737, + "f1_weighted": 0.297199 + }, + { + "accuracy": 0.397461, + "f1": 0.383137, + "f1_weighted": 0.402588 + }, + { + "accuracy": 0.409668, + "f1": 0.376871, + "f1_weighted": 0.405715 + }, + { + "accuracy": 0.400391, + "f1": 0.364042, + "f1_weighted": 0.393788 + }, + { + "accuracy": 0.322754, + "f1": 0.32214, + "f1_weighted": 0.322859 + }, + { + "accuracy": 0.433594, + "f1": 0.409077, + "f1_weighted": 0.430353 + } + ], + "main_score": 0.377441, + "hf_subset": "default", + "languages": [ + "kor-Hang" + ] + } + ] + }, + "evaluation_time": 2.4838995933532715, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/KorSarcasmClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/KorSarcasmClassification.json new file mode 100644 index 0000000000..002ee8a857 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/KorSarcasmClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "3d96e36e10a88d5b7a3f617cf8362d997504494b", + "task_name": "KorSarcasmClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.521729, + "f1": 0.516151, + "f1_weighted": 0.516164, + "ap": 0.512694, + "ap_weighted": 0.512694, + "scores_per_experiment": [ + { + "accuracy": 0.583984, + "f1": 0.580139, + "f1_weighted": 0.580217, + "ap": 0.549604, + "ap_weighted": 0.549604 + }, + { + "accuracy": 0.575684, + "f1": 0.573546, + "f1_weighted": 0.573605, + "ap": 0.543444, + "ap_weighted": 0.543444 + }, + { + "accuracy": 0.503906, + "f1": 0.47779, + "f1_weighted": 0.478018, + "ap": 0.500781, + "ap_weighted": 0.500781 + }, + { + "accuracy": 0.472168, + "f1": 0.465125, + "f1_weighted": 0.465006, + "ap": 0.485841, + "ap_weighted": 0.485841 + }, + { + "accuracy": 0.462891, + "f1": 0.462188, + "f1_weighted": 0.462226, + "ap": 0.481924, + "ap_weighted": 0.481924 + }, + { + "accuracy": 0.555176, + "f1": 0.555167, + "f1_weighted": 0.555163, + "ap": 0.529636, + "ap_weighted": 0.529636 + }, + { + "accuracy": 0.555176, + "f1": 0.554513, + "f1_weighted": 0.554546, + "ap": 0.529864, + "ap_weighted": 0.529864 + }, + { + "accuracy": 0.562012, + "f1": 0.550839, + "f1_weighted": 0.550701, + "ap": 0.533138, + "ap_weighted": 0.533138 + }, + { + "accuracy": 0.477051, + "f1": 0.473717, + "f1_weighted": 0.473635, + "ap": 0.488076, + "ap_weighted": 0.488076 + }, + { + "accuracy": 0.469238, + "f1": 0.468487, + "f1_weighted": 0.468526, + "ap": 0.484635, + "ap_weighted": 0.484635 + } + ], + "main_score": 0.521729, + "hf_subset": "default", + "languages": [ + "kor-Hang" + ] + } + ] + }, + "evaluation_time": 2.7858073711395264, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/KurdishSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/KurdishSentimentClassification.json new file mode 100644 index 0000000000..03df9b6e14 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/KurdishSentimentClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "f334d90a9f68cc3af78cc2a2ece6a3b69408124c", + "task_name": "KurdishSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.583392, + "f1": 0.552718, + "f1_weighted": 0.556468, + "ap": 0.579879, + "ap_weighted": 0.579879, + "scores_per_experiment": [ + { + "accuracy": 0.586814, + "f1": 0.533451, + "f1_weighted": 0.544807, + "ap": 0.570978, + "ap_weighted": 0.570978 + }, + { + "accuracy": 0.568193, + "f1": 0.525029, + "f1_weighted": 0.535334, + "ap": 0.561996, + "ap_weighted": 0.561996 + }, + { + "accuracy": 0.514343, + "f1": 0.507748, + "f1_weighted": 0.511848, + "ap": 0.540345, + "ap_weighted": 0.540345 + }, + { + "accuracy": 0.560644, + "f1": 0.556503, + "f1_weighted": 0.559587, + "ap": 0.566759, + "ap_weighted": 0.566759 + }, + { + "accuracy": 0.665325, + "f1": 0.627656, + "f1_weighted": 0.636179, + "ap": 0.621039, + "ap_weighted": 0.621039 + }, + { + "accuracy": 0.51686, + "f1": 0.446372, + "f1_weighted": 0.432155, + "ap": 0.56741, + "ap_weighted": 0.56741 + }, + { + "accuracy": 0.663312, + "f1": 0.626211, + "f1_weighted": 0.634686, + "ap": 0.61988, + "ap_weighted": 0.61988 + }, + { + "accuracy": 0.649723, + "f1": 0.621453, + "f1_weighted": 0.628898, + "ap": 0.613438, + "ap_weighted": 0.613438 + }, + { + "accuracy": 0.528435, + "f1": 0.512613, + "f1_weighted": 0.506294, + "ap": 0.561159, + "ap_weighted": 0.561159 + }, + { + "accuracy": 0.580272, + "f1": 0.570144, + "f1_weighted": 0.574892, + "ap": 0.575787, + "ap_weighted": 0.575787 + } + ], + "main_score": 0.583392, + "hf_subset": "default", + "languages": [ + "kur-Arab" + ] + } + ] + }, + "evaluation_time": 2.0652244091033936, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/LanguageClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/LanguageClassification.json new file mode 100644 index 0000000000..d210431dcc --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/LanguageClassification.json @@ -0,0 +1,92 @@ +{ + "dataset_revision": "aa56583bf2bc52b0565770607d6fc3faebecf9e2", + "task_name": "LanguageClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.734912, + "f1": 0.703868, + "f1_weighted": 0.704296, + "scores_per_experiment": [ + { + "accuracy": 0.75, + "f1": 0.732661, + "f1_weighted": 0.733043 + }, + { + "accuracy": 0.737793, + "f1": 0.718111, + "f1_weighted": 0.718436 + }, + { + "accuracy": 0.728027, + "f1": 0.695799, + "f1_weighted": 0.696316 + }, + { + "accuracy": 0.731934, + "f1": 0.694034, + "f1_weighted": 0.694511 + }, + { + "accuracy": 0.733887, + "f1": 0.700438, + "f1_weighted": 0.700972 + }, + { + "accuracy": 0.733887, + "f1": 0.69223, + "f1_weighted": 0.692614 + }, + { + "accuracy": 0.754395, + "f1": 0.724371, + "f1_weighted": 0.724791 + }, + { + "accuracy": 0.717285, + "f1": 0.68757, + "f1_weighted": 0.688027 + }, + { + "accuracy": 0.726562, + "f1": 0.687241, + "f1_weighted": 0.687679 + }, + { + "accuracy": 0.735352, + "f1": 0.706221, + "f1_weighted": 0.706572 + } + ], + "main_score": 0.734912, + "hf_subset": "default", + "languages": [ + "ara-Arab", + "bul-Cyrl", + "deu-Latn", + "ell-Grek", + "eng-Latn", + "spa-Latn", + "fra-Latn", + "hin-Deva", + "ita-Latn", + "jpn-Jpan", + "nld-Latn", + "pol-Latn", + "por-Latn", + "rus-Cyrl", + "swa-Latn", + "tha-Thai", + "tur-Latn", + "urd-Arab", + "vie-Latn", + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 8.414838552474976, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/LccSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/LccSentimentClassification.json new file mode 100644 index 0000000000..05afbf9203 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/LccSentimentClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "de7ba3406ee55ea2cc52a0a41408fa6aede6d3c6", + "task_name": "LccSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.382, + "f1": 0.356361, + "f1_weighted": 0.397017, + "scores_per_experiment": [ + { + "accuracy": 0.326667, + "f1": 0.328317, + "f1_weighted": 0.321977 + }, + { + "accuracy": 0.333333, + "f1": 0.3128, + "f1_weighted": 0.356689 + }, + { + "accuracy": 0.326667, + "f1": 0.308405, + "f1_weighted": 0.348581 + }, + { + "accuracy": 0.44, + "f1": 0.409895, + "f1_weighted": 0.453571 + }, + { + "accuracy": 0.373333, + "f1": 0.358119, + "f1_weighted": 0.387978 + }, + { + "accuracy": 0.42, + "f1": 0.360886, + "f1_weighted": 0.434019 + }, + { + "accuracy": 0.413333, + "f1": 0.368881, + "f1_weighted": 0.438374 + }, + { + "accuracy": 0.46, + "f1": 0.417915, + "f1_weighted": 0.471354 + }, + { + "accuracy": 0.413333, + "f1": 0.397265, + "f1_weighted": 0.428353 + }, + { + "accuracy": 0.313333, + "f1": 0.301129, + "f1_weighted": 0.329277 + } + ], + "main_score": 0.382, + "hf_subset": "default", + "languages": [ + "dan-Latn" + ] + } + ] + }, + "evaluation_time": 1.140068769454956, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsBenefitsLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsBenefitsLegalBenchClassification.json new file mode 100644 index 0000000000..19721bef85 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsBenefitsLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "LearnedHandsBenefitsLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.681818, + "f1": 0.681159, + "f1_weighted": 0.681159, + "ap": 0.627273, + "ap_weighted": 0.627273, + "scores_per_experiment": [ + { + "accuracy": 0.681818, + "f1": 0.681159, + "f1_weighted": 0.681159, + "ap": 0.627273, + "ap_weighted": 0.627273 + }, + { + "accuracy": 0.681818, + "f1": 0.681159, + "f1_weighted": 0.681159, + "ap": 0.627273, + "ap_weighted": 0.627273 + }, + { + "accuracy": 0.681818, + "f1": 0.681159, + "f1_weighted": 0.681159, + "ap": 0.627273, + "ap_weighted": 0.627273 + }, + { + "accuracy": 0.681818, + "f1": 0.681159, + "f1_weighted": 0.681159, + "ap": 0.627273, + "ap_weighted": 0.627273 + }, + { + "accuracy": 0.681818, + "f1": 0.681159, + "f1_weighted": 0.681159, + "ap": 0.627273, + "ap_weighted": 0.627273 + }, + { + "accuracy": 0.681818, + "f1": 0.681159, + "f1_weighted": 0.681159, + "ap": 0.627273, + "ap_weighted": 0.627273 + }, + { + "accuracy": 0.681818, + "f1": 0.681159, + "f1_weighted": 0.681159, + "ap": 0.627273, + "ap_weighted": 0.627273 + }, + { + "accuracy": 0.681818, + "f1": 0.681159, + "f1_weighted": 0.681159, + "ap": 0.627273, + "ap_weighted": 0.627273 + }, + { + "accuracy": 0.681818, + "f1": 0.681159, + "f1_weighted": 0.681159, + "ap": 0.627273, + "ap_weighted": 0.627273 + }, + { + "accuracy": 0.681818, + "f1": 0.681159, + "f1_weighted": 0.681159, + "ap": 0.627273, + "ap_weighted": 0.627273 + } + ], + "main_score": 0.681818, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.4119112491607666, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsBusinessLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsBusinessLegalBenchClassification.json new file mode 100644 index 0000000000..7a3f0edae7 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsBusinessLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "LearnedHandsBusinessLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.810345, + "f1": 0.806347, + "f1_weighted": 0.806347, + "ap": 0.790323, + "ap_weighted": 0.790323, + "scores_per_experiment": [ + { + "accuracy": 0.810345, + "f1": 0.806347, + "f1_weighted": 0.806347, + "ap": 0.790323, + "ap_weighted": 0.790323 + }, + { + "accuracy": 0.810345, + "f1": 0.806347, + "f1_weighted": 0.806347, + "ap": 0.790323, + "ap_weighted": 0.790323 + }, + { + "accuracy": 0.810345, + "f1": 0.806347, + "f1_weighted": 0.806347, + "ap": 0.790323, + "ap_weighted": 0.790323 + }, + { + "accuracy": 0.810345, + "f1": 0.806347, + "f1_weighted": 0.806347, + "ap": 0.790323, + "ap_weighted": 0.790323 + }, + { + "accuracy": 0.810345, + "f1": 0.806347, + "f1_weighted": 0.806347, + "ap": 0.790323, + "ap_weighted": 0.790323 + }, + { + "accuracy": 0.810345, + "f1": 0.806347, + "f1_weighted": 0.806347, + "ap": 0.790323, + "ap_weighted": 0.790323 + }, + { + "accuracy": 0.810345, + "f1": 0.806347, + "f1_weighted": 0.806347, + "ap": 0.790323, + "ap_weighted": 0.790323 + }, + { + "accuracy": 0.810345, + "f1": 0.806347, + "f1_weighted": 0.806347, + "ap": 0.790323, + "ap_weighted": 0.790323 + }, + { + "accuracy": 0.810345, + "f1": 0.806347, + "f1_weighted": 0.806347, + "ap": 0.790323, + "ap_weighted": 0.790323 + }, + { + "accuracy": 0.810345, + "f1": 0.806347, + "f1_weighted": 0.806347, + "ap": 0.790323, + "ap_weighted": 0.790323 + } + ], + "main_score": 0.810345, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.897629737854004, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsConsumerLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsConsumerLegalBenchClassification.json new file mode 100644 index 0000000000..8ab4af15ba --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsConsumerLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "LearnedHandsConsumerLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.636808, + "f1": 0.636382, + "f1_weighted": 0.636382, + "ap": 0.588495, + "ap_weighted": 0.588495, + "scores_per_experiment": [ + { + "accuracy": 0.636808, + "f1": 0.636382, + "f1_weighted": 0.636382, + "ap": 0.588495, + "ap_weighted": 0.588495 + }, + { + "accuracy": 0.636808, + "f1": 0.636382, + "f1_weighted": 0.636382, + "ap": 0.588495, + "ap_weighted": 0.588495 + }, + { + "accuracy": 0.636808, + "f1": 0.636382, + "f1_weighted": 0.636382, + "ap": 0.588495, + "ap_weighted": 0.588495 + }, + { + "accuracy": 0.636808, + "f1": 0.636382, + "f1_weighted": 0.636382, + "ap": 0.588495, + "ap_weighted": 0.588495 + }, + { + "accuracy": 0.636808, + "f1": 0.636382, + "f1_weighted": 0.636382, + "ap": 0.588495, + "ap_weighted": 0.588495 + }, + { + "accuracy": 0.636808, + "f1": 0.636382, + "f1_weighted": 0.636382, + "ap": 0.588495, + "ap_weighted": 0.588495 + }, + { + "accuracy": 0.636808, + "f1": 0.636382, + "f1_weighted": 0.636382, + "ap": 0.588495, + "ap_weighted": 0.588495 + }, + { + "accuracy": 0.636808, + "f1": 0.636382, + "f1_weighted": 0.636382, + "ap": 0.588495, + "ap_weighted": 0.588495 + }, + { + "accuracy": 0.636808, + "f1": 0.636382, + "f1_weighted": 0.636382, + "ap": 0.588495, + "ap_weighted": 0.588495 + }, + { + "accuracy": 0.636808, + "f1": 0.636382, + "f1_weighted": 0.636382, + "ap": 0.588495, + "ap_weighted": 0.588495 + } + ], + "main_score": 0.636808, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 4.252107620239258, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsCourtsLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsCourtsLegalBenchClassification.json new file mode 100644 index 0000000000..7b39041c35 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsCourtsLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "LearnedHandsCourtsLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.671875, + "f1": 0.671438, + "f1_weighted": 0.671438, + "ap": 0.613471, + "ap_weighted": 0.613471, + "scores_per_experiment": [ + { + "accuracy": 0.671875, + "f1": 0.671438, + "f1_weighted": 0.671438, + "ap": 0.613471, + "ap_weighted": 0.613471 + }, + { + "accuracy": 0.671875, + "f1": 0.671438, + "f1_weighted": 0.671438, + "ap": 0.613471, + "ap_weighted": 0.613471 + }, + { + "accuracy": 0.671875, + "f1": 0.671438, + "f1_weighted": 0.671438, + "ap": 0.613471, + "ap_weighted": 0.613471 + }, + { + "accuracy": 0.671875, + "f1": 0.671438, + "f1_weighted": 0.671438, + "ap": 0.613471, + "ap_weighted": 0.613471 + }, + { + "accuracy": 0.671875, + "f1": 0.671438, + "f1_weighted": 0.671438, + "ap": 0.613471, + "ap_weighted": 0.613471 + }, + { + "accuracy": 0.671875, + "f1": 0.671438, + "f1_weighted": 0.671438, + "ap": 0.613471, + "ap_weighted": 0.613471 + }, + { + "accuracy": 0.671875, + "f1": 0.671438, + "f1_weighted": 0.671438, + "ap": 0.613471, + "ap_weighted": 0.613471 + }, + { + "accuracy": 0.671875, + "f1": 0.671438, + "f1_weighted": 0.671438, + "ap": 0.613471, + "ap_weighted": 0.613471 + }, + { + "accuracy": 0.671875, + "f1": 0.671438, + "f1_weighted": 0.671438, + "ap": 0.613471, + "ap_weighted": 0.613471 + }, + { + "accuracy": 0.671875, + "f1": 0.671438, + "f1_weighted": 0.671438, + "ap": 0.613471, + "ap_weighted": 0.613471 + } + ], + "main_score": 0.671875, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.0329501628875732, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsCrimeLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsCrimeLegalBenchClassification.json new file mode 100644 index 0000000000..3239d2efc8 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsCrimeLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "LearnedHandsCrimeLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.796512, + "f1": 0.793998, + "f1_weighted": 0.793998, + "ap": 0.720266, + "ap_weighted": 0.720266, + "scores_per_experiment": [ + { + "accuracy": 0.796512, + "f1": 0.793998, + "f1_weighted": 0.793998, + "ap": 0.720266, + "ap_weighted": 0.720266 + }, + { + "accuracy": 0.796512, + "f1": 0.793998, + "f1_weighted": 0.793998, + "ap": 0.720266, + "ap_weighted": 0.720266 + }, + { + "accuracy": 0.796512, + "f1": 0.793998, + "f1_weighted": 0.793998, + "ap": 0.720266, + "ap_weighted": 0.720266 + }, + { + "accuracy": 0.796512, + "f1": 0.793998, + "f1_weighted": 0.793998, + "ap": 0.720266, + "ap_weighted": 0.720266 + }, + { + "accuracy": 0.796512, + "f1": 0.793998, + "f1_weighted": 0.793998, + "ap": 0.720266, + "ap_weighted": 0.720266 + }, + { + "accuracy": 0.796512, + "f1": 0.793998, + "f1_weighted": 0.793998, + "ap": 0.720266, + "ap_weighted": 0.720266 + }, + { + "accuracy": 0.796512, + "f1": 0.793998, + "f1_weighted": 0.793998, + "ap": 0.720266, + "ap_weighted": 0.720266 + }, + { + "accuracy": 0.796512, + "f1": 0.793998, + "f1_weighted": 0.793998, + "ap": 0.720266, + "ap_weighted": 0.720266 + }, + { + "accuracy": 0.796512, + "f1": 0.793998, + "f1_weighted": 0.793998, + "ap": 0.720266, + "ap_weighted": 0.720266 + }, + { + "accuracy": 0.796512, + "f1": 0.793998, + "f1_weighted": 0.793998, + "ap": 0.720266, + "ap_weighted": 0.720266 + } + ], + "main_score": 0.796512, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 4.654632091522217, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsDivorceLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsDivorceLegalBenchClassification.json new file mode 100644 index 0000000000..c6995f72a8 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsDivorceLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "LearnedHandsDivorceLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.82, + "f1": 0.819027, + "f1_weighted": 0.819027, + "ap": 0.749302, + "ap_weighted": 0.749302, + "scores_per_experiment": [ + { + "accuracy": 0.82, + "f1": 0.819027, + "f1_weighted": 0.819027, + "ap": 0.749302, + "ap_weighted": 0.749302 + }, + { + "accuracy": 0.82, + "f1": 0.819027, + "f1_weighted": 0.819027, + "ap": 0.749302, + "ap_weighted": 0.749302 + }, + { + "accuracy": 0.82, + "f1": 0.819027, + "f1_weighted": 0.819027, + "ap": 0.749302, + "ap_weighted": 0.749302 + }, + { + "accuracy": 0.82, + "f1": 0.819027, + "f1_weighted": 0.819027, + "ap": 0.749302, + "ap_weighted": 0.749302 + }, + { + "accuracy": 0.82, + "f1": 0.819027, + "f1_weighted": 0.819027, + "ap": 0.749302, + "ap_weighted": 0.749302 + }, + { + "accuracy": 0.82, + "f1": 0.819027, + "f1_weighted": 0.819027, + "ap": 0.749302, + "ap_weighted": 0.749302 + }, + { + "accuracy": 0.82, + "f1": 0.819027, + "f1_weighted": 0.819027, + "ap": 0.749302, + "ap_weighted": 0.749302 + }, + { + "accuracy": 0.82, + "f1": 0.819027, + "f1_weighted": 0.819027, + "ap": 0.749302, + "ap_weighted": 0.749302 + }, + { + "accuracy": 0.82, + "f1": 0.819027, + "f1_weighted": 0.819027, + "ap": 0.749302, + "ap_weighted": 0.749302 + }, + { + "accuracy": 0.82, + "f1": 0.819027, + "f1_weighted": 0.819027, + "ap": 0.749302, + "ap_weighted": 0.749302 + } + ], + "main_score": 0.82, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.7702853679656982, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsDomesticViolenceLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsDomesticViolenceLegalBenchClassification.json new file mode 100644 index 0000000000..16c1b7f6be --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsDomesticViolenceLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "LearnedHandsDomesticViolenceLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.770115, + "f1": 0.767037, + "f1_weighted": 0.767037, + "ap": 0.694382, + "ap_weighted": 0.694382, + "scores_per_experiment": [ + { + "accuracy": 0.770115, + "f1": 0.767037, + "f1_weighted": 0.767037, + "ap": 0.694382, + "ap_weighted": 0.694382 + }, + { + "accuracy": 0.770115, + "f1": 0.767037, + "f1_weighted": 0.767037, + "ap": 0.694382, + "ap_weighted": 0.694382 + }, + { + "accuracy": 0.770115, + "f1": 0.767037, + "f1_weighted": 0.767037, + "ap": 0.694382, + "ap_weighted": 0.694382 + }, + { + "accuracy": 0.770115, + "f1": 0.767037, + "f1_weighted": 0.767037, + "ap": 0.694382, + "ap_weighted": 0.694382 + }, + { + "accuracy": 0.770115, + "f1": 0.767037, + "f1_weighted": 0.767037, + "ap": 0.694382, + "ap_weighted": 0.694382 + }, + { + "accuracy": 0.770115, + "f1": 0.767037, + "f1_weighted": 0.767037, + "ap": 0.694382, + "ap_weighted": 0.694382 + }, + { + "accuracy": 0.770115, + "f1": 0.767037, + "f1_weighted": 0.767037, + "ap": 0.694382, + "ap_weighted": 0.694382 + }, + { + "accuracy": 0.770115, + "f1": 0.767037, + "f1_weighted": 0.767037, + "ap": 0.694382, + "ap_weighted": 0.694382 + }, + { + "accuracy": 0.770115, + "f1": 0.767037, + "f1_weighted": 0.767037, + "ap": 0.694382, + "ap_weighted": 0.694382 + }, + { + "accuracy": 0.770115, + "f1": 0.767037, + "f1_weighted": 0.767037, + "ap": 0.694382, + "ap_weighted": 0.694382 + } + ], + "main_score": 0.770115, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.1417171955108643, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsEducationLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsEducationLegalBenchClassification.json new file mode 100644 index 0000000000..4b10a0812e --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsEducationLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "LearnedHandsEducationLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.767857, + "f1": 0.761702, + "f1_weighted": 0.761702, + "ap": 0.688224, + "ap_weighted": 0.688224, + "scores_per_experiment": [ + { + "accuracy": 0.767857, + "f1": 0.761702, + "f1_weighted": 0.761702, + "ap": 0.688224, + "ap_weighted": 0.688224 + }, + { + "accuracy": 0.767857, + "f1": 0.761702, + "f1_weighted": 0.761702, + "ap": 0.688224, + "ap_weighted": 0.688224 + }, + { + "accuracy": 0.767857, + "f1": 0.761702, + "f1_weighted": 0.761702, + "ap": 0.688224, + "ap_weighted": 0.688224 + }, + { + "accuracy": 0.767857, + "f1": 0.761702, + "f1_weighted": 0.761702, + "ap": 0.688224, + "ap_weighted": 0.688224 + }, + { + "accuracy": 0.767857, + "f1": 0.761702, + "f1_weighted": 0.761702, + "ap": 0.688224, + "ap_weighted": 0.688224 + }, + { + "accuracy": 0.767857, + "f1": 0.761702, + "f1_weighted": 0.761702, + "ap": 0.688224, + "ap_weighted": 0.688224 + }, + { + "accuracy": 0.767857, + "f1": 0.761702, + "f1_weighted": 0.761702, + "ap": 0.688224, + "ap_weighted": 0.688224 + }, + { + "accuracy": 0.767857, + "f1": 0.761702, + "f1_weighted": 0.761702, + "ap": 0.688224, + "ap_weighted": 0.688224 + }, + { + "accuracy": 0.767857, + "f1": 0.761702, + "f1_weighted": 0.761702, + "ap": 0.688224, + "ap_weighted": 0.688224 + }, + { + "accuracy": 0.767857, + "f1": 0.761702, + "f1_weighted": 0.761702, + "ap": 0.688224, + "ap_weighted": 0.688224 + } + ], + "main_score": 0.767857, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.1281981468200684, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsEmploymentLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsEmploymentLegalBenchClassification.json new file mode 100644 index 0000000000..aff92704ae --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsEmploymentLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "LearnedHandsEmploymentLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.823944, + "f1": 0.823169, + "f1_weighted": 0.823169, + "ap": 0.782925, + "ap_weighted": 0.782925, + "scores_per_experiment": [ + { + "accuracy": 0.823944, + "f1": 0.823169, + "f1_weighted": 0.823169, + "ap": 0.782925, + "ap_weighted": 0.782925 + }, + { + "accuracy": 0.823944, + "f1": 0.823169, + "f1_weighted": 0.823169, + "ap": 0.782925, + "ap_weighted": 0.782925 + }, + { + "accuracy": 0.823944, + "f1": 0.823169, + "f1_weighted": 0.823169, + "ap": 0.782925, + "ap_weighted": 0.782925 + }, + { + "accuracy": 0.823944, + "f1": 0.823169, + "f1_weighted": 0.823169, + "ap": 0.782925, + "ap_weighted": 0.782925 + }, + { + "accuracy": 0.823944, + "f1": 0.823169, + "f1_weighted": 0.823169, + "ap": 0.782925, + "ap_weighted": 0.782925 + }, + { + "accuracy": 0.823944, + "f1": 0.823169, + "f1_weighted": 0.823169, + "ap": 0.782925, + "ap_weighted": 0.782925 + }, + { + "accuracy": 0.823944, + "f1": 0.823169, + "f1_weighted": 0.823169, + "ap": 0.782925, + "ap_weighted": 0.782925 + }, + { + "accuracy": 0.823944, + "f1": 0.823169, + "f1_weighted": 0.823169, + "ap": 0.782925, + "ap_weighted": 0.782925 + }, + { + "accuracy": 0.823944, + "f1": 0.823169, + "f1_weighted": 0.823169, + "ap": 0.782925, + "ap_weighted": 0.782925 + }, + { + "accuracy": 0.823944, + "f1": 0.823169, + "f1_weighted": 0.823169, + "ap": 0.782925, + "ap_weighted": 0.782925 + } + ], + "main_score": 0.823944, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 4.928300619125366, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsEstatesLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsEstatesLegalBenchClassification.json new file mode 100644 index 0000000000..432a4bb259 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsEstatesLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "LearnedHandsEstatesLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.764045, + "f1": 0.756164, + "f1_weighted": 0.756164, + "ap": 0.683304, + "ap_weighted": 0.683304, + "scores_per_experiment": [ + { + "accuracy": 0.764045, + "f1": 0.756164, + "f1_weighted": 0.756164, + "ap": 0.683304, + "ap_weighted": 0.683304 + }, + { + "accuracy": 0.764045, + "f1": 0.756164, + "f1_weighted": 0.756164, + "ap": 0.683304, + "ap_weighted": 0.683304 + }, + { + "accuracy": 0.764045, + "f1": 0.756164, + "f1_weighted": 0.756164, + "ap": 0.683304, + "ap_weighted": 0.683304 + }, + { + "accuracy": 0.764045, + "f1": 0.756164, + "f1_weighted": 0.756164, + "ap": 0.683304, + "ap_weighted": 0.683304 + }, + { + "accuracy": 0.764045, + "f1": 0.756164, + "f1_weighted": 0.756164, + "ap": 0.683304, + "ap_weighted": 0.683304 + }, + { + "accuracy": 0.764045, + "f1": 0.756164, + "f1_weighted": 0.756164, + "ap": 0.683304, + "ap_weighted": 0.683304 + }, + { + "accuracy": 0.764045, + "f1": 0.756164, + "f1_weighted": 0.756164, + "ap": 0.683304, + "ap_weighted": 0.683304 + }, + { + "accuracy": 0.764045, + "f1": 0.756164, + "f1_weighted": 0.756164, + "ap": 0.683304, + "ap_weighted": 0.683304 + }, + { + "accuracy": 0.764045, + "f1": 0.756164, + "f1_weighted": 0.756164, + "ap": 0.683304, + "ap_weighted": 0.683304 + }, + { + "accuracy": 0.764045, + "f1": 0.756164, + "f1_weighted": 0.756164, + "ap": 0.683304, + "ap_weighted": 0.683304 + } + ], + "main_score": 0.764045, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.7755417823791504, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsFamilyLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsFamilyLegalBenchClassification.json new file mode 100644 index 0000000000..8ee09ffe85 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsFamilyLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "LearnedHandsFamilyLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.84082, + "f1": 0.838996, + "f1_weighted": 0.838996, + "ap": 0.76618, + "ap_weighted": 0.76618, + "scores_per_experiment": [ + { + "accuracy": 0.84082, + "f1": 0.838996, + "f1_weighted": 0.838996, + "ap": 0.76618, + "ap_weighted": 0.76618 + }, + { + "accuracy": 0.84082, + "f1": 0.838996, + "f1_weighted": 0.838996, + "ap": 0.76618, + "ap_weighted": 0.76618 + }, + { + "accuracy": 0.84082, + "f1": 0.838996, + "f1_weighted": 0.838996, + "ap": 0.76618, + "ap_weighted": 0.76618 + }, + { + "accuracy": 0.84082, + "f1": 0.838996, + "f1_weighted": 0.838996, + "ap": 0.76618, + "ap_weighted": 0.76618 + }, + { + "accuracy": 0.84082, + "f1": 0.838996, + "f1_weighted": 0.838996, + "ap": 0.76618, + "ap_weighted": 0.76618 + }, + { + "accuracy": 0.84082, + "f1": 0.838996, + "f1_weighted": 0.838996, + "ap": 0.76618, + "ap_weighted": 0.76618 + }, + { + "accuracy": 0.84082, + "f1": 0.838996, + "f1_weighted": 0.838996, + "ap": 0.76618, + "ap_weighted": 0.76618 + }, + { + "accuracy": 0.84082, + "f1": 0.838996, + "f1_weighted": 0.838996, + "ap": 0.76618, + "ap_weighted": 0.76618 + }, + { + "accuracy": 0.84082, + "f1": 0.838996, + "f1_weighted": 0.838996, + "ap": 0.76618, + "ap_weighted": 0.76618 + }, + { + "accuracy": 0.84082, + "f1": 0.838996, + "f1_weighted": 0.838996, + "ap": 0.76618, + "ap_weighted": 0.76618 + } + ], + "main_score": 0.84082, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 13.119625329971313, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsHealthLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsHealthLegalBenchClassification.json new file mode 100644 index 0000000000..8421869f93 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsHealthLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "LearnedHandsHealthLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.575221, + "f1": 0.564162, + "f1_weighted": 0.564162, + "ap": 0.541902, + "ap_weighted": 0.541902, + "scores_per_experiment": [ + { + "accuracy": 0.575221, + "f1": 0.564162, + "f1_weighted": 0.564162, + "ap": 0.541902, + "ap_weighted": 0.541902 + }, + { + "accuracy": 0.575221, + "f1": 0.564162, + "f1_weighted": 0.564162, + "ap": 0.541902, + "ap_weighted": 0.541902 + }, + { + "accuracy": 0.575221, + "f1": 0.564162, + "f1_weighted": 0.564162, + "ap": 0.541902, + "ap_weighted": 0.541902 + }, + { + "accuracy": 0.575221, + "f1": 0.564162, + "f1_weighted": 0.564162, + "ap": 0.541902, + "ap_weighted": 0.541902 + }, + { + "accuracy": 0.575221, + "f1": 0.564162, + "f1_weighted": 0.564162, + "ap": 0.541902, + "ap_weighted": 0.541902 + }, + { + "accuracy": 0.575221, + "f1": 0.564162, + "f1_weighted": 0.564162, + "ap": 0.541902, + "ap_weighted": 0.541902 + }, + { + "accuracy": 0.575221, + "f1": 0.564162, + "f1_weighted": 0.564162, + "ap": 0.541902, + "ap_weighted": 0.541902 + }, + { + "accuracy": 0.575221, + "f1": 0.564162, + "f1_weighted": 0.564162, + "ap": 0.541902, + "ap_weighted": 0.541902 + }, + { + "accuracy": 0.575221, + "f1": 0.564162, + "f1_weighted": 0.564162, + "ap": 0.541902, + "ap_weighted": 0.541902 + }, + { + "accuracy": 0.575221, + "f1": 0.564162, + "f1_weighted": 0.564162, + "ap": 0.541902, + "ap_weighted": 0.541902 + } + ], + "main_score": 0.575221, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.2227635383605957, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsHousingLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsHousingLegalBenchClassification.json new file mode 100644 index 0000000000..c25b2a2412 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsHousingLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "LearnedHandsHousingLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.772461, + "f1": 0.771423, + "f1_weighted": 0.771423, + "ap": 0.701649, + "ap_weighted": 0.701649, + "scores_per_experiment": [ + { + "accuracy": 0.772461, + "f1": 0.771423, + "f1_weighted": 0.771423, + "ap": 0.701649, + "ap_weighted": 0.701649 + }, + { + "accuracy": 0.772461, + "f1": 0.771423, + "f1_weighted": 0.771423, + "ap": 0.701649, + "ap_weighted": 0.701649 + }, + { + "accuracy": 0.772461, + "f1": 0.771423, + "f1_weighted": 0.771423, + "ap": 0.701649, + "ap_weighted": 0.701649 + }, + { + "accuracy": 0.772461, + "f1": 0.771423, + "f1_weighted": 0.771423, + "ap": 0.701649, + "ap_weighted": 0.701649 + }, + { + "accuracy": 0.772461, + "f1": 0.771423, + "f1_weighted": 0.771423, + "ap": 0.701649, + "ap_weighted": 0.701649 + }, + { + "accuracy": 0.772461, + "f1": 0.771423, + "f1_weighted": 0.771423, + "ap": 0.701649, + "ap_weighted": 0.701649 + }, + { + "accuracy": 0.772461, + "f1": 0.771423, + "f1_weighted": 0.771423, + "ap": 0.701649, + "ap_weighted": 0.701649 + }, + { + "accuracy": 0.772461, + "f1": 0.771423, + "f1_weighted": 0.771423, + "ap": 0.701649, + "ap_weighted": 0.701649 + }, + { + "accuracy": 0.772461, + "f1": 0.771423, + "f1_weighted": 0.771423, + "ap": 0.701649, + "ap_weighted": 0.701649 + }, + { + "accuracy": 0.772461, + "f1": 0.771423, + "f1_weighted": 0.771423, + "ap": 0.701649, + "ap_weighted": 0.701649 + } + ], + "main_score": 0.772461, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 12.961327314376831, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsImmigrationLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsImmigrationLegalBenchClassification.json new file mode 100644 index 0000000000..f3c24742ce --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsImmigrationLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "LearnedHandsImmigrationLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.902985, + "f1": 0.90298, + "f1_weighted": 0.90298, + "ap": 0.861501, + "ap_weighted": 0.861501, + "scores_per_experiment": [ + { + "accuracy": 0.902985, + "f1": 0.90298, + "f1_weighted": 0.90298, + "ap": 0.861501, + "ap_weighted": 0.861501 + }, + { + "accuracy": 0.902985, + "f1": 0.90298, + "f1_weighted": 0.90298, + "ap": 0.861501, + "ap_weighted": 0.861501 + }, + { + "accuracy": 0.902985, + "f1": 0.90298, + "f1_weighted": 0.90298, + "ap": 0.861501, + "ap_weighted": 0.861501 + }, + { + "accuracy": 0.902985, + "f1": 0.90298, + "f1_weighted": 0.90298, + "ap": 0.861501, + "ap_weighted": 0.861501 + }, + { + "accuracy": 0.902985, + "f1": 0.90298, + "f1_weighted": 0.90298, + "ap": 0.861501, + "ap_weighted": 0.861501 + }, + { + "accuracy": 0.902985, + "f1": 0.90298, + "f1_weighted": 0.90298, + "ap": 0.861501, + "ap_weighted": 0.861501 + }, + { + "accuracy": 0.902985, + "f1": 0.90298, + "f1_weighted": 0.90298, + "ap": 0.861501, + "ap_weighted": 0.861501 + }, + { + "accuracy": 0.902985, + "f1": 0.90298, + "f1_weighted": 0.90298, + "ap": 0.861501, + "ap_weighted": 0.861501 + }, + { + "accuracy": 0.902985, + "f1": 0.90298, + "f1_weighted": 0.90298, + "ap": 0.861501, + "ap_weighted": 0.861501 + }, + { + "accuracy": 0.902985, + "f1": 0.90298, + "f1_weighted": 0.90298, + "ap": 0.861501, + "ap_weighted": 0.861501 + } + ], + "main_score": 0.902985, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.4094913005828857, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsTortsLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsTortsLegalBenchClassification.json new file mode 100644 index 0000000000..f5e3165c1a --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsTortsLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "LearnedHandsTortsLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.710648, + "f1": 0.703946, + "f1_weighted": 0.703946, + "ap": 0.639433, + "ap_weighted": 0.639433, + "scores_per_experiment": [ + { + "accuracy": 0.710648, + "f1": 0.703946, + "f1_weighted": 0.703946, + "ap": 0.639433, + "ap_weighted": 0.639433 + }, + { + "accuracy": 0.710648, + "f1": 0.703946, + "f1_weighted": 0.703946, + "ap": 0.639433, + "ap_weighted": 0.639433 + }, + { + "accuracy": 0.710648, + "f1": 0.703946, + "f1_weighted": 0.703946, + "ap": 0.639433, + "ap_weighted": 0.639433 + }, + { + "accuracy": 0.710648, + "f1": 0.703946, + "f1_weighted": 0.703946, + "ap": 0.639433, + "ap_weighted": 0.639433 + }, + { + "accuracy": 0.710648, + "f1": 0.703946, + "f1_weighted": 0.703946, + "ap": 0.639433, + "ap_weighted": 0.639433 + }, + { + "accuracy": 0.710648, + "f1": 0.703946, + "f1_weighted": 0.703946, + "ap": 0.639433, + "ap_weighted": 0.639433 + }, + { + "accuracy": 0.710648, + "f1": 0.703946, + "f1_weighted": 0.703946, + "ap": 0.639433, + "ap_weighted": 0.639433 + }, + { + "accuracy": 0.710648, + "f1": 0.703946, + "f1_weighted": 0.703946, + "ap": 0.639433, + "ap_weighted": 0.639433 + }, + { + "accuracy": 0.710648, + "f1": 0.703946, + "f1_weighted": 0.703946, + "ap": 0.639433, + "ap_weighted": 0.639433 + }, + { + "accuracy": 0.710648, + "f1": 0.703946, + "f1_weighted": 0.703946, + "ap": 0.639433, + "ap_weighted": 0.639433 + } + ], + "main_score": 0.710648, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.3027493953704834, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsTrafficLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsTrafficLegalBenchClassification.json new file mode 100644 index 0000000000..b2980a6b7f --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/LearnedHandsTrafficLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "LearnedHandsTrafficLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.72482, + "f1": 0.703529, + "f1_weighted": 0.703529, + "ap": 0.645317, + "ap_weighted": 0.645317, + "scores_per_experiment": [ + { + "accuracy": 0.72482, + "f1": 0.703529, + "f1_weighted": 0.703529, + "ap": 0.645317, + "ap_weighted": 0.645317 + }, + { + "accuracy": 0.72482, + "f1": 0.703529, + "f1_weighted": 0.703529, + "ap": 0.645317, + "ap_weighted": 0.645317 + }, + { + "accuracy": 0.72482, + "f1": 0.703529, + "f1_weighted": 0.703529, + "ap": 0.645317, + "ap_weighted": 0.645317 + }, + { + "accuracy": 0.72482, + "f1": 0.703529, + "f1_weighted": 0.703529, + "ap": 0.645317, + "ap_weighted": 0.645317 + }, + { + "accuracy": 0.72482, + "f1": 0.703529, + "f1_weighted": 0.703529, + "ap": 0.645317, + "ap_weighted": 0.645317 + }, + { + "accuracy": 0.72482, + "f1": 0.703529, + "f1_weighted": 0.703529, + "ap": 0.645317, + "ap_weighted": 0.645317 + }, + { + "accuracy": 0.72482, + "f1": 0.703529, + "f1_weighted": 0.703529, + "ap": 0.645317, + "ap_weighted": 0.645317 + }, + { + "accuracy": 0.72482, + "f1": 0.703529, + "f1_weighted": 0.703529, + "ap": 0.645317, + "ap_weighted": 0.645317 + }, + { + "accuracy": 0.72482, + "f1": 0.703529, + "f1_weighted": 0.703529, + "ap": 0.645317, + "ap_weighted": 0.645317 + }, + { + "accuracy": 0.72482, + "f1": 0.703529, + "f1_weighted": 0.703529, + "ap": 0.645317, + "ap_weighted": 0.645317 + } + ], + "main_score": 0.72482, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.8368465900421143, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/LegalReasoningCausalityLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/LegalReasoningCausalityLegalBenchClassification.json new file mode 100644 index 0000000000..d78aff7bd7 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/LegalReasoningCausalityLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "LegalReasoningCausalityLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.581818, + "f1": 0.581264, + "f1_weighted": 0.583202, + "ap": 0.613106, + "ap_weighted": 0.613106, + "scores_per_experiment": [ + { + "accuracy": 0.581818, + "f1": 0.581264, + "f1_weighted": 0.583202, + "ap": 0.613106, + "ap_weighted": 0.613106 + }, + { + "accuracy": 0.581818, + "f1": 0.581264, + "f1_weighted": 0.583202, + "ap": 0.613106, + "ap_weighted": 0.613106 + }, + { + "accuracy": 0.581818, + "f1": 0.581264, + "f1_weighted": 0.583202, + "ap": 0.613106, + "ap_weighted": 0.613106 + }, + { + "accuracy": 0.581818, + "f1": 0.581264, + "f1_weighted": 0.583202, + "ap": 0.613106, + "ap_weighted": 0.613106 + }, + { + "accuracy": 0.581818, + "f1": 0.581264, + "f1_weighted": 0.583202, + "ap": 0.613106, + "ap_weighted": 0.613106 + }, + { + "accuracy": 0.581818, + "f1": 0.581264, + "f1_weighted": 0.583202, + "ap": 0.613106, + "ap_weighted": 0.613106 + }, + { + "accuracy": 0.581818, + "f1": 0.581264, + "f1_weighted": 0.583202, + "ap": 0.613106, + "ap_weighted": 0.613106 + }, + { + "accuracy": 0.581818, + "f1": 0.581264, + "f1_weighted": 0.583202, + "ap": 0.613106, + "ap_weighted": 0.613106 + }, + { + "accuracy": 0.581818, + "f1": 0.581264, + "f1_weighted": 0.583202, + "ap": 0.613106, + "ap_weighted": 0.613106 + }, + { + "accuracy": 0.581818, + "f1": 0.581264, + "f1_weighted": 0.583202, + "ap": 0.613106, + "ap_weighted": 0.613106 + } + ], + "main_score": 0.581818, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.2482845783233643, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/MAUDLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/MAUDLegalBenchClassification.json new file mode 100644 index 0000000000..114a33996c --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/MAUDLegalBenchClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "MAUDLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.308447, + "f1": 0.166171, + "f1_weighted": 0.321251, + "scores_per_experiment": [ + { + "accuracy": 0.333496, + "f1": 0.173643, + "f1_weighted": 0.341918 + }, + { + "accuracy": 0.290039, + "f1": 0.162772, + "f1_weighted": 0.313248 + }, + { + "accuracy": 0.272461, + "f1": 0.16554, + "f1_weighted": 0.294891 + }, + { + "accuracy": 0.28418, + "f1": 0.148765, + "f1_weighted": 0.280682 + }, + { + "accuracy": 0.241211, + "f1": 0.149859, + "f1_weighted": 0.255597 + }, + { + "accuracy": 0.34082, + "f1": 0.187322, + "f1_weighted": 0.349984 + }, + { + "accuracy": 0.335449, + "f1": 0.176636, + "f1_weighted": 0.371444 + }, + { + "accuracy": 0.317871, + "f1": 0.172664, + "f1_weighted": 0.335387 + }, + { + "accuracy": 0.325195, + "f1": 0.159333, + "f1_weighted": 0.321884 + }, + { + "accuracy": 0.34375, + "f1": 0.165174, + "f1_weighted": 0.347477 + } + ], + "main_score": 0.308447, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 18.820935487747192, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/MSMARCO.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/MSMARCO.json deleted file mode 100644 index a45639b4e4..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/MSMARCO.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "dev": { - "evaluation_time": 21809.04, - "map_at_1": 0.22007, - "map_at_10": 0.34217, - "map_at_100": 0.35449, - "map_at_1000": 0.35495, - "map_at_3": 0.30302, - "map_at_5": 0.3255, - "ndcg_at_1": 0.22665, - "ndcg_at_10": 0.41159, - "ndcg_at_100": 0.47041, - "ndcg_at_1000": 0.48152, - "ndcg_at_3": 0.332, - "ndcg_at_5": 0.372, - "precision_at_1": 0.22665, - "precision_at_10": 0.06537, - "precision_at_100": 0.00948, - "precision_at_1000": 0.00104, - "precision_at_3": 0.14164, - "precision_at_5": 0.1053, - "recall_at_1": 0.22007, - "recall_at_10": 0.62516, - "recall_at_100": 0.89613, - "recall_at_1000": 0.98017, - "recall_at_3": 0.40923, - "recall_at_5": 0.50507 - }, - "mteb_version": "0.0.2", - "test": { - "evaluation_time": 19322.61, - "map_at_1": 0.02293, - "map_at_10": 0.14746, - "map_at_100": 0.36103, - "map_at_1000": 0.42862, - "map_at_3": 0.06613, - "map_at_5": 0.09531, - "ndcg_at_1": 0.74419, - "ndcg_at_10": 0.68745, - "ndcg_at_100": 0.6048, - "ndcg_at_1000": 0.67682, - "ndcg_at_3": 0.72005, - "ndcg_at_5": 0.71245, - "precision_at_1": 0.93023, - "precision_at_10": 0.77209, - "precision_at_100": 0.35628, - "precision_at_1000": 0.06556, - "precision_at_3": 0.87597, - "precision_at_5": 0.85581, - "recall_at_1": 0.02293, - "recall_at_10": 0.15753, - "recall_at_100": 0.48008, - "recall_at_1000": 0.73771, - "recall_at_3": 0.06772, - "recall_at_5": 0.09867 - }, - "mteb_dataset_name": "MSMARCO", - "dataset_revision": "e6838a846e2408f22cf5cc337ebc83e0bcf77849" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/MTOPDomainClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/MTOPDomainClassification.json index 060c45524a..2759984312 100644 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/MTOPDomainClassification.json +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/MTOPDomainClassification.json @@ -1,15 +1,757 @@ { - "test": { - "en": { - "accuracy": 0.9242134062927496, - "accuracy_stderr": 0.010291889184474006, - "f1": 0.9216698103249558, - "f1_stderr": 0.00977815776141202, - "main_score": 0.9242134062927496 - }, - "evaluation_time": 25.57 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "MTOPDomainClassification", - "dataset_revision": "a7e2a951126a26fc8c6a69f835f33a346ba259e3" + "dataset_revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf", + "task_name": "MTOPDomainClassification", + "mteb_version": "1.34.14", + "scores": { + "validation": [ + { + "accuracy": 0.931007, + "f1": 0.930998, + "f1_weighted": 0.930923, + "scores_per_experiment": [ + { + "accuracy": 0.928859, + "f1": 0.925571, + "f1_weighted": 0.928929 + }, + { + "accuracy": 0.934228, + "f1": 0.936394, + "f1_weighted": 0.933888 + }, + { + "accuracy": 0.923937, + "f1": 0.922352, + "f1_weighted": 0.923564 + }, + { + "accuracy": 0.936018, + "f1": 0.936221, + "f1_weighted": 0.935894 + }, + { + "accuracy": 0.933333, + "f1": 0.933886, + "f1_weighted": 0.933682 + }, + { + "accuracy": 0.93557, + "f1": 0.938851, + "f1_weighted": 0.935465 + }, + { + "accuracy": 0.920358, + "f1": 0.918627, + "f1_weighted": 0.920057 + }, + { + "accuracy": 0.932438, + "f1": 0.932295, + "f1_weighted": 0.932475 + }, + { + "accuracy": 0.923043, + "f1": 0.924293, + "f1_weighted": 0.923011 + }, + { + "accuracy": 0.942282, + "f1": 0.941492, + "f1_weighted": 0.942269 + } + ], + "main_score": 0.931007, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.783361, + "f1": 0.763064, + "f1_weighted": 0.782262, + "scores_per_experiment": [ + { + "accuracy": 0.758127, + "f1": 0.743045, + "f1_weighted": 0.757866 + }, + { + "accuracy": 0.782369, + "f1": 0.764105, + "f1_weighted": 0.780829 + }, + { + "accuracy": 0.786777, + "f1": 0.758956, + "f1_weighted": 0.779553 + }, + { + "accuracy": 0.799449, + "f1": 0.778715, + "f1_weighted": 0.799353 + }, + { + "accuracy": 0.792287, + "f1": 0.765174, + "f1_weighted": 0.789387 + }, + { + "accuracy": 0.787328, + "f1": 0.756765, + "f1_weighted": 0.780784 + }, + { + "accuracy": 0.778512, + "f1": 0.759734, + "f1_weighted": 0.778315 + }, + { + "accuracy": 0.763085, + "f1": 0.750805, + "f1_weighted": 0.768269 + }, + { + "accuracy": 0.799449, + "f1": 0.778916, + "f1_weighted": 0.798703 + }, + { + "accuracy": 0.786226, + "f1": 0.774426, + "f1_weighted": 0.789563 + } + ], + "main_score": 0.783361, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ] + }, + { + "accuracy": 0.776031, + "f1": 0.772638, + "f1_weighted": 0.773181, + "scores_per_experiment": [ + { + "accuracy": 0.790439, + "f1": 0.785047, + "f1_weighted": 0.790356 + }, + { + "accuracy": 0.764244, + "f1": 0.765637, + "f1_weighted": 0.758293 + }, + { + "accuracy": 0.764244, + "f1": 0.763608, + "f1_weighted": 0.75447 + }, + { + "accuracy": 0.745907, + "f1": 0.74179, + "f1_weighted": 0.741087 + }, + { + "accuracy": 0.791094, + "f1": 0.787943, + "f1_weighted": 0.787941 + }, + { + "accuracy": 0.787164, + "f1": 0.782051, + "f1_weighted": 0.784668 + }, + { + "accuracy": 0.777996, + "f1": 0.779655, + "f1_weighted": 0.778047 + }, + { + "accuracy": 0.773412, + "f1": 0.766149, + "f1_weighted": 0.772492 + }, + { + "accuracy": 0.777996, + "f1": 0.775563, + "f1_weighted": 0.776468 + }, + { + "accuracy": 0.787819, + "f1": 0.778941, + "f1_weighted": 0.787986 + } + ], + "main_score": 0.776031, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ] + }, + { + "accuracy": 0.792771, + "f1": 0.793404, + "f1_weighted": 0.791727, + "scores_per_experiment": [ + { + "accuracy": 0.775523, + "f1": 0.77629, + "f1_weighted": 0.776772 + }, + { + "accuracy": 0.797083, + "f1": 0.79795, + "f1_weighted": 0.796548 + }, + { + "accuracy": 0.78884, + "f1": 0.787618, + "f1_weighted": 0.788019 + }, + { + "accuracy": 0.783767, + "f1": 0.785982, + "f1_weighted": 0.780796 + }, + { + "accuracy": 0.80279, + "f1": 0.801454, + "f1_weighted": 0.802961 + }, + { + "accuracy": 0.790108, + "f1": 0.796279, + "f1_weighted": 0.787985 + }, + { + "accuracy": 0.811668, + "f1": 0.813513, + "f1_weighted": 0.810785 + }, + { + "accuracy": 0.80279, + "f1": 0.80291, + "f1_weighted": 0.800792 + }, + { + "accuracy": 0.779962, + "f1": 0.780427, + "f1_weighted": 0.777384 + }, + { + "accuracy": 0.795181, + "f1": 0.791614, + "f1_weighted": 0.795232 + } + ], + "main_score": 0.792771, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ] + }, + { + "accuracy": 0.216302, + "f1": 0.165642, + "f1_weighted": 0.167101, + "scores_per_experiment": [ + { + "accuracy": 0.237078, + "f1": 0.158359, + "f1_weighted": 0.172832 + }, + { + "accuracy": 0.20328, + "f1": 0.16383, + "f1_weighted": 0.16297 + }, + { + "accuracy": 0.214215, + "f1": 0.166821, + "f1_weighted": 0.161869 + }, + { + "accuracy": 0.242545, + "f1": 0.204672, + "f1_weighted": 0.21312 + }, + { + "accuracy": 0.19334, + "f1": 0.144517, + "f1_weighted": 0.138451 + }, + { + "accuracy": 0.170477, + "f1": 0.139665, + "f1_weighted": 0.128025 + }, + { + "accuracy": 0.268887, + "f1": 0.185256, + "f1_weighted": 0.205203 + }, + { + "accuracy": 0.21173, + "f1": 0.164668, + "f1_weighted": 0.160797 + }, + { + "accuracy": 0.201789, + "f1": 0.155446, + "f1_weighted": 0.157884 + }, + { + "accuracy": 0.219682, + "f1": 0.173182, + "f1_weighted": 0.169855 + } + ], + "main_score": 0.216302, + "hf_subset": "hi", + "languages": [ + "hin-Deva" + ] + }, + { + "accuracy": 0.174985, + "f1": 0.119758, + "f1_weighted": 0.125244, + "scores_per_experiment": [ + { + "accuracy": 0.187911, + "f1": 0.145123, + "f1_weighted": 0.14774 + }, + { + "accuracy": 0.196888, + "f1": 0.105247, + "f1_weighted": 0.129257 + }, + { + "accuracy": 0.160383, + "f1": 0.103684, + "f1_weighted": 0.101869 + }, + { + "accuracy": 0.175344, + "f1": 0.137243, + "f1_weighted": 0.145945 + }, + { + "accuracy": 0.163375, + "f1": 0.11832, + "f1_weighted": 0.126053 + }, + { + "accuracy": 0.183124, + "f1": 0.123941, + "f1_weighted": 0.124035 + }, + { + "accuracy": 0.166367, + "f1": 0.102753, + "f1_weighted": 0.101414 + }, + { + "accuracy": 0.158588, + "f1": 0.124909, + "f1_weighted": 0.137756 + }, + { + "accuracy": 0.186116, + "f1": 0.124079, + "f1_weighted": 0.119465 + }, + { + "accuracy": 0.171753, + "f1": 0.112282, + "f1_weighted": 0.118903 + } + ], + "main_score": 0.174985, + "hf_subset": "th", + "languages": [ + "tha-Thai" + ] + } + ], + "test": [ + { + "accuracy": 0.930233, + "f1": 0.927331, + "f1_weighted": 0.930303, + "scores_per_experiment": [ + { + "accuracy": 0.929777, + "f1": 0.925192, + "f1_weighted": 0.930123 + }, + { + "accuracy": 0.935477, + "f1": 0.932289, + "f1_weighted": 0.935557 + }, + { + "accuracy": 0.924761, + "f1": 0.922173, + "f1_weighted": 0.924208 + }, + { + "accuracy": 0.937528, + "f1": 0.934984, + "f1_weighted": 0.937676 + }, + { + "accuracy": 0.927725, + "f1": 0.924992, + "f1_weighted": 0.927849 + }, + { + "accuracy": 0.930005, + "f1": 0.930435, + "f1_weighted": 0.929849 + }, + { + "accuracy": 0.923393, + "f1": 0.91938, + "f1_weighted": 0.923259 + }, + { + "accuracy": 0.926585, + "f1": 0.923119, + "f1_weighted": 0.92692 + }, + { + "accuracy": 0.927041, + "f1": 0.924781, + "f1_weighted": 0.927323 + }, + { + "accuracy": 0.940036, + "f1": 0.93597, + "f1_weighted": 0.940268 + } + ], + "main_score": 0.930233, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.781685, + "f1": 0.761814, + "f1_weighted": 0.780748, + "scores_per_experiment": [ + { + "accuracy": 0.757396, + "f1": 0.74188, + "f1_weighted": 0.756881 + }, + { + "accuracy": 0.774866, + "f1": 0.757486, + "f1_weighted": 0.773498 + }, + { + "accuracy": 0.803888, + "f1": 0.774827, + "f1_weighted": 0.797688 + }, + { + "accuracy": 0.789518, + "f1": 0.772779, + "f1_weighted": 0.791379 + }, + { + "accuracy": 0.798817, + "f1": 0.772712, + "f1_weighted": 0.795327 + }, + { + "accuracy": 0.78501, + "f1": 0.75787, + "f1_weighted": 0.781352 + }, + { + "accuracy": 0.782756, + "f1": 0.762944, + "f1_weighted": 0.782299 + }, + { + "accuracy": 0.757678, + "f1": 0.745162, + "f1_weighted": 0.761483 + }, + { + "accuracy": 0.790363, + "f1": 0.770303, + "f1_weighted": 0.788723 + }, + { + "accuracy": 0.776557, + "f1": 0.76218, + "f1_weighted": 0.778847 + } + ], + "main_score": 0.781685, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ] + }, + { + "accuracy": 0.764877, + "f1": 0.756515, + "f1_weighted": 0.762688, + "scores_per_experiment": [ + { + "accuracy": 0.77952, + "f1": 0.772594, + "f1_weighted": 0.777928 + }, + { + "accuracy": 0.752502, + "f1": 0.748896, + "f1_weighted": 0.747425 + }, + { + "accuracy": 0.766177, + "f1": 0.758866, + "f1_weighted": 0.760418 + }, + { + "accuracy": 0.723482, + "f1": 0.71544, + "f1_weighted": 0.718978 + }, + { + "accuracy": 0.769179, + "f1": 0.7601, + "f1_weighted": 0.767971 + }, + { + "accuracy": 0.774183, + "f1": 0.763839, + "f1_weighted": 0.772272 + }, + { + "accuracy": 0.773516, + "f1": 0.769864, + "f1_weighted": 0.772194 + }, + { + "accuracy": 0.75984, + "f1": 0.749617, + "f1_weighted": 0.759384 + }, + { + "accuracy": 0.778519, + "f1": 0.771501, + "f1_weighted": 0.77807 + }, + { + "accuracy": 0.771848, + "f1": 0.754432, + "f1_weighted": 0.772245 + } + ], + "main_score": 0.764877, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ] + }, + { + "accuracy": 0.785124, + "f1": 0.784194, + "f1_weighted": 0.784009, + "scores_per_experiment": [ + { + "accuracy": 0.783902, + "f1": 0.781227, + "f1_weighted": 0.785416 + }, + { + "accuracy": 0.788287, + "f1": 0.791124, + "f1_weighted": 0.787829 + }, + { + "accuracy": 0.797682, + "f1": 0.79122, + "f1_weighted": 0.796119 + }, + { + "accuracy": 0.765424, + "f1": 0.769135, + "f1_weighted": 0.763523 + }, + { + "accuracy": 0.792985, + "f1": 0.789266, + "f1_weighted": 0.792918 + }, + { + "accuracy": 0.780144, + "f1": 0.787247, + "f1_weighted": 0.776683 + }, + { + "accuracy": 0.804886, + "f1": 0.800644, + "f1_weighted": 0.804532 + }, + { + "accuracy": 0.78954, + "f1": 0.787259, + "f1_weighted": 0.78758 + }, + { + "accuracy": 0.762606, + "f1": 0.762058, + "f1_weighted": 0.760215 + }, + { + "accuracy": 0.785781, + "f1": 0.782757, + "f1_weighted": 0.785278 + } + ], + "main_score": 0.785124, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ] + }, + { + "accuracy": 0.219111, + "f1": 0.157887, + "f1_weighted": 0.166763, + "scores_per_experiment": [ + { + "accuracy": 0.256006, + "f1": 0.16015, + "f1_weighted": 0.187328 + }, + { + "accuracy": 0.19792, + "f1": 0.152267, + "f1_weighted": 0.158596 + }, + { + "accuracy": 0.212262, + "f1": 0.156254, + "f1_weighted": 0.1598 + }, + { + "accuracy": 0.237361, + "f1": 0.183919, + "f1_weighted": 0.207948 + }, + { + "accuracy": 0.200789, + "f1": 0.14039, + "f1_weighted": 0.136247 + }, + { + "accuracy": 0.168878, + "f1": 0.130979, + "f1_weighted": 0.122447 + }, + { + "accuracy": 0.285766, + "f1": 0.170419, + "f1_weighted": 0.202245 + }, + { + "accuracy": 0.208677, + "f1": 0.160221, + "f1_weighted": 0.158438 + }, + { + "accuracy": 0.185371, + "f1": 0.138012, + "f1_weighted": 0.142916 + }, + { + "accuracy": 0.238078, + "f1": 0.186258, + "f1_weighted": 0.191667 + } + ], + "main_score": 0.219111, + "hf_subset": "hi", + "languages": [ + "hin-Deva" + ] + }, + { + "accuracy": 0.168391, + "f1": 0.114622, + "f1_weighted": 0.123339, + "scores_per_experiment": [ + { + "accuracy": 0.192043, + "f1": 0.141216, + "f1_weighted": 0.151941 + }, + { + "accuracy": 0.229656, + "f1": 0.113492, + "f1_weighted": 0.151192 + }, + { + "accuracy": 0.135624, + "f1": 0.098516, + "f1_weighted": 0.096096 + }, + { + "accuracy": 0.160217, + "f1": 0.119792, + "f1_weighted": 0.124897 + }, + { + "accuracy": 0.133816, + "f1": 0.102391, + "f1_weighted": 0.10813 + }, + { + "accuracy": 0.166365, + "f1": 0.114984, + "f1_weighted": 0.117256 + }, + { + "accuracy": 0.160217, + "f1": 0.105741, + "f1_weighted": 0.10921 + }, + { + "accuracy": 0.151899, + "f1": 0.11701, + "f1_weighted": 0.126612 + }, + { + "accuracy": 0.16528, + "f1": 0.113645, + "f1_weighted": 0.113294 + }, + { + "accuracy": 0.188788, + "f1": 0.119431, + "f1_weighted": 0.134763 + } + ], + "main_score": 0.168391, + "hf_subset": "th", + "languages": [ + "tha-Thai" + ] + } + ] + }, + "evaluation_time": 33.55765724182129, + "kg_co2_emissions": null } \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/MTOPIntentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/MTOPIntentClassification.json index 125dfc08aa..8d9b5185ae 100644 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/MTOPIntentClassification.json +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/MTOPIntentClassification.json @@ -1,15 +1,757 @@ { - "test": { - "en": { - "accuracy": 0.6244186046511628, - "accuracy_stderr": 0.021094464681249384, - "f1": 0.45483515455875656, - "f1_stderr": 0.01102713508069922, - "main_score": 0.6244186046511628 - }, - "evaluation_time": 31.95 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "MTOPIntentClassification", - "dataset_revision": "6299947a7777084cc2d4b64235bf7190381ce755" + "dataset_revision": "ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba", + "task_name": "MTOPIntentClassification", + "mteb_version": "1.34.14", + "scores": { + "validation": [ + { + "accuracy": 0.623221, + "f1": 0.431361, + "f1_weighted": 0.648799, + "scores_per_experiment": [ + { + "accuracy": 0.583445, + "f1": 0.406608, + "f1_weighted": 0.616661 + }, + { + "accuracy": 0.617897, + "f1": 0.428347, + "f1_weighted": 0.637496 + }, + { + "accuracy": 0.653691, + "f1": 0.429124, + "f1_weighted": 0.679944 + }, + { + "accuracy": 0.63132, + "f1": 0.438603, + "f1_weighted": 0.657616 + }, + { + "accuracy": 0.621029, + "f1": 0.457375, + "f1_weighted": 0.642253 + }, + { + "accuracy": 0.636689, + "f1": 0.437971, + "f1_weighted": 0.665241 + }, + { + "accuracy": 0.602237, + "f1": 0.426794, + "f1_weighted": 0.629017 + }, + { + "accuracy": 0.607606, + "f1": 0.428154, + "f1_weighted": 0.637508 + }, + { + "accuracy": 0.630425, + "f1": 0.41912, + "f1_weighted": 0.651994 + }, + { + "accuracy": 0.647875, + "f1": 0.441513, + "f1_weighted": 0.670261 + } + ], + "main_score": 0.623221, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.478678, + "f1": 0.277303, + "f1_weighted": 0.515048, + "scores_per_experiment": [ + { + "accuracy": 0.487052, + "f1": 0.269107, + "f1_weighted": 0.528882 + }, + { + "accuracy": 0.507438, + "f1": 0.301529, + "f1_weighted": 0.542761 + }, + { + "accuracy": 0.478788, + "f1": 0.278275, + "f1_weighted": 0.516694 + }, + { + "accuracy": 0.426446, + "f1": 0.250731, + "f1_weighted": 0.46351 + }, + { + "accuracy": 0.494766, + "f1": 0.282252, + "f1_weighted": 0.532283 + }, + { + "accuracy": 0.424793, + "f1": 0.263388, + "f1_weighted": 0.450199 + }, + { + "accuracy": 0.507989, + "f1": 0.293866, + "f1_weighted": 0.543785 + }, + { + "accuracy": 0.497521, + "f1": 0.293383, + "f1_weighted": 0.533655 + }, + { + "accuracy": 0.459504, + "f1": 0.261429, + "f1_weighted": 0.495707 + }, + { + "accuracy": 0.502479, + "f1": 0.27907, + "f1_weighted": 0.543006 + } + ], + "main_score": 0.478678, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ] + }, + { + "accuracy": 0.463654, + "f1": 0.290997, + "f1_weighted": 0.496982, + "scores_per_experiment": [ + { + "accuracy": 0.40668, + "f1": 0.262481, + "f1_weighted": 0.419024 + }, + { + "accuracy": 0.466274, + "f1": 0.293588, + "f1_weighted": 0.499234 + }, + { + "accuracy": 0.48592, + "f1": 0.311309, + "f1_weighted": 0.52788 + }, + { + "accuracy": 0.441388, + "f1": 0.288242, + "f1_weighted": 0.468744 + }, + { + "accuracy": 0.492469, + "f1": 0.29663, + "f1_weighted": 0.523043 + }, + { + "accuracy": 0.486575, + "f1": 0.294045, + "f1_weighted": 0.526398 + }, + { + "accuracy": 0.480681, + "f1": 0.285514, + "f1_weighted": 0.505039 + }, + { + "accuracy": 0.449247, + "f1": 0.315887, + "f1_weighted": 0.487474 + }, + { + "accuracy": 0.461035, + "f1": 0.29684, + "f1_weighted": 0.499272 + }, + { + "accuracy": 0.466274, + "f1": 0.265438, + "f1_weighted": 0.513712 + } + ], + "main_score": 0.463654, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ] + }, + { + "accuracy": 0.471338, + "f1": 0.298561, + "f1_weighted": 0.511284, + "scores_per_experiment": [ + { + "accuracy": 0.454027, + "f1": 0.300504, + "f1_weighted": 0.494894 + }, + { + "accuracy": 0.457197, + "f1": 0.294054, + "f1_weighted": 0.489255 + }, + { + "accuracy": 0.500317, + "f1": 0.287375, + "f1_weighted": 0.542109 + }, + { + "accuracy": 0.517438, + "f1": 0.317143, + "f1_weighted": 0.554699 + }, + { + "accuracy": 0.489537, + "f1": 0.293236, + "f1_weighted": 0.538491 + }, + { + "accuracy": 0.486367, + "f1": 0.290025, + "f1_weighted": 0.52207 + }, + { + "accuracy": 0.448954, + "f1": 0.305, + "f1_weighted": 0.486105 + }, + { + "accuracy": 0.463538, + "f1": 0.300005, + "f1_weighted": 0.499811 + }, + { + "accuracy": 0.462904, + "f1": 0.301779, + "f1_weighted": 0.508716 + }, + { + "accuracy": 0.433101, + "f1": 0.296483, + "f1_weighted": 0.476687 + } + ], + "main_score": 0.471338, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ] + }, + { + "accuracy": 0.037276, + "f1": 0.013838, + "f1_weighted": 0.029066, + "scores_per_experiment": [ + { + "accuracy": 0.041252, + "f1": 0.011628, + "f1_weighted": 0.017045 + }, + { + "accuracy": 0.053181, + "f1": 0.014658, + "f1_weighted": 0.051605 + }, + { + "accuracy": 0.033797, + "f1": 0.013814, + "f1_weighted": 0.041857 + }, + { + "accuracy": 0.014911, + "f1": 0.008781, + "f1_weighted": 0.009822 + }, + { + "accuracy": 0.070577, + "f1": 0.022422, + "f1_weighted": 0.056199 + }, + { + "accuracy": 0.027833, + "f1": 0.015034, + "f1_weighted": 0.014724 + }, + { + "accuracy": 0.04672, + "f1": 0.012465, + "f1_weighted": 0.032446 + }, + { + "accuracy": 0.030815, + "f1": 0.014784, + "f1_weighted": 0.018526 + }, + { + "accuracy": 0.032306, + "f1": 0.013318, + "f1_weighted": 0.035704 + }, + { + "accuracy": 0.021372, + "f1": 0.011475, + "f1_weighted": 0.012729 + } + ], + "main_score": 0.037276, + "hf_subset": "hi", + "languages": [ + "hin-Deva" + ] + }, + { + "accuracy": 0.050569, + "f1": 0.016755, + "f1_weighted": 0.046517, + "scores_per_experiment": [ + { + "accuracy": 0.043686, + "f1": 0.012975, + "f1_weighted": 0.046294 + }, + { + "accuracy": 0.062837, + "f1": 0.017279, + "f1_weighted": 0.045413 + }, + { + "accuracy": 0.035308, + "f1": 0.0165, + "f1_weighted": 0.036665 + }, + { + "accuracy": 0.055057, + "f1": 0.013716, + "f1_weighted": 0.03537 + }, + { + "accuracy": 0.013166, + "f1": 0.01245, + "f1_weighted": 0.010502 + }, + { + "accuracy": 0.052663, + "f1": 0.015738, + "f1_weighted": 0.053159 + }, + { + "accuracy": 0.076601, + "f1": 0.023255, + "f1_weighted": 0.075865 + }, + { + "accuracy": 0.076601, + "f1": 0.022788, + "f1_weighted": 0.078463 + }, + { + "accuracy": 0.062837, + "f1": 0.019589, + "f1_weighted": 0.056983 + }, + { + "accuracy": 0.02693, + "f1": 0.013263, + "f1_weighted": 0.026457 + } + ], + "main_score": 0.050569, + "hf_subset": "th", + "languages": [ + "tha-Thai" + ] + } + ], + "test": [ + { + "accuracy": 0.614318, + "f1": 0.444515, + "f1_weighted": 0.6374, + "scores_per_experiment": [ + { + "accuracy": 0.567487, + "f1": 0.418867, + "f1_weighted": 0.591529 + }, + { + "accuracy": 0.602143, + "f1": 0.44403, + "f1_weighted": 0.622462 + }, + { + "accuracy": 0.652987, + "f1": 0.455929, + "f1_weighted": 0.673653 + }, + { + "accuracy": 0.633151, + "f1": 0.432945, + "f1_weighted": 0.658025 + }, + { + "accuracy": 0.608527, + "f1": 0.457719, + "f1_weighted": 0.628473 + }, + { + "accuracy": 0.628135, + "f1": 0.456935, + "f1_weighted": 0.652793 + }, + { + "accuracy": 0.589375, + "f1": 0.449915, + "f1_weighted": 0.610924 + }, + { + "accuracy": 0.610351, + "f1": 0.453569, + "f1_weighted": 0.635992 + }, + { + "accuracy": 0.624943, + "f1": 0.440704, + "f1_weighted": 0.652798 + }, + { + "accuracy": 0.626083, + "f1": 0.434532, + "f1_weighted": 0.647348 + } + ], + "main_score": 0.614318, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.485714, + "f1": 0.29192, + "f1_weighted": 0.520078, + "scores_per_experiment": [ + { + "accuracy": 0.510848, + "f1": 0.302935, + "f1_weighted": 0.549445 + }, + { + "accuracy": 0.489997, + "f1": 0.292242, + "f1_weighted": 0.522922 + }, + { + "accuracy": 0.497605, + "f1": 0.300278, + "f1_weighted": 0.540072 + }, + { + "accuracy": 0.41082, + "f1": 0.273674, + "f1_weighted": 0.445294 + }, + { + "accuracy": 0.510566, + "f1": 0.297767, + "f1_weighted": 0.544427 + }, + { + "accuracy": 0.424063, + "f1": 0.268728, + "f1_weighted": 0.453881 + }, + { + "accuracy": 0.520428, + "f1": 0.288017, + "f1_weighted": 0.552956 + }, + { + "accuracy": 0.512257, + "f1": 0.312267, + "f1_weighted": 0.54362 + }, + { + "accuracy": 0.479572, + "f1": 0.289175, + "f1_weighted": 0.510049 + }, + { + "accuracy": 0.500986, + "f1": 0.294118, + "f1_weighted": 0.538111 + } + ], + "main_score": 0.485714, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ] + }, + { + "accuracy": 0.444797, + "f1": 0.279149, + "f1_weighted": 0.47767, + "scores_per_experiment": [ + { + "accuracy": 0.38559, + "f1": 0.257443, + "f1_weighted": 0.393959 + }, + { + "accuracy": 0.442628, + "f1": 0.274549, + "f1_weighted": 0.470131 + }, + { + "accuracy": 0.457305, + "f1": 0.274522, + "f1_weighted": 0.496388 + }, + { + "accuracy": 0.429953, + "f1": 0.27034, + "f1_weighted": 0.454554 + }, + { + "accuracy": 0.474316, + "f1": 0.297457, + "f1_weighted": 0.507392 + }, + { + "accuracy": 0.476651, + "f1": 0.292966, + "f1_weighted": 0.519427 + }, + { + "accuracy": 0.466978, + "f1": 0.273069, + "f1_weighted": 0.492883 + }, + { + "accuracy": 0.411274, + "f1": 0.282811, + "f1_weighted": 0.455204 + }, + { + "accuracy": 0.45964, + "f1": 0.288848, + "f1_weighted": 0.49519 + }, + { + "accuracy": 0.443629, + "f1": 0.279485, + "f1_weighted": 0.491573 + } + ], + "main_score": 0.444797, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ] + }, + { + "accuracy": 0.464861, + "f1": 0.305138, + "f1_weighted": 0.494282, + "scores_per_experiment": [ + { + "accuracy": 0.440965, + "f1": 0.291703, + "f1_weighted": 0.474064 + }, + { + "accuracy": 0.456937, + "f1": 0.316046, + "f1_weighted": 0.478154 + }, + { + "accuracy": 0.51331, + "f1": 0.313643, + "f1_weighted": 0.544686 + }, + { + "accuracy": 0.491074, + "f1": 0.319118, + "f1_weighted": 0.508774 + }, + { + "accuracy": 0.481365, + "f1": 0.313973, + "f1_weighted": 0.522371 + }, + { + "accuracy": 0.478234, + "f1": 0.312067, + "f1_weighted": 0.504072 + }, + { + "accuracy": 0.448794, + "f1": 0.300286, + "f1_weighted": 0.47248 + }, + { + "accuracy": 0.463201, + "f1": 0.297302, + "f1_weighted": 0.490452 + }, + { + "accuracy": 0.46508, + "f1": 0.294193, + "f1_weighted": 0.499532 + }, + { + "accuracy": 0.409646, + "f1": 0.293045, + "f1_weighted": 0.448239 + } + ], + "main_score": 0.464861, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ] + }, + { + "accuracy": 0.03625, + "f1": 0.010231, + "f1_weighted": 0.025296, + "scores_per_experiment": [ + { + "accuracy": 0.045895, + "f1": 0.010512, + "f1_weighted": 0.017769 + }, + { + "accuracy": 0.050556, + "f1": 0.012408, + "f1_weighted": 0.04323 + }, + { + "accuracy": 0.029043, + "f1": 0.011272, + "f1_weighted": 0.033289 + }, + { + "accuracy": 0.015418, + "f1": 0.006704, + "f1_weighted": 0.005435 + }, + { + "accuracy": 0.061312, + "f1": 0.015595, + "f1_weighted": 0.050825 + }, + { + "accuracy": 0.02474, + "f1": 0.006633, + "f1_weighted": 0.010427 + }, + { + "accuracy": 0.050197, + "f1": 0.01244, + "f1_weighted": 0.035572 + }, + { + "accuracy": 0.030477, + "f1": 0.00777, + "f1_weighted": 0.011671 + }, + { + "accuracy": 0.031553, + "f1": 0.009606, + "f1_weighted": 0.029959 + }, + { + "accuracy": 0.023306, + "f1": 0.009365, + "f1_weighted": 0.014782 + } + ], + "main_score": 0.03625, + "hf_subset": "hi", + "languages": [ + "hin-Deva" + ] + }, + { + "accuracy": 0.05226, + "f1": 0.015333, + "f1_weighted": 0.044511, + "scores_per_experiment": [ + { + "accuracy": 0.036166, + "f1": 0.008343, + "f1_weighted": 0.03156 + }, + { + "accuracy": 0.067993, + "f1": 0.021288, + "f1_weighted": 0.046375 + }, + { + "accuracy": 0.034358, + "f1": 0.018796, + "f1_weighted": 0.032825 + }, + { + "accuracy": 0.064376, + "f1": 0.016631, + "f1_weighted": 0.040435 + }, + { + "accuracy": 0.020253, + "f1": 0.009162, + "f1_weighted": 0.013346 + }, + { + "accuracy": 0.049548, + "f1": 0.011298, + "f1_weighted": 0.048123 + }, + { + "accuracy": 0.082459, + "f1": 0.016325, + "f1_weighted": 0.081283 + }, + { + "accuracy": 0.075949, + "f1": 0.017471, + "f1_weighted": 0.068208 + }, + { + "accuracy": 0.05859, + "f1": 0.024093, + "f1_weighted": 0.05392 + }, + { + "accuracy": 0.032911, + "f1": 0.009926, + "f1_weighted": 0.029031 + } + ], + "main_score": 0.05226, + "hf_subset": "th", + "languages": [ + "tha-Thai" + ] + } + ] + }, + "evaluation_time": 215.594988822937, + "kg_co2_emissions": null } \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/MacedonianTweetSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/MacedonianTweetSentimentClassification.json new file mode 100644 index 0000000000..f1c9047aab --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/MacedonianTweetSentimentClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "957e075ba35e4417ba7837987fd7053a6533a1a2", + "task_name": "MacedonianTweetSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.287357, + "f1": 0.243706, + "f1_weighted": 0.244762, + "scores_per_experiment": [ + { + "accuracy": 0.298507, + "f1": 0.293493, + "f1_weighted": 0.289783 + }, + { + "accuracy": 0.385426, + "f1": 0.290427, + "f1_weighted": 0.332168 + }, + { + "accuracy": 0.233538, + "f1": 0.21983, + "f1_weighted": 0.210108 + }, + { + "accuracy": 0.201054, + "f1": 0.161403, + "f1_weighted": 0.127355 + }, + { + "accuracy": 0.244074, + "f1": 0.240077, + "f1_weighted": 0.231955 + }, + { + "accuracy": 0.334504, + "f1": 0.282924, + "f1_weighted": 0.304802 + }, + { + "accuracy": 0.328358, + "f1": 0.29677, + "f1_weighted": 0.293472 + }, + { + "accuracy": 0.216857, + "f1": 0.189214, + "f1_weighted": 0.167556 + }, + { + "accuracy": 0.4223, + "f1": 0.268949, + "f1_weighted": 0.313833 + }, + { + "accuracy": 0.208955, + "f1": 0.19397, + "f1_weighted": 0.176594 + } + ], + "main_score": 0.287357, + "hf_subset": "default", + "languages": [ + "mkd-Cyrl" + ] + } + ] + }, + "evaluation_time": 2.225999355316162, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/MalayalamNewsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/MalayalamNewsClassification.json new file mode 100644 index 0000000000..6cea80a1be --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/MalayalamNewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "666f63bba2387456d8f846ea4d0565181bd47b81", + "task_name": "MalayalamNewsClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.418095, + "f1": 0.403999, + "f1_weighted": 0.403853, + "scores_per_experiment": [ + { + "accuracy": 0.453175, + "f1": 0.446783, + "f1_weighted": 0.448745 + }, + { + "accuracy": 0.447619, + "f1": 0.443998, + "f1_weighted": 0.442251 + }, + { + "accuracy": 0.461905, + "f1": 0.453623, + "f1_weighted": 0.455938 + }, + { + "accuracy": 0.330952, + "f1": 0.316897, + "f1_weighted": 0.315115 + }, + { + "accuracy": 0.444444, + "f1": 0.437543, + "f1_weighted": 0.439398 + }, + { + "accuracy": 0.455556, + "f1": 0.448395, + "f1_weighted": 0.450722 + }, + { + "accuracy": 0.346825, + "f1": 0.328827, + "f1_weighted": 0.327162 + }, + { + "accuracy": 0.383333, + "f1": 0.375432, + "f1_weighted": 0.374303 + }, + { + "accuracy": 0.434127, + "f1": 0.421689, + "f1_weighted": 0.420369 + }, + { + "accuracy": 0.423016, + "f1": 0.366801, + "f1_weighted": 0.364524 + } + ], + "main_score": 0.418095, + "hf_subset": "default", + "languages": [ + "mal-Mlym" + ] + } + ] + }, + "evaluation_time": 2.0245180130004883, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/MarathiNewsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/MarathiNewsClassification.json new file mode 100644 index 0000000000..747f35bcd1 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/MarathiNewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "7640cf8132cca1f99995ac71512a670e3c965cf1", + "task_name": "MarathiNewsClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.484961, + "f1": 0.385995, + "f1_weighted": 0.502658, + "scores_per_experiment": [ + { + "accuracy": 0.439941, + "f1": 0.384854, + "f1_weighted": 0.482186 + }, + { + "accuracy": 0.548828, + "f1": 0.391045, + "f1_weighted": 0.528954 + }, + { + "accuracy": 0.496582, + "f1": 0.400617, + "f1_weighted": 0.522534 + }, + { + "accuracy": 0.453125, + "f1": 0.351763, + "f1_weighted": 0.484606 + }, + { + "accuracy": 0.543457, + "f1": 0.443441, + "f1_weighted": 0.563358 + }, + { + "accuracy": 0.476562, + "f1": 0.333335, + "f1_weighted": 0.494474 + }, + { + "accuracy": 0.541016, + "f1": 0.391404, + "f1_weighted": 0.527906 + }, + { + "accuracy": 0.370117, + "f1": 0.337574, + "f1_weighted": 0.382138 + }, + { + "accuracy": 0.560059, + "f1": 0.443817, + "f1_weighted": 0.58104 + }, + { + "accuracy": 0.419922, + "f1": 0.382095, + "f1_weighted": 0.459383 + } + ], + "main_score": 0.385995, + "hf_subset": "default", + "languages": [ + "mar-Deva" + ] + } + ] + }, + "evaluation_time": 2.4426684379577637, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/MasakhaNEWSClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/MasakhaNEWSClassification.json new file mode 100644 index 0000000000..e31b1a5a14 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/MasakhaNEWSClassification.json @@ -0,0 +1,1003 @@ +{ + "dataset_revision": "18193f187b92da67168c655c9973a165ed9593dd", + "task_name": "MasakhaNEWSClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.357713, + "f1": 0.342347, + "f1_weighted": 0.347096, + "scores_per_experiment": [ + { + "accuracy": 0.348404, + "f1": 0.309922, + "f1_weighted": 0.321199 + }, + { + "accuracy": 0.37234, + "f1": 0.326028, + "f1_weighted": 0.33892 + }, + { + "accuracy": 0.420213, + "f1": 0.418135, + "f1_weighted": 0.420387 + }, + { + "accuracy": 0.332447, + "f1": 0.309379, + "f1_weighted": 0.316312 + }, + { + "accuracy": 0.340426, + "f1": 0.342919, + "f1_weighted": 0.344819 + }, + { + "accuracy": 0.375, + "f1": 0.368728, + "f1_weighted": 0.368494 + }, + { + "accuracy": 0.375, + "f1": 0.371097, + "f1_weighted": 0.37188 + }, + { + "accuracy": 0.37766, + "f1": 0.373, + "f1_weighted": 0.37597 + }, + { + "accuracy": 0.351064, + "f1": 0.319287, + "f1_weighted": 0.327382 + }, + { + "accuracy": 0.284574, + "f1": 0.284975, + "f1_weighted": 0.285597 + } + ], + "main_score": 0.357713, + "hf_subset": "amh", + "languages": [ + "amh-Ethi" + ] + }, + { + "accuracy": 0.74135, + "f1": 0.72955, + "f1_weighted": 0.738888, + "scores_per_experiment": [ + { + "accuracy": 0.74789, + "f1": 0.734648, + "f1_weighted": 0.746033 + }, + { + "accuracy": 0.738397, + "f1": 0.725787, + "f1_weighted": 0.734543 + }, + { + "accuracy": 0.751055, + "f1": 0.7401, + "f1_weighted": 0.750375 + }, + { + "accuracy": 0.731013, + "f1": 0.71279, + "f1_weighted": 0.723313 + }, + { + "accuracy": 0.751055, + "f1": 0.745728, + "f1_weighted": 0.751545 + }, + { + "accuracy": 0.719409, + "f1": 0.705837, + "f1_weighted": 0.717434 + }, + { + "accuracy": 0.726793, + "f1": 0.71702, + "f1_weighted": 0.72503 + }, + { + "accuracy": 0.743671, + "f1": 0.732975, + "f1_weighted": 0.741795 + }, + { + "accuracy": 0.728903, + "f1": 0.715215, + "f1_weighted": 0.725906 + }, + { + "accuracy": 0.775316, + "f1": 0.765405, + "f1_weighted": 0.772905 + } + ], + "main_score": 0.74135, + "hf_subset": "eng", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.733649, + "f1": 0.692356, + "f1_weighted": 0.737403, + "scores_per_experiment": [ + { + "accuracy": 0.808057, + "f1": 0.767932, + "f1_weighted": 0.8054 + }, + { + "accuracy": 0.734597, + "f1": 0.693322, + "f1_weighted": 0.73871 + }, + { + "accuracy": 0.758294, + "f1": 0.721201, + "f1_weighted": 0.760406 + }, + { + "accuracy": 0.654028, + "f1": 0.612427, + "f1_weighted": 0.660113 + }, + { + "accuracy": 0.741706, + "f1": 0.700236, + "f1_weighted": 0.744821 + }, + { + "accuracy": 0.706161, + "f1": 0.661888, + "f1_weighted": 0.706029 + }, + { + "accuracy": 0.744076, + "f1": 0.696098, + "f1_weighted": 0.759819 + }, + { + "accuracy": 0.729858, + "f1": 0.689571, + "f1_weighted": 0.72461 + }, + { + "accuracy": 0.758294, + "f1": 0.719803, + "f1_weighted": 0.761946 + }, + { + "accuracy": 0.701422, + "f1": 0.661078, + "f1_weighted": 0.712178 + } + ], + "main_score": 0.733649, + "hf_subset": "fra", + "languages": [ + "fra-Latn" + ] + }, + { + "accuracy": 0.590424, + "f1": 0.569955, + "f1_weighted": 0.588413, + "scores_per_experiment": [ + { + "accuracy": 0.582418, + "f1": 0.560976, + "f1_weighted": 0.580723 + }, + { + "accuracy": 0.583987, + "f1": 0.563264, + "f1_weighted": 0.585019 + }, + { + "accuracy": 0.599686, + "f1": 0.577166, + "f1_weighted": 0.595724 + }, + { + "accuracy": 0.601256, + "f1": 0.584001, + "f1_weighted": 0.6046 + }, + { + "accuracy": 0.585557, + "f1": 0.564974, + "f1_weighted": 0.585422 + }, + { + "accuracy": 0.609105, + "f1": 0.585955, + "f1_weighted": 0.602847 + }, + { + "accuracy": 0.599686, + "f1": 0.582491, + "f1_weighted": 0.598273 + }, + { + "accuracy": 0.590267, + "f1": 0.561552, + "f1_weighted": 0.576588 + }, + { + "accuracy": 0.591837, + "f1": 0.57712, + "f1_weighted": 0.595031 + }, + { + "accuracy": 0.56044, + "f1": 0.542047, + "f1_weighted": 0.559909 + } + ], + "main_score": 0.590424, + "hf_subset": "hau", + "languages": [ + "hau-Latn" + ] + }, + { + "accuracy": 0.496923, + "f1": 0.476734, + "f1_weighted": 0.49696, + "scores_per_experiment": [ + { + "accuracy": 0.507692, + "f1": 0.485923, + "f1_weighted": 0.513492 + }, + { + "accuracy": 0.482051, + "f1": 0.463445, + "f1_weighted": 0.484528 + }, + { + "accuracy": 0.489744, + "f1": 0.479189, + "f1_weighted": 0.487277 + }, + { + "accuracy": 0.479487, + "f1": 0.446589, + "f1_weighted": 0.481548 + }, + { + "accuracy": 0.453846, + "f1": 0.429197, + "f1_weighted": 0.435284 + }, + { + "accuracy": 0.507692, + "f1": 0.497327, + "f1_weighted": 0.51207 + }, + { + "accuracy": 0.515385, + "f1": 0.496199, + "f1_weighted": 0.502618 + }, + { + "accuracy": 0.5, + "f1": 0.47168, + "f1_weighted": 0.495166 + }, + { + "accuracy": 0.494872, + "f1": 0.475751, + "f1_weighted": 0.500799 + }, + { + "accuracy": 0.538462, + "f1": 0.522036, + "f1_weighted": 0.556814 + } + ], + "main_score": 0.496923, + "hf_subset": "ibo", + "languages": [ + "ibo-Latn" + ] + }, + { + "accuracy": 0.606857, + "f1": 0.624609, + "f1_weighted": 0.622266, + "scores_per_experiment": [ + { + "accuracy": 0.617143, + "f1": 0.646605, + "f1_weighted": 0.649619 + }, + { + "accuracy": 0.662857, + "f1": 0.675192, + "f1_weighted": 0.679372 + }, + { + "accuracy": 0.594286, + "f1": 0.616563, + "f1_weighted": 0.603779 + }, + { + "accuracy": 0.617143, + "f1": 0.592664, + "f1_weighted": 0.63199 + }, + { + "accuracy": 0.525714, + "f1": 0.564311, + "f1_weighted": 0.54425 + }, + { + "accuracy": 0.537143, + "f1": 0.589648, + "f1_weighted": 0.544586 + }, + { + "accuracy": 0.685714, + "f1": 0.698693, + "f1_weighted": 0.697291 + }, + { + "accuracy": 0.525714, + "f1": 0.570973, + "f1_weighted": 0.538494 + }, + { + "accuracy": 0.594286, + "f1": 0.604136, + "f1_weighted": 0.61403 + }, + { + "accuracy": 0.708571, + "f1": 0.687307, + "f1_weighted": 0.719246 + } + ], + "main_score": 0.606857, + "hf_subset": "lin", + "languages": [ + "lin-Latn" + ] + }, + { + "accuracy": 0.546637, + "f1": 0.507458, + "f1_weighted": 0.563988, + "scores_per_experiment": [ + { + "accuracy": 0.547085, + "f1": 0.478974, + "f1_weighted": 0.547039 + }, + { + "accuracy": 0.538117, + "f1": 0.498898, + "f1_weighted": 0.567022 + }, + { + "accuracy": 0.565022, + "f1": 0.526956, + "f1_weighted": 0.578574 + }, + { + "accuracy": 0.596413, + "f1": 0.556908, + "f1_weighted": 0.608949 + }, + { + "accuracy": 0.573991, + "f1": 0.506235, + "f1_weighted": 0.592412 + }, + { + "accuracy": 0.502242, + "f1": 0.486759, + "f1_weighted": 0.522876 + }, + { + "accuracy": 0.497758, + "f1": 0.459413, + "f1_weighted": 0.520727 + }, + { + "accuracy": 0.533632, + "f1": 0.501224, + "f1_weighted": 0.552459 + }, + { + "accuracy": 0.58296, + "f1": 0.543116, + "f1_weighted": 0.599771 + }, + { + "accuracy": 0.529148, + "f1": 0.516102, + "f1_weighted": 0.550056 + } + ], + "main_score": 0.546637, + "hf_subset": "lug", + "languages": [ + "lug-Latn" + ] + }, + { + "accuracy": 0.463077, + "f1": 0.415284, + "f1_weighted": 0.473647, + "scores_per_experiment": [ + { + "accuracy": 0.486154, + "f1": 0.430988, + "f1_weighted": 0.516003 + }, + { + "accuracy": 0.467692, + "f1": 0.418125, + "f1_weighted": 0.468654 + }, + { + "accuracy": 0.455385, + "f1": 0.389501, + "f1_weighted": 0.448477 + }, + { + "accuracy": 0.510769, + "f1": 0.44979, + "f1_weighted": 0.512027 + }, + { + "accuracy": 0.532308, + "f1": 0.478352, + "f1_weighted": 0.561158 + }, + { + "accuracy": 0.396923, + "f1": 0.34702, + "f1_weighted": 0.411227 + }, + { + "accuracy": 0.393846, + "f1": 0.370811, + "f1_weighted": 0.386998 + }, + { + "accuracy": 0.458462, + "f1": 0.421454, + "f1_weighted": 0.47544 + }, + { + "accuracy": 0.433846, + "f1": 0.391859, + "f1_weighted": 0.459722 + }, + { + "accuracy": 0.495385, + "f1": 0.454938, + "f1_weighted": 0.496764 + } + ], + "main_score": 0.463077, + "hf_subset": "orm", + "languages": [ + "orm-Ethi" + ] + }, + { + "accuracy": 0.886557, + "f1": 0.862561, + "f1_weighted": 0.888413, + "scores_per_experiment": [ + { + "accuracy": 0.911475, + "f1": 0.885059, + "f1_weighted": 0.912645 + }, + { + "accuracy": 0.878689, + "f1": 0.85486, + "f1_weighted": 0.880713 + }, + { + "accuracy": 0.872131, + "f1": 0.84785, + "f1_weighted": 0.873967 + }, + { + "accuracy": 0.914754, + "f1": 0.897067, + "f1_weighted": 0.915043 + }, + { + "accuracy": 0.878689, + "f1": 0.866436, + "f1_weighted": 0.879712 + }, + { + "accuracy": 0.878689, + "f1": 0.843922, + "f1_weighted": 0.879656 + }, + { + "accuracy": 0.885246, + "f1": 0.855958, + "f1_weighted": 0.888783 + }, + { + "accuracy": 0.832787, + "f1": 0.804608, + "f1_weighted": 0.836856 + }, + { + "accuracy": 0.918033, + "f1": 0.899501, + "f1_weighted": 0.920468 + }, + { + "accuracy": 0.895082, + "f1": 0.870346, + "f1_weighted": 0.896285 + } + ], + "main_score": 0.886557, + "hf_subset": "pcm", + "languages": [ + "pcm-Latn" + ] + }, + { + "accuracy": 0.556832, + "f1": 0.476063, + "f1_weighted": 0.586977, + "scores_per_experiment": [ + { + "accuracy": 0.5, + "f1": 0.442544, + "f1_weighted": 0.544125 + }, + { + "accuracy": 0.515528, + "f1": 0.439068, + "f1_weighted": 0.558055 + }, + { + "accuracy": 0.552795, + "f1": 0.439267, + "f1_weighted": 0.571255 + }, + { + "accuracy": 0.562112, + "f1": 0.485481, + "f1_weighted": 0.600582 + }, + { + "accuracy": 0.593168, + "f1": 0.531704, + "f1_weighted": 0.612512 + }, + { + "accuracy": 0.599379, + "f1": 0.483908, + "f1_weighted": 0.632888 + }, + { + "accuracy": 0.546584, + "f1": 0.458239, + "f1_weighted": 0.581997 + }, + { + "accuracy": 0.552795, + "f1": 0.500553, + "f1_weighted": 0.583272 + }, + { + "accuracy": 0.524845, + "f1": 0.467705, + "f1_weighted": 0.561805 + }, + { + "accuracy": 0.621118, + "f1": 0.512159, + "f1_weighted": 0.623282 + } + ], + "main_score": 0.556832, + "hf_subset": "run", + "languages": [ + "run-Latn" + ] + }, + { + "accuracy": 0.710027, + "f1": 0.712603, + "f1_weighted": 0.704602, + "scores_per_experiment": [ + { + "accuracy": 0.731707, + "f1": 0.740762, + "f1_weighted": 0.733469 + }, + { + "accuracy": 0.715447, + "f1": 0.720384, + "f1_weighted": 0.713059 + }, + { + "accuracy": 0.672087, + "f1": 0.674721, + "f1_weighted": 0.661717 + }, + { + "accuracy": 0.704607, + "f1": 0.709639, + "f1_weighted": 0.703436 + }, + { + "accuracy": 0.780488, + "f1": 0.784696, + "f1_weighted": 0.77949 + }, + { + "accuracy": 0.728997, + "f1": 0.731636, + "f1_weighted": 0.724897 + }, + { + "accuracy": 0.693767, + "f1": 0.69799, + "f1_weighted": 0.689205 + }, + { + "accuracy": 0.723577, + "f1": 0.725751, + "f1_weighted": 0.720519 + }, + { + "accuracy": 0.685637, + "f1": 0.67371, + "f1_weighted": 0.662152 + }, + { + "accuracy": 0.663957, + "f1": 0.666742, + "f1_weighted": 0.658075 + } + ], + "main_score": 0.710027, + "hf_subset": "sna", + "languages": [ + "sna-Latn" + ] + }, + { + "accuracy": 0.442517, + "f1": 0.411483, + "f1_weighted": 0.456515, + "scores_per_experiment": [ + { + "accuracy": 0.459184, + "f1": 0.462817, + "f1_weighted": 0.479997 + }, + { + "accuracy": 0.442177, + "f1": 0.388765, + "f1_weighted": 0.453038 + }, + { + "accuracy": 0.5, + "f1": 0.470434, + "f1_weighted": 0.520894 + }, + { + "accuracy": 0.489796, + "f1": 0.455899, + "f1_weighted": 0.50743 + }, + { + "accuracy": 0.465986, + "f1": 0.415064, + "f1_weighted": 0.476562 + }, + { + "accuracy": 0.391156, + "f1": 0.370556, + "f1_weighted": 0.408655 + }, + { + "accuracy": 0.503401, + "f1": 0.453603, + "f1_weighted": 0.523675 + }, + { + "accuracy": 0.363946, + "f1": 0.339013, + "f1_weighted": 0.376027 + }, + { + "accuracy": 0.370748, + "f1": 0.340972, + "f1_weighted": 0.363138 + }, + { + "accuracy": 0.438776, + "f1": 0.417712, + "f1_weighted": 0.455736 + } + ], + "main_score": 0.442517, + "hf_subset": "som", + "languages": [ + "som-Latn" + ] + }, + { + "accuracy": 0.498739, + "f1": 0.447779, + "f1_weighted": 0.508744, + "scores_per_experiment": [ + { + "accuracy": 0.460084, + "f1": 0.411914, + "f1_weighted": 0.477346 + }, + { + "accuracy": 0.506303, + "f1": 0.446064, + "f1_weighted": 0.511154 + }, + { + "accuracy": 0.533613, + "f1": 0.491423, + "f1_weighted": 0.524511 + }, + { + "accuracy": 0.483193, + "f1": 0.426795, + "f1_weighted": 0.500718 + }, + { + "accuracy": 0.487395, + "f1": 0.430228, + "f1_weighted": 0.497624 + }, + { + "accuracy": 0.447479, + "f1": 0.399618, + "f1_weighted": 0.455651 + }, + { + "accuracy": 0.529412, + "f1": 0.494095, + "f1_weighted": 0.550203 + }, + { + "accuracy": 0.504202, + "f1": 0.455088, + "f1_weighted": 0.521611 + }, + { + "accuracy": 0.539916, + "f1": 0.49764, + "f1_weighted": 0.557796 + }, + { + "accuracy": 0.495798, + "f1": 0.424929, + "f1_weighted": 0.490827 + } + ], + "main_score": 0.498739, + "hf_subset": "swa", + "languages": [ + "swa-Latn" + ] + }, + { + "accuracy": 0.292647, + "f1": 0.250172, + "f1_weighted": 0.287934, + "scores_per_experiment": [ + { + "accuracy": 0.330882, + "f1": 0.278432, + "f1_weighted": 0.329602 + }, + { + "accuracy": 0.363971, + "f1": 0.296718, + "f1_weighted": 0.367385 + }, + { + "accuracy": 0.261029, + "f1": 0.247502, + "f1_weighted": 0.249227 + }, + { + "accuracy": 0.220588, + "f1": 0.226728, + "f1_weighted": 0.222323 + }, + { + "accuracy": 0.338235, + "f1": 0.237493, + "f1_weighted": 0.323353 + }, + { + "accuracy": 0.275735, + "f1": 0.254477, + "f1_weighted": 0.279979 + }, + { + "accuracy": 0.290441, + "f1": 0.253614, + "f1_weighted": 0.294893 + }, + { + "accuracy": 0.3125, + "f1": 0.256741, + "f1_weighted": 0.305452 + }, + { + "accuracy": 0.279412, + "f1": 0.213071, + "f1_weighted": 0.247741 + }, + { + "accuracy": 0.253676, + "f1": 0.236943, + "f1_weighted": 0.259381 + } + ], + "main_score": 0.292647, + "hf_subset": "tir", + "languages": [ + "tir-Ethi" + ] + }, + { + "accuracy": 0.574074, + "f1": 0.467681, + "f1_weighted": 0.595365, + "scores_per_experiment": [ + { + "accuracy": 0.575758, + "f1": 0.471056, + "f1_weighted": 0.57344 + }, + { + "accuracy": 0.572391, + "f1": 0.442537, + "f1_weighted": 0.610641 + }, + { + "accuracy": 0.612795, + "f1": 0.503007, + "f1_weighted": 0.635984 + }, + { + "accuracy": 0.548822, + "f1": 0.459295, + "f1_weighted": 0.565031 + }, + { + "accuracy": 0.569024, + "f1": 0.474686, + "f1_weighted": 0.597109 + }, + { + "accuracy": 0.592593, + "f1": 0.457315, + "f1_weighted": 0.611311 + }, + { + "accuracy": 0.52862, + "f1": 0.431376, + "f1_weighted": 0.561585 + }, + { + "accuracy": 0.616162, + "f1": 0.523947, + "f1_weighted": 0.638863 + }, + { + "accuracy": 0.599327, + "f1": 0.487881, + "f1_weighted": 0.607157 + }, + { + "accuracy": 0.525253, + "f1": 0.425709, + "f1_weighted": 0.552528 + } + ], + "main_score": 0.574074, + "hf_subset": "xho", + "languages": [ + "xho-Latn" + ] + }, + { + "accuracy": 0.574453, + "f1": 0.579724, + "f1_weighted": 0.576597, + "scores_per_experiment": [ + { + "accuracy": 0.545012, + "f1": 0.552224, + "f1_weighted": 0.549677 + }, + { + "accuracy": 0.596107, + "f1": 0.602522, + "f1_weighted": 0.601371 + }, + { + "accuracy": 0.545012, + "f1": 0.553287, + "f1_weighted": 0.549228 + }, + { + "accuracy": 0.600973, + "f1": 0.607623, + "f1_weighted": 0.603292 + }, + { + "accuracy": 0.576642, + "f1": 0.590791, + "f1_weighted": 0.584873 + }, + { + "accuracy": 0.523114, + "f1": 0.526677, + "f1_weighted": 0.526084 + }, + { + "accuracy": 0.569343, + "f1": 0.567169, + "f1_weighted": 0.565953 + }, + { + "accuracy": 0.603406, + "f1": 0.609103, + "f1_weighted": 0.601368 + }, + { + "accuracy": 0.600973, + "f1": 0.604299, + "f1_weighted": 0.603512 + }, + { + "accuracy": 0.583942, + "f1": 0.58355, + "f1_weighted": 0.580615 + } + ], + "main_score": 0.574453, + "hf_subset": "yor", + "languages": [ + "yor-Latn" + ] + } + ] + }, + "evaluation_time": 152.59578013420105, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/MassiveIntentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/MassiveIntentClassification.json index 81eff1195b..79157daad9 100644 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/MassiveIntentClassification.json +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/MassiveIntentClassification.json @@ -1,15 +1,6337 @@ { - "test": { - "en": { - "accuracy": 0.6705110961667786, - "accuracy_stderr": 0.014293840351128607, - "f1": 0.6406652441082242, - "f1_stderr": 0.014492653713078963, - "main_score": 0.6705110961667786 - }, - "evaluation_time": 26.92 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "MassiveIntentClassification", - "dataset_revision": "072a486a144adf7f4479a4a0dddb2152e161e1ea" + "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6", + "task_name": "MassiveIntentClassification", + "mteb_version": "1.34.14", + "scores": { + "validation": [ + { + "accuracy": 0.339105, + "f1": 0.309992, + "f1_weighted": 0.324636, + "scores_per_experiment": [ + { + "accuracy": 0.35514, + "f1": 0.329596, + "f1_weighted": 0.330582 + }, + { + "accuracy": 0.32907, + "f1": 0.299839, + "f1_weighted": 0.318264 + }, + { + "accuracy": 0.335957, + "f1": 0.31779, + "f1_weighted": 0.320272 + }, + { + "accuracy": 0.315298, + "f1": 0.293433, + "f1_weighted": 0.293035 + }, + { + "accuracy": 0.372848, + "f1": 0.323399, + "f1_weighted": 0.357797 + }, + { + "accuracy": 0.338416, + "f1": 0.311745, + "f1_weighted": 0.331294 + }, + { + "accuracy": 0.330054, + "f1": 0.300843, + "f1_weighted": 0.320868 + }, + { + "accuracy": 0.345794, + "f1": 0.306982, + "f1_weighted": 0.329447 + }, + { + "accuracy": 0.333989, + "f1": 0.313802, + "f1_weighted": 0.318318 + }, + { + "accuracy": 0.334481, + "f1": 0.302492, + "f1_weighted": 0.326482 + } + ], + "main_score": 0.339105, + "hf_subset": "tr", + "languages": [ + "tur-Latn" + ] + }, + { + "accuracy": 0.022774, + "f1": 0.006226, + "f1_weighted": 0.007577, + "scores_per_experiment": [ + { + "accuracy": 0.01574, + "f1": 0.003126, + "f1_weighted": 0.001793 + }, + { + "accuracy": 0.028529, + "f1": 0.006713, + "f1_weighted": 0.016674 + }, + { + "accuracy": 0.022627, + "f1": 0.006504, + "f1_weighted": 0.00768 + }, + { + "accuracy": 0.01033, + "f1": 0.005142, + "f1_weighted": 0.002039 + }, + { + "accuracy": 0.029021, + "f1": 0.008224, + "f1_weighted": 0.009646 + }, + { + "accuracy": 0.027545, + "f1": 0.011323, + "f1_weighted": 0.009025 + }, + { + "accuracy": 0.025086, + "f1": 0.005732, + "f1_weighted": 0.007727 + }, + { + "accuracy": 0.021151, + "f1": 0.007096, + "f1_weighted": 0.006995 + }, + { + "accuracy": 0.015248, + "f1": 0.002008, + "f1_weighted": 0.004218 + }, + { + "accuracy": 0.032464, + "f1": 0.006388, + "f1_weighted": 0.009975 + } + ], + "main_score": 0.022774, + "hf_subset": "ka", + "languages": [ + "kat-Geor" + ] + }, + { + "accuracy": 0.026513, + "f1": 0.006644, + "f1_weighted": 0.00931, + "scores_per_experiment": [ + { + "accuracy": 0.04968, + "f1": 0.008119, + "f1_weighted": 0.012102 + }, + { + "accuracy": 0.025578, + "f1": 0.008941, + "f1_weighted": 0.01266 + }, + { + "accuracy": 0.012789, + "f1": 0.006124, + "f1_weighted": 0.0077 + }, + { + "accuracy": 0.0182, + "f1": 0.007195, + "f1_weighted": 0.008992 + }, + { + "accuracy": 0.024594, + "f1": 0.011799, + "f1_weighted": 0.014534 + }, + { + "accuracy": 0.018692, + "f1": 0.002385, + "f1_weighted": 0.002276 + }, + { + "accuracy": 0.016724, + "f1": 0.003849, + "f1_weighted": 0.007477 + }, + { + "accuracy": 0.01033, + "f1": 0.00289, + "f1_weighted": 0.00529 + }, + { + "accuracy": 0.063453, + "f1": 0.010499, + "f1_weighted": 0.019 + }, + { + "accuracy": 0.025086, + "f1": 0.004634, + "f1_weighted": 0.003066 + } + ], + "main_score": 0.026513, + "hf_subset": "ko", + "languages": [ + "kor-Kore" + ] + }, + { + "accuracy": 0.441859, + "f1": 0.431586, + "f1_weighted": 0.422223, + "scores_per_experiment": [ + { + "accuracy": 0.452533, + "f1": 0.437698, + "f1_weighted": 0.439746 + }, + { + "accuracy": 0.451549, + "f1": 0.440834, + "f1_weighted": 0.433372 + }, + { + "accuracy": 0.445155, + "f1": 0.430827, + "f1_weighted": 0.417362 + }, + { + "accuracy": 0.447614, + "f1": 0.431999, + "f1_weighted": 0.434257 + }, + { + "accuracy": 0.460403, + "f1": 0.43601, + "f1_weighted": 0.442983 + }, + { + "accuracy": 0.435809, + "f1": 0.436595, + "f1_weighted": 0.41218 + }, + { + "accuracy": 0.457452, + "f1": 0.455751, + "f1_weighted": 0.438589 + }, + { + "accuracy": 0.434333, + "f1": 0.420765, + "f1_weighted": 0.412972 + }, + { + "accuracy": 0.401869, + "f1": 0.400877, + "f1_weighted": 0.37176 + }, + { + "accuracy": 0.431874, + "f1": 0.424504, + "f1_weighted": 0.419008 + } + ], + "main_score": 0.441859, + "hf_subset": "it", + "languages": [ + "ita-Latn" + ] + }, + { + "accuracy": 0.172258, + "f1": 0.151467, + "f1_weighted": 0.158539, + "scores_per_experiment": [ + { + "accuracy": 0.170684, + "f1": 0.156643, + "f1_weighted": 0.152978 + }, + { + "accuracy": 0.154943, + "f1": 0.141641, + "f1_weighted": 0.142283 + }, + { + "accuracy": 0.161338, + "f1": 0.140027, + "f1_weighted": 0.157589 + }, + { + "accuracy": 0.172159, + "f1": 0.145876, + "f1_weighted": 0.160166 + }, + { + "accuracy": 0.212494, + "f1": 0.172396, + "f1_weighted": 0.207664 + }, + { + "accuracy": 0.15937, + "f1": 0.149351, + "f1_weighted": 0.136771 + }, + { + "accuracy": 0.173143, + "f1": 0.153297, + "f1_weighted": 0.15619 + }, + { + "accuracy": 0.186424, + "f1": 0.148144, + "f1_weighted": 0.173651 + }, + { + "accuracy": 0.15937, + "f1": 0.153449, + "f1_weighted": 0.140913 + }, + { + "accuracy": 0.172651, + "f1": 0.153851, + "f1_weighted": 0.157181 + } + ], + "main_score": 0.172258, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + }, + { + "accuracy": 0.035022, + "f1": 0.006874, + "f1_weighted": 0.010255, + "scores_per_experiment": [ + { + "accuracy": 0.059518, + "f1": 0.006497, + "f1_weighted": 0.013752 + }, + { + "accuracy": 0.009346, + "f1": 0.006524, + "f1_weighted": 0.003341 + }, + { + "accuracy": 0.009346, + "f1": 0.006876, + "f1_weighted": 0.005063 + }, + { + "accuracy": 0.048205, + "f1": 0.008363, + "f1_weighted": 0.016926 + }, + { + "accuracy": 0.064437, + "f1": 0.007198, + "f1_weighted": 0.016367 + }, + { + "accuracy": 0.005903, + "f1": 0.00339, + "f1_weighted": 0.001662 + }, + { + "accuracy": 0.010821, + "f1": 0.00941, + "f1_weighted": 0.007434 + }, + { + "accuracy": 0.060994, + "f1": 0.005656, + "f1_weighted": 0.016589 + }, + { + "accuracy": 0.062469, + "f1": 0.012485, + "f1_weighted": 0.017579 + }, + { + "accuracy": 0.019183, + "f1": 0.002346, + "f1_weighted": 0.003834 + } + ], + "main_score": 0.035022, + "hf_subset": "fa", + "languages": [ + "fas-Arab" + ] + }, + { + "accuracy": 0.160059, + "f1": 0.146011, + "f1_weighted": 0.137443, + "scores_per_experiment": [ + { + "accuracy": 0.172159, + "f1": 0.154962, + "f1_weighted": 0.148554 + }, + { + "accuracy": 0.145106, + "f1": 0.12975, + "f1_weighted": 0.131942 + }, + { + "accuracy": 0.16183, + "f1": 0.142748, + "f1_weighted": 0.146913 + }, + { + "accuracy": 0.177078, + "f1": 0.15299, + "f1_weighted": 0.154792 + }, + { + "accuracy": 0.157403, + "f1": 0.149861, + "f1_weighted": 0.141838 + }, + { + "accuracy": 0.155927, + "f1": 0.140605, + "f1_weighted": 0.129509 + }, + { + "accuracy": 0.167241, + "f1": 0.155147, + "f1_weighted": 0.147504 + }, + { + "accuracy": 0.168224, + "f1": 0.160904, + "f1_weighted": 0.137113 + }, + { + "accuracy": 0.15396, + "f1": 0.144752, + "f1_weighted": 0.125765 + }, + { + "accuracy": 0.141663, + "f1": 0.128394, + "f1_weighted": 0.110503 + } + ], + "main_score": 0.160059, + "hf_subset": "mn", + "languages": [ + "mon-Cyrl" + ] + }, + { + "accuracy": 0.026513, + "f1": 0.013957, + "f1_weighted": 0.014861, + "scores_per_experiment": [ + { + "accuracy": 0.027054, + "f1": 0.016499, + "f1_weighted": 0.017387 + }, + { + "accuracy": 0.009346, + "f1": 0.005521, + "f1_weighted": 0.002912 + }, + { + "accuracy": 0.0182, + "f1": 0.01036, + "f1_weighted": 0.007621 + }, + { + "accuracy": 0.030005, + "f1": 0.01813, + "f1_weighted": 0.021008 + }, + { + "accuracy": 0.032464, + "f1": 0.011092, + "f1_weighted": 0.015514 + }, + { + "accuracy": 0.032956, + "f1": 0.016367, + "f1_weighted": 0.019005 + }, + { + "accuracy": 0.015248, + "f1": 0.005741, + "f1_weighted": 0.005074 + }, + { + "accuracy": 0.016724, + "f1": 0.016564, + "f1_weighted": 0.015977 + }, + { + "accuracy": 0.05755, + "f1": 0.017468, + "f1_weighted": 0.022392 + }, + { + "accuracy": 0.025578, + "f1": 0.021824, + "f1_weighted": 0.02172 + } + ], + "main_score": 0.026513, + "hf_subset": "kn", + "languages": [ + "kan-Knda" + ] + }, + { + "accuracy": 0.029808, + "f1": 0.009204, + "f1_weighted": 0.012027, + "scores_per_experiment": [ + { + "accuracy": 0.056075, + "f1": 0.004865, + "f1_weighted": 0.011527 + }, + { + "accuracy": 0.016232, + "f1": 0.007365, + "f1_weighted": 0.004044 + }, + { + "accuracy": 0.016724, + "f1": 0.006687, + "f1_weighted": 0.007523 + }, + { + "accuracy": 0.022627, + "f1": 0.00617, + "f1_weighted": 0.00718 + }, + { + "accuracy": 0.028037, + "f1": 0.011673, + "f1_weighted": 0.012181 + }, + { + "accuracy": 0.01574, + "f1": 0.005625, + "f1_weighted": 0.006582 + }, + { + "accuracy": 0.013281, + "f1": 0.011088, + "f1_weighted": 0.009255 + }, + { + "accuracy": 0.061485, + "f1": 0.018557, + "f1_weighted": 0.03226 + }, + { + "accuracy": 0.045745, + "f1": 0.011973, + "f1_weighted": 0.020246 + }, + { + "accuracy": 0.022135, + "f1": 0.00804, + "f1_weighted": 0.009468 + } + ], + "main_score": 0.029808, + "hf_subset": "hy", + "languages": [ + "hye-Armn" + ] + }, + { + "accuracy": 0.041515, + "f1": 0.025743, + "f1_weighted": 0.02977, + "scores_per_experiment": [ + { + "accuracy": 0.070339, + "f1": 0.035949, + "f1_weighted": 0.057247 + }, + { + "accuracy": 0.028529, + "f1": 0.016311, + "f1_weighted": 0.014843 + }, + { + "accuracy": 0.043286, + "f1": 0.038223, + "f1_weighted": 0.038782 + }, + { + "accuracy": 0.031481, + "f1": 0.022273, + "f1_weighted": 0.015637 + }, + { + "accuracy": 0.024102, + "f1": 0.015102, + "f1_weighted": 0.016041 + }, + { + "accuracy": 0.053615, + "f1": 0.040578, + "f1_weighted": 0.040938 + }, + { + "accuracy": 0.035416, + "f1": 0.02407, + "f1_weighted": 0.03203 + }, + { + "accuracy": 0.032464, + "f1": 0.024288, + "f1_weighted": 0.031278 + }, + { + "accuracy": 0.074766, + "f1": 0.02505, + "f1_weighted": 0.041186 + }, + { + "accuracy": 0.021151, + "f1": 0.015583, + "f1_weighted": 0.00972 + } + ], + "main_score": 0.041515, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ] + }, + { + "accuracy": 0.438072, + "f1": 0.398294, + "f1_weighted": 0.426488, + "scores_per_experiment": [ + { + "accuracy": 0.438269, + "f1": 0.40366, + "f1_weighted": 0.434951 + }, + { + "accuracy": 0.450566, + "f1": 0.408064, + "f1_weighted": 0.443384 + }, + { + "accuracy": 0.420561, + "f1": 0.384126, + "f1_weighted": 0.410685 + }, + { + "accuracy": 0.454993, + "f1": 0.406883, + "f1_weighted": 0.457422 + }, + { + "accuracy": 0.472209, + "f1": 0.412413, + "f1_weighted": 0.455862 + }, + { + "accuracy": 0.435809, + "f1": 0.407618, + "f1_weighted": 0.424519 + }, + { + "accuracy": 0.436793, + "f1": 0.401537, + "f1_weighted": 0.429979 + }, + { + "accuracy": 0.446631, + "f1": 0.402889, + "f1_weighted": 0.434322 + }, + { + "accuracy": 0.416134, + "f1": 0.381398, + "f1_weighted": 0.384264 + }, + { + "accuracy": 0.408756, + "f1": 0.37435, + "f1_weighted": 0.389488 + } + ], + "main_score": 0.438072, + "hf_subset": "ro", + "languages": [ + "ron-Latn" + ] + }, + { + "accuracy": 0.38603, + "f1": 0.356998, + "f1_weighted": 0.37033, + "scores_per_experiment": [ + { + "accuracy": 0.39154, + "f1": 0.369151, + "f1_weighted": 0.380627 + }, + { + "accuracy": 0.367929, + "f1": 0.34231, + "f1_weighted": 0.354427 + }, + { + "accuracy": 0.383669, + "f1": 0.348153, + "f1_weighted": 0.367329 + }, + { + "accuracy": 0.402361, + "f1": 0.374194, + "f1_weighted": 0.38162 + }, + { + "accuracy": 0.405804, + "f1": 0.372255, + "f1_weighted": 0.391544 + }, + { + "accuracy": 0.379242, + "f1": 0.360324, + "f1_weighted": 0.366911 + }, + { + "accuracy": 0.377767, + "f1": 0.338931, + "f1_weighted": 0.365598 + }, + { + "accuracy": 0.385145, + "f1": 0.344794, + "f1_weighted": 0.369387 + }, + { + "accuracy": 0.376783, + "f1": 0.359233, + "f1_weighted": 0.351472 + }, + { + "accuracy": 0.390064, + "f1": 0.360637, + "f1_weighted": 0.374389 + } + ], + "main_score": 0.38603, + "hf_subset": "nb", + "languages": [ + "nob-Latn" + ] + }, + { + "accuracy": 0.372602, + "f1": 0.345177, + "f1_weighted": 0.357665, + "scores_per_experiment": [ + { + "accuracy": 0.393999, + "f1": 0.361304, + "f1_weighted": 0.372734 + }, + { + "accuracy": 0.34727, + "f1": 0.329574, + "f1_weighted": 0.33162 + }, + { + "accuracy": 0.355632, + "f1": 0.338953, + "f1_weighted": 0.331396 + }, + { + "accuracy": 0.383178, + "f1": 0.340281, + "f1_weighted": 0.369158 + }, + { + "accuracy": 0.389572, + "f1": 0.347419, + "f1_weighted": 0.371179 + }, + { + "accuracy": 0.372848, + "f1": 0.354626, + "f1_weighted": 0.365917 + }, + { + "accuracy": 0.364486, + "f1": 0.330965, + "f1_weighted": 0.361678 + }, + { + "accuracy": 0.382686, + "f1": 0.34976, + "f1_weighted": 0.364379 + }, + { + "accuracy": 0.354156, + "f1": 0.34389, + "f1_weighted": 0.327091 + }, + { + "accuracy": 0.382194, + "f1": 0.354998, + "f1_weighted": 0.3815 + } + ], + "main_score": 0.372602, + "hf_subset": "sv", + "languages": [ + "swe-Latn" + ] + }, + { + "accuracy": 0.358928, + "f1": 0.3295, + "f1_weighted": 0.353516, + "scores_per_experiment": [ + { + "accuracy": 0.364486, + "f1": 0.339876, + "f1_weighted": 0.358621 + }, + { + "accuracy": 0.351205, + "f1": 0.327638, + "f1_weighted": 0.343592 + }, + { + "accuracy": 0.365962, + "f1": 0.337668, + "f1_weighted": 0.367382 + }, + { + "accuracy": 0.365962, + "f1": 0.345438, + "f1_weighted": 0.367942 + }, + { + "accuracy": 0.36547, + "f1": 0.316463, + "f1_weighted": 0.363972 + }, + { + "accuracy": 0.369897, + "f1": 0.338942, + "f1_weighted": 0.365019 + }, + { + "accuracy": 0.33694, + "f1": 0.307924, + "f1_weighted": 0.321879 + }, + { + "accuracy": 0.348254, + "f1": 0.318099, + "f1_weighted": 0.340209 + }, + { + "accuracy": 0.361535, + "f1": 0.333007, + "f1_weighted": 0.349629 + }, + { + "accuracy": 0.359567, + "f1": 0.329949, + "f1_weighted": 0.356914 + } + ], + "main_score": 0.358928, + "hf_subset": "pl", + "languages": [ + "pol-Latn" + ] + }, + { + "accuracy": 0.028431, + "f1": 0.017615, + "f1_weighted": 0.023587, + "scores_per_experiment": [ + { + "accuracy": 0.036399, + "f1": 0.029347, + "f1_weighted": 0.02989 + }, + { + "accuracy": 0.030005, + "f1": 0.014606, + "f1_weighted": 0.018593 + }, + { + "accuracy": 0.026562, + "f1": 0.018028, + "f1_weighted": 0.025537 + }, + { + "accuracy": 0.035908, + "f1": 0.018266, + "f1_weighted": 0.027079 + }, + { + "accuracy": 0.037875, + "f1": 0.024911, + "f1_weighted": 0.027262 + }, + { + "accuracy": 0.027545, + "f1": 0.022165, + "f1_weighted": 0.030694 + }, + { + "accuracy": 0.025086, + "f1": 0.016132, + "f1_weighted": 0.027354 + }, + { + "accuracy": 0.009346, + "f1": 0.00074, + "f1_weighted": 0.000216 + }, + { + "accuracy": 0.030005, + "f1": 0.011628, + "f1_weighted": 0.020275 + }, + { + "accuracy": 0.025578, + "f1": 0.020329, + "f1_weighted": 0.028968 + } + ], + "main_score": 0.028431, + "hf_subset": "zh-TW", + "languages": [ + "cmo-Hant" + ] + }, + { + "accuracy": 0.367782, + "f1": 0.330159, + "f1_weighted": 0.346769, + "scores_per_experiment": [ + { + "accuracy": 0.367437, + "f1": 0.338137, + "f1_weighted": 0.348383 + }, + { + "accuracy": 0.372356, + "f1": 0.325221, + "f1_weighted": 0.355903 + }, + { + "accuracy": 0.398426, + "f1": 0.348431, + "f1_weighted": 0.380363 + }, + { + "accuracy": 0.376291, + "f1": 0.336447, + "f1_weighted": 0.348973 + }, + { + "accuracy": 0.381702, + "f1": 0.322604, + "f1_weighted": 0.359131 + }, + { + "accuracy": 0.3576, + "f1": 0.318372, + "f1_weighted": 0.33933 + }, + { + "accuracy": 0.383669, + "f1": 0.345039, + "f1_weighted": 0.362996 + }, + { + "accuracy": 0.363994, + "f1": 0.31889, + "f1_weighted": 0.341097 + }, + { + "accuracy": 0.32366, + "f1": 0.320409, + "f1_weighted": 0.298326 + }, + { + "accuracy": 0.352681, + "f1": 0.328037, + "f1_weighted": 0.333184 + } + ], + "main_score": 0.367782, + "hf_subset": "tl", + "languages": [ + "tgl-Latn" + ] + }, + { + "accuracy": 0.02912, + "f1": 0.006615, + "f1_weighted": 0.010712, + "scores_per_experiment": [ + { + "accuracy": 0.054599, + "f1": 0.005575, + "f1_weighted": 0.013146 + }, + { + "accuracy": 0.014265, + "f1": 0.004057, + "f1_weighted": 0.003298 + }, + { + "accuracy": 0.018692, + "f1": 0.005464, + "f1_weighted": 0.007931 + }, + { + "accuracy": 0.022135, + "f1": 0.009242, + "f1_weighted": 0.010342 + }, + { + "accuracy": 0.027054, + "f1": 0.007388, + "f1_weighted": 0.012319 + }, + { + "accuracy": 0.033448, + "f1": 0.008813, + "f1_weighted": 0.015858 + }, + { + "accuracy": 0.01574, + "f1": 0.004936, + "f1_weighted": 0.009514 + }, + { + "accuracy": 0.00787, + "f1": 0.00585, + "f1_weighted": 0.006287 + }, + { + "accuracy": 0.071323, + "f1": 0.010991, + "f1_weighted": 0.023884 + }, + { + "accuracy": 0.02607, + "f1": 0.003836, + "f1_weighted": 0.004544 + } + ], + "main_score": 0.02912, + "hf_subset": "ml", + "languages": [ + "mal-Mlym" + ] + }, + { + "accuracy": 0.447073, + "f1": 0.421453, + "f1_weighted": 0.436943, + "scores_per_experiment": [ + { + "accuracy": 0.442204, + "f1": 0.433739, + "f1_weighted": 0.435384 + }, + { + "accuracy": 0.459911, + "f1": 0.417339, + "f1_weighted": 0.462139 + }, + { + "accuracy": 0.455485, + "f1": 0.4302, + "f1_weighted": 0.443425 + }, + { + "accuracy": 0.45696, + "f1": 0.425743, + "f1_weighted": 0.448251 + }, + { + "accuracy": 0.476636, + "f1": 0.437163, + "f1_weighted": 0.466577 + }, + { + "accuracy": 0.452533, + "f1": 0.423749, + "f1_weighted": 0.443949 + }, + { + "accuracy": 0.42302, + "f1": 0.411435, + "f1_weighted": 0.412714 + }, + { + "accuracy": 0.439252, + "f1": 0.395678, + "f1_weighted": 0.422779 + }, + { + "accuracy": 0.427939, + "f1": 0.422184, + "f1_weighted": 0.408441 + }, + { + "accuracy": 0.436793, + "f1": 0.417299, + "f1_weighted": 0.425771 + } + ], + "main_score": 0.447073, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ] + }, + { + "accuracy": 0.678062, + "f1": 0.631173, + "f1_weighted": 0.66225, + "scores_per_experiment": [ + { + "accuracy": 0.677324, + "f1": 0.620352, + "f1_weighted": 0.66432 + }, + { + "accuracy": 0.695524, + "f1": 0.649813, + "f1_weighted": 0.686637 + }, + { + "accuracy": 0.69454, + "f1": 0.636937, + "f1_weighted": 0.680295 + }, + { + "accuracy": 0.694048, + "f1": 0.655345, + "f1_weighted": 0.682222 + }, + { + "accuracy": 0.67634, + "f1": 0.615007, + "f1_weighted": 0.654064 + }, + { + "accuracy": 0.65273, + "f1": 0.624593, + "f1_weighted": 0.634758 + }, + { + "accuracy": 0.664535, + "f1": 0.622276, + "f1_weighted": 0.649674 + }, + { + "accuracy": 0.666011, + "f1": 0.617394, + "f1_weighted": 0.649733 + }, + { + "accuracy": 0.66306, + "f1": 0.620689, + "f1_weighted": 0.633232 + }, + { + "accuracy": 0.696508, + "f1": 0.649327, + "f1_weighted": 0.687569 + } + ], + "main_score": 0.678062, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.336399, + "f1": 0.308853, + "f1_weighted": 0.330161, + "scores_per_experiment": [ + { + "accuracy": 0.368421, + "f1": 0.340767, + "f1_weighted": 0.360153 + }, + { + "accuracy": 0.319233, + "f1": 0.285816, + "f1_weighted": 0.313711 + }, + { + "accuracy": 0.326611, + "f1": 0.30097, + "f1_weighted": 0.321871 + }, + { + "accuracy": 0.360059, + "f1": 0.333374, + "f1_weighted": 0.360713 + }, + { + "accuracy": 0.349729, + "f1": 0.296296, + "f1_weighted": 0.337526 + }, + { + "accuracy": 0.307919, + "f1": 0.295867, + "f1_weighted": 0.30385 + }, + { + "accuracy": 0.342843, + "f1": 0.315608, + "f1_weighted": 0.333121 + }, + { + "accuracy": 0.349238, + "f1": 0.312382, + "f1_weighted": 0.338385 + }, + { + "accuracy": 0.32907, + "f1": 0.320766, + "f1_weighted": 0.32727 + }, + { + "accuracy": 0.310871, + "f1": 0.286689, + "f1_weighted": 0.305011 + } + ], + "main_score": 0.336399, + "hf_subset": "cy", + "languages": [ + "cym-Latn" + ] + }, + { + "accuracy": 0.443974, + "f1": 0.400664, + "f1_weighted": 0.434786, + "scores_per_experiment": [ + { + "accuracy": 0.437285, + "f1": 0.398548, + "f1_weighted": 0.433323 + }, + { + "accuracy": 0.455976, + "f1": 0.407058, + "f1_weighted": 0.453634 + }, + { + "accuracy": 0.45942, + "f1": 0.399981, + "f1_weighted": 0.445941 + }, + { + "accuracy": 0.461387, + "f1": 0.42169, + "f1_weighted": 0.449565 + }, + { + "accuracy": 0.454501, + "f1": 0.409178, + "f1_weighted": 0.443185 + }, + { + "accuracy": 0.443187, + "f1": 0.399259, + "f1_weighted": 0.437322 + }, + { + "accuracy": 0.431382, + "f1": 0.384953, + "f1_weighted": 0.423171 + }, + { + "accuracy": 0.448598, + "f1": 0.398322, + "f1_weighted": 0.435559 + }, + { + "accuracy": 0.424004, + "f1": 0.403494, + "f1_weighted": 0.409709 + }, + { + "accuracy": 0.424004, + "f1": 0.384152, + "f1_weighted": 0.416448 + } + ], + "main_score": 0.443974, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ] + }, + { + "accuracy": 0.334235, + "f1": 0.309714, + "f1_weighted": 0.320909, + "scores_per_experiment": [ + { + "accuracy": 0.341859, + "f1": 0.325402, + "f1_weighted": 0.327262 + }, + { + "accuracy": 0.341859, + "f1": 0.313532, + "f1_weighted": 0.332834 + }, + { + "accuracy": 0.349729, + "f1": 0.325712, + "f1_weighted": 0.341499 + }, + { + "accuracy": 0.324643, + "f1": 0.305397, + "f1_weighted": 0.317696 + }, + { + "accuracy": 0.353173, + "f1": 0.31523, + "f1_weighted": 0.341835 + }, + { + "accuracy": 0.335465, + "f1": 0.311936, + "f1_weighted": 0.313946 + }, + { + "accuracy": 0.325627, + "f1": 0.311236, + "f1_weighted": 0.30726 + }, + { + "accuracy": 0.339892, + "f1": 0.303821, + "f1_weighted": 0.324859 + }, + { + "accuracy": 0.3212, + "f1": 0.308639, + "f1_weighted": 0.3085 + }, + { + "accuracy": 0.308903, + "f1": 0.27623, + "f1_weighted": 0.293396 + } + ], + "main_score": 0.334235, + "hf_subset": "jv", + "languages": [ + "jav-Latn" + ] + }, + { + "accuracy": 0.380374, + "f1": 0.343252, + "f1_weighted": 0.370775, + "scores_per_experiment": [ + { + "accuracy": 0.390064, + "f1": 0.356716, + "f1_weighted": 0.381027 + }, + { + "accuracy": 0.392523, + "f1": 0.351228, + "f1_weighted": 0.391943 + }, + { + "accuracy": 0.371372, + "f1": 0.343448, + "f1_weighted": 0.363434 + }, + { + "accuracy": 0.37334, + "f1": 0.335405, + "f1_weighted": 0.355368 + }, + { + "accuracy": 0.40728, + "f1": 0.354354, + "f1_weighted": 0.40441 + }, + { + "accuracy": 0.387113, + "f1": 0.351624, + "f1_weighted": 0.384842 + }, + { + "accuracy": 0.354156, + "f1": 0.325273, + "f1_weighted": 0.335909 + }, + { + "accuracy": 0.392031, + "f1": 0.34672, + "f1_weighted": 0.381804 + }, + { + "accuracy": 0.36301, + "f1": 0.342946, + "f1_weighted": 0.341366 + }, + { + "accuracy": 0.372848, + "f1": 0.324807, + "f1_weighted": 0.367644 + } + ], + "main_score": 0.380374, + "hf_subset": "af", + "languages": [ + "afr-Latn" + ] + }, + { + "accuracy": 0.023856, + "f1": 0.009866, + "f1_weighted": 0.011076, + "scores_per_experiment": [ + { + "accuracy": 0.059026, + "f1": 0.008506, + "f1_weighted": 0.018347 + }, + { + "accuracy": 0.018692, + "f1": 0.00761, + "f1_weighted": 0.006413 + }, + { + "accuracy": 0.023119, + "f1": 0.023609, + "f1_weighted": 0.017035 + }, + { + "accuracy": 0.014757, + "f1": 0.006995, + "f1_weighted": 0.00843 + }, + { + "accuracy": 0.015248, + "f1": 0.004873, + "f1_weighted": 0.00578 + }, + { + "accuracy": 0.016232, + "f1": 0.008471, + "f1_weighted": 0.009043 + }, + { + "accuracy": 0.018692, + "f1": 0.005925, + "f1_weighted": 0.009143 + }, + { + "accuracy": 0.03394, + "f1": 0.009005, + "f1_weighted": 0.011447 + }, + { + "accuracy": 0.021151, + "f1": 0.013811, + "f1_weighted": 0.019213 + }, + { + "accuracy": 0.017708, + "f1": 0.009852, + "f1_weighted": 0.005908 + } + ], + "main_score": 0.023856, + "hf_subset": "bn", + "languages": [ + "ben-Beng" + ] + }, + { + "accuracy": 0.516232, + "f1": 0.480241, + "f1_weighted": 0.507009, + "scores_per_experiment": [ + { + "accuracy": 0.505657, + "f1": 0.470651, + "f1_weighted": 0.49973 + }, + { + "accuracy": 0.540089, + "f1": 0.50554, + "f1_weighted": 0.538162 + }, + { + "accuracy": 0.496803, + "f1": 0.465203, + "f1_weighted": 0.491371 + }, + { + "accuracy": 0.531727, + "f1": 0.48809, + "f1_weighted": 0.524171 + }, + { + "accuracy": 0.542056, + "f1": 0.49154, + "f1_weighted": 0.533132 + }, + { + "accuracy": 0.50664, + "f1": 0.48541, + "f1_weighted": 0.491643 + }, + { + "accuracy": 0.526316, + "f1": 0.489654, + "f1_weighted": 0.522425 + }, + { + "accuracy": 0.515986, + "f1": 0.462897, + "f1_weighted": 0.506782 + }, + { + "accuracy": 0.491392, + "f1": 0.485733, + "f1_weighted": 0.468616 + }, + { + "accuracy": 0.505657, + "f1": 0.457688, + "f1_weighted": 0.494056 + } + ], + "main_score": 0.516232, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ] + }, + { + "accuracy": 0.044712, + "f1": 0.01519, + "f1_weighted": 0.018539, + "scores_per_experiment": [ + { + "accuracy": 0.09395, + "f1": 0.028928, + "f1_weighted": 0.037438 + }, + { + "accuracy": 0.035908, + "f1": 0.016178, + "f1_weighted": 0.015783 + }, + { + "accuracy": 0.020659, + "f1": 0.010139, + "f1_weighted": 0.011601 + }, + { + "accuracy": 0.020659, + "f1": 0.007018, + "f1_weighted": 0.007056 + }, + { + "accuracy": 0.070339, + "f1": 0.0185, + "f1_weighted": 0.026193 + }, + { + "accuracy": 0.036399, + "f1": 0.014883, + "f1_weighted": 0.016853 + }, + { + "accuracy": 0.04968, + "f1": 0.018051, + "f1_weighted": 0.021883 + }, + { + "accuracy": 0.017708, + "f1": 0.009349, + "f1_weighted": 0.009535 + }, + { + "accuracy": 0.065912, + "f1": 0.012264, + "f1_weighted": 0.02083 + }, + { + "accuracy": 0.035908, + "f1": 0.016588, + "f1_weighted": 0.018218 + } + ], + "main_score": 0.044712, + "hf_subset": "my", + "languages": [ + "mya-Mymr" + ] + }, + { + "accuracy": 0.383079, + "f1": 0.350178, + "f1_weighted": 0.364084, + "scores_per_experiment": [ + { + "accuracy": 0.378751, + "f1": 0.355383, + "f1_weighted": 0.363526 + }, + { + "accuracy": 0.380226, + "f1": 0.35102, + "f1_weighted": 0.377564 + }, + { + "accuracy": 0.367437, + "f1": 0.336314, + "f1_weighted": 0.34445 + }, + { + "accuracy": 0.37088, + "f1": 0.345976, + "f1_weighted": 0.340916 + }, + { + "accuracy": 0.414166, + "f1": 0.352779, + "f1_weighted": 0.394704 + }, + { + "accuracy": 0.37334, + "f1": 0.341771, + "f1_weighted": 0.346671 + }, + { + "accuracy": 0.382194, + "f1": 0.357128, + "f1_weighted": 0.359927 + }, + { + "accuracy": 0.399902, + "f1": 0.358039, + "f1_weighted": 0.380691 + }, + { + "accuracy": 0.378751, + "f1": 0.359532, + "f1_weighted": 0.361651 + }, + { + "accuracy": 0.385145, + "f1": 0.34384, + "f1_weighted": 0.370742 + } + ], + "main_score": 0.383079, + "hf_subset": "nl", + "languages": [ + "nld-Latn" + ] + }, + { + "accuracy": 0.331923, + "f1": 0.301541, + "f1_weighted": 0.313868, + "scores_per_experiment": [ + { + "accuracy": 0.349729, + "f1": 0.315519, + "f1_weighted": 0.345734 + }, + { + "accuracy": 0.309887, + "f1": 0.272558, + "f1_weighted": 0.285414 + }, + { + "accuracy": 0.332514, + "f1": 0.3099, + "f1_weighted": 0.304964 + }, + { + "accuracy": 0.326611, + "f1": 0.294463, + "f1_weighted": 0.309417 + }, + { + "accuracy": 0.355632, + "f1": 0.30464, + "f1_weighted": 0.342553 + }, + { + "accuracy": 0.357108, + "f1": 0.334193, + "f1_weighted": 0.342723 + }, + { + "accuracy": 0.318249, + "f1": 0.294475, + "f1_weighted": 0.296715 + }, + { + "accuracy": 0.313822, + "f1": 0.282329, + "f1_weighted": 0.294911 + }, + { + "accuracy": 0.315298, + "f1": 0.297736, + "f1_weighted": 0.290841 + }, + { + "accuracy": 0.340384, + "f1": 0.309595, + "f1_weighted": 0.325411 + } + ], + "main_score": 0.331923, + "hf_subset": "hu", + "languages": [ + "hun-Latn" + ] + }, + { + "accuracy": 0.377865, + "f1": 0.347278, + "f1_weighted": 0.364891, + "scores_per_experiment": [ + { + "accuracy": 0.419085, + "f1": 0.377065, + "f1_weighted": 0.409355 + }, + { + "accuracy": 0.395967, + "f1": 0.372272, + "f1_weighted": 0.382104 + }, + { + "accuracy": 0.367929, + "f1": 0.339241, + "f1_weighted": 0.350043 + }, + { + "accuracy": 0.372848, + "f1": 0.345962, + "f1_weighted": 0.364613 + }, + { + "accuracy": 0.374816, + "f1": 0.335731, + "f1_weighted": 0.360868 + }, + { + "accuracy": 0.353173, + "f1": 0.318648, + "f1_weighted": 0.347022 + }, + { + "accuracy": 0.378259, + "f1": 0.348721, + "f1_weighted": 0.366629 + }, + { + "accuracy": 0.392523, + "f1": 0.356283, + "f1_weighted": 0.375228 + }, + { + "accuracy": 0.357108, + "f1": 0.346623, + "f1_weighted": 0.337881 + }, + { + "accuracy": 0.366945, + "f1": 0.332234, + "f1_weighted": 0.355173 + } + ], + "main_score": 0.377865, + "hf_subset": "lv", + "languages": [ + "lav-Latn" + ] + }, + { + "accuracy": 0.404033, + "f1": 0.380979, + "f1_weighted": 0.393748, + "scores_per_experiment": [ + { + "accuracy": 0.410723, + "f1": 0.377713, + "f1_weighted": 0.40593 + }, + { + "accuracy": 0.413674, + "f1": 0.387855, + "f1_weighted": 0.409368 + }, + { + "accuracy": 0.395967, + "f1": 0.37799, + "f1_weighted": 0.380355 + }, + { + "accuracy": 0.384653, + "f1": 0.36714, + "f1_weighted": 0.374862 + }, + { + "accuracy": 0.42548, + "f1": 0.390673, + "f1_weighted": 0.417871 + }, + { + "accuracy": 0.393507, + "f1": 0.382971, + "f1_weighted": 0.38271 + }, + { + "accuracy": 0.412199, + "f1": 0.386563, + "f1_weighted": 0.398964 + }, + { + "accuracy": 0.416134, + "f1": 0.393917, + "f1_weighted": 0.408148 + }, + { + "accuracy": 0.386129, + "f1": 0.373343, + "f1_weighted": 0.365271 + }, + { + "accuracy": 0.401869, + "f1": 0.371625, + "f1_weighted": 0.393999 + } + ], + "main_score": 0.404033, + "hf_subset": "da", + "languages": [ + "dan-Latn" + ] + }, + { + "accuracy": 0.318938, + "f1": 0.297325, + "f1_weighted": 0.306406, + "scores_per_experiment": [ + { + "accuracy": 0.322184, + "f1": 0.303026, + "f1_weighted": 0.305503 + }, + { + "accuracy": 0.303492, + "f1": 0.289255, + "f1_weighted": 0.285829 + }, + { + "accuracy": 0.310871, + "f1": 0.290511, + "f1_weighted": 0.292068 + }, + { + "accuracy": 0.334973, + "f1": 0.31491, + "f1_weighted": 0.330961 + }, + { + "accuracy": 0.3394, + "f1": 0.313569, + "f1_weighted": 0.343819 + }, + { + "accuracy": 0.303492, + "f1": 0.278936, + "f1_weighted": 0.295088 + }, + { + "accuracy": 0.325627, + "f1": 0.303382, + "f1_weighted": 0.309913 + }, + { + "accuracy": 0.30546, + "f1": 0.280335, + "f1_weighted": 0.287129 + }, + { + "accuracy": 0.307919, + "f1": 0.289406, + "f1_weighted": 0.284889 + }, + { + "accuracy": 0.335957, + "f1": 0.309921, + "f1_weighted": 0.328865 + } + ], + "main_score": 0.318938, + "hf_subset": "is", + "languages": [ + "isl-Latn" + ] + }, + { + "accuracy": 0.086129, + "f1": 0.051784, + "f1_weighted": 0.065402, + "scores_per_experiment": [ + { + "accuracy": 0.087063, + "f1": 0.064085, + "f1_weighted": 0.074098 + }, + { + "accuracy": 0.089031, + "f1": 0.051917, + "f1_weighted": 0.059715 + }, + { + "accuracy": 0.073783, + "f1": 0.053858, + "f1_weighted": 0.064491 + }, + { + "accuracy": 0.10182, + "f1": 0.053675, + "f1_weighted": 0.068987 + }, + { + "accuracy": 0.080669, + "f1": 0.051799, + "f1_weighted": 0.069712 + }, + { + "accuracy": 0.084112, + "f1": 0.054684, + "f1_weighted": 0.07359 + }, + { + "accuracy": 0.072307, + "f1": 0.038963, + "f1_weighted": 0.056205 + }, + { + "accuracy": 0.077718, + "f1": 0.053856, + "f1_weighted": 0.060568 + }, + { + "accuracy": 0.095425, + "f1": 0.038636, + "f1_weighted": 0.053494 + }, + { + "accuracy": 0.099361, + "f1": 0.056369, + "f1_weighted": 0.07316 + } + ], + "main_score": 0.086129, + "hf_subset": "el", + "languages": [ + "ell-Grek" + ] + }, + { + "accuracy": 0.358633, + "f1": 0.326631, + "f1_weighted": 0.34353, + "scores_per_experiment": [ + { + "accuracy": 0.381702, + "f1": 0.336906, + "f1_weighted": 0.362743 + }, + { + "accuracy": 0.353665, + "f1": 0.316705, + "f1_weighted": 0.349312 + }, + { + "accuracy": 0.332022, + "f1": 0.327736, + "f1_weighted": 0.318184 + }, + { + "accuracy": 0.372356, + "f1": 0.332222, + "f1_weighted": 0.346957 + }, + { + "accuracy": 0.383669, + "f1": 0.332801, + "f1_weighted": 0.373075 + }, + { + "accuracy": 0.347762, + "f1": 0.325151, + "f1_weighted": 0.334344 + }, + { + "accuracy": 0.36547, + "f1": 0.329327, + "f1_weighted": 0.353394 + }, + { + "accuracy": 0.343827, + "f1": 0.305716, + "f1_weighted": 0.33058 + }, + { + "accuracy": 0.349238, + "f1": 0.31964, + "f1_weighted": 0.32251 + }, + { + "accuracy": 0.356616, + "f1": 0.340106, + "f1_weighted": 0.344198 + } + ], + "main_score": 0.358633, + "hf_subset": "sl", + "languages": [ + "slv-Latn" + ] + }, + { + "accuracy": 0.312887, + "f1": 0.289417, + "f1_weighted": 0.307867, + "scores_per_experiment": [ + { + "accuracy": 0.298574, + "f1": 0.289479, + "f1_weighted": 0.288746 + }, + { + "accuracy": 0.316773, + "f1": 0.283978, + "f1_weighted": 0.311086 + }, + { + "accuracy": 0.283817, + "f1": 0.271028, + "f1_weighted": 0.271468 + }, + { + "accuracy": 0.345303, + "f1": 0.313412, + "f1_weighted": 0.344682 + }, + { + "accuracy": 0.334973, + "f1": 0.295295, + "f1_weighted": 0.339994 + }, + { + "accuracy": 0.338908, + "f1": 0.303546, + "f1_weighted": 0.33526 + }, + { + "accuracy": 0.301525, + "f1": 0.283292, + "f1_weighted": 0.299931 + }, + { + "accuracy": 0.28726, + "f1": 0.268878, + "f1_weighted": 0.278022 + }, + { + "accuracy": 0.296114, + "f1": 0.284042, + "f1_weighted": 0.27536 + }, + { + "accuracy": 0.325627, + "f1": 0.301217, + "f1_weighted": 0.334124 + } + ], + "main_score": 0.312887, + "hf_subset": "az", + "languages": [ + "aze-Latn" + ] + }, + { + "accuracy": 0.357108, + "f1": 0.332966, + "f1_weighted": 0.341692, + "scores_per_experiment": [ + { + "accuracy": 0.369405, + "f1": 0.34811, + "f1_weighted": 0.349826 + }, + { + "accuracy": 0.337924, + "f1": 0.337285, + "f1_weighted": 0.333665 + }, + { + "accuracy": 0.326119, + "f1": 0.32229, + "f1_weighted": 0.307155 + }, + { + "accuracy": 0.362518, + "f1": 0.326867, + "f1_weighted": 0.342718 + }, + { + "accuracy": 0.367929, + "f1": 0.331214, + "f1_weighted": 0.35142 + }, + { + "accuracy": 0.388096, + "f1": 0.340113, + "f1_weighted": 0.372149 + }, + { + "accuracy": 0.344319, + "f1": 0.32632, + "f1_weighted": 0.328544 + }, + { + "accuracy": 0.378259, + "f1": 0.332494, + "f1_weighted": 0.36453 + }, + { + "accuracy": 0.327595, + "f1": 0.321537, + "f1_weighted": 0.303892 + }, + { + "accuracy": 0.368913, + "f1": 0.343428, + "f1_weighted": 0.363023 + } + ], + "main_score": 0.357108, + "hf_subset": "sw", + "languages": [ + "swa-Latn" + ] + }, + { + "accuracy": 0.258731, + "f1": 0.225324, + "f1_weighted": 0.236139, + "scores_per_experiment": [ + { + "accuracy": 0.265125, + "f1": 0.249589, + "f1_weighted": 0.246175 + }, + { + "accuracy": 0.257255, + "f1": 0.228687, + "f1_weighted": 0.234857 + }, + { + "accuracy": 0.27939, + "f1": 0.24051, + "f1_weighted": 0.269354 + }, + { + "accuracy": 0.230694, + "f1": 0.195862, + "f1_weighted": 0.20259 + }, + { + "accuracy": 0.257747, + "f1": 0.214703, + "f1_weighted": 0.230735 + }, + { + "accuracy": 0.254304, + "f1": 0.224378, + "f1_weighted": 0.218768 + }, + { + "accuracy": 0.249385, + "f1": 0.225291, + "f1_weighted": 0.223924 + }, + { + "accuracy": 0.272504, + "f1": 0.227598, + "f1_weighted": 0.262168 + }, + { + "accuracy": 0.280374, + "f1": 0.235791, + "f1_weighted": 0.254579 + }, + { + "accuracy": 0.240531, + "f1": 0.210828, + "f1_weighted": 0.21824 + } + ], + "main_score": 0.258731, + "hf_subset": "vi", + "languages": [ + "vie-Latn" + ] + }, + { + "accuracy": 0.014461, + "f1": 0.00411, + "f1_weighted": 0.00675, + "scores_per_experiment": [ + { + "accuracy": 0.012297, + "f1": 0.003405, + "f1_weighted": 0.006993 + }, + { + "accuracy": 0.008854, + "f1": 0.003938, + "f1_weighted": 0.002754 + }, + { + "accuracy": 0.016724, + "f1": 0.004808, + "f1_weighted": 0.014936 + }, + { + "accuracy": 0.015248, + "f1": 0.003749, + "f1_weighted": 0.00369 + }, + { + "accuracy": 0.018692, + "f1": 0.00512, + "f1_weighted": 0.003328 + }, + { + "accuracy": 0.006394, + "f1": 0.001521, + "f1_weighted": 0.004566 + }, + { + "accuracy": 0.013281, + "f1": 0.00529, + "f1_weighted": 0.005696 + }, + { + "accuracy": 0.007378, + "f1": 0.004194, + "f1_weighted": 0.005339 + }, + { + "accuracy": 0.024594, + "f1": 0.005429, + "f1_weighted": 0.016376 + }, + { + "accuracy": 0.021151, + "f1": 0.00364, + "f1_weighted": 0.003826 + } + ], + "main_score": 0.014461, + "hf_subset": "ta", + "languages": [ + "tam-Taml" + ] + }, + { + "accuracy": 0.011264, + "f1": 0.002257, + "f1_weighted": 0.000956, + "scores_per_experiment": [ + { + "accuracy": 0.026562, + "f1": 0.001881, + "f1_weighted": 0.001555 + }, + { + "accuracy": 0.004919, + "f1": 0.004508, + "f1_weighted": 0.000877 + }, + { + "accuracy": 0.014265, + "f1": 0.006089, + "f1_weighted": 0.002832 + }, + { + "accuracy": 0.001968, + "f1": 6.8e-05, + "f1_weighted": 1e-05 + }, + { + "accuracy": 0.012789, + "f1": 0.001517, + "f1_weighted": 0.000466 + }, + { + "accuracy": 0.004427, + "f1": 0.002255, + "f1_weighted": 0.000339 + }, + { + "accuracy": 0.008362, + "f1": 0.000837, + "f1_weighted": 0.000216 + }, + { + "accuracy": 0.009346, + "f1": 0.002831, + "f1_weighted": 0.002057 + }, + { + "accuracy": 0.020167, + "f1": 0.001237, + "f1_weighted": 0.000881 + }, + { + "accuracy": 0.009838, + "f1": 0.00135, + "f1_weighted": 0.000327 + } + ], + "main_score": 0.011264, + "hf_subset": "zh-CN", + "languages": [ + "cmo-Hans" + ] + }, + { + "accuracy": 0.400541, + "f1": 0.362332, + "f1_weighted": 0.390112, + "scores_per_experiment": [ + { + "accuracy": 0.397442, + "f1": 0.360391, + "f1_weighted": 0.393361 + }, + { + "accuracy": 0.401377, + "f1": 0.363253, + "f1_weighted": 0.394066 + }, + { + "accuracy": 0.412199, + "f1": 0.361566, + "f1_weighted": 0.39158 + }, + { + "accuracy": 0.40728, + "f1": 0.363489, + "f1_weighted": 0.390997 + }, + { + "accuracy": 0.416626, + "f1": 0.36844, + "f1_weighted": 0.403686 + }, + { + "accuracy": 0.412691, + "f1": 0.382119, + "f1_weighted": 0.403297 + }, + { + "accuracy": 0.38121, + "f1": 0.35899, + "f1_weighted": 0.377649 + }, + { + "accuracy": 0.415642, + "f1": 0.363797, + "f1_weighted": 0.413167 + }, + { + "accuracy": 0.374324, + "f1": 0.340272, + "f1_weighted": 0.359103 + }, + { + "accuracy": 0.386621, + "f1": 0.360999, + "f1_weighted": 0.374213 + } + ], + "main_score": 0.400541, + "hf_subset": "sq", + "languages": [ + "sqi-Latn" + ] + }, + { + "accuracy": 0.021643, + "f1": 0.005515, + "f1_weighted": 0.007316, + "scores_per_experiment": [ + { + "accuracy": 0.007378, + "f1": 0.001065, + "f1_weighted": 0.000521 + }, + { + "accuracy": 0.008362, + "f1": 0.006185, + "f1_weighted": 0.004577 + }, + { + "accuracy": 0.006886, + "f1": 0.004403, + "f1_weighted": 0.002937 + }, + { + "accuracy": 0.021151, + "f1": 0.007387, + "f1_weighted": 0.010988 + }, + { + "accuracy": 0.054107, + "f1": 0.004393, + "f1_weighted": 0.012695 + }, + { + "accuracy": 0.01033, + "f1": 0.006406, + "f1_weighted": 0.006246 + }, + { + "accuracy": 0.008854, + "f1": 0.000877, + "f1_weighted": 0.000346 + }, + { + "accuracy": 0.008854, + "f1": 0.005497, + "f1_weighted": 0.003509 + }, + { + "accuracy": 0.06788, + "f1": 0.010654, + "f1_weighted": 0.023018 + }, + { + "accuracy": 0.022627, + "f1": 0.008283, + "f1_weighted": 0.008325 + } + ], + "main_score": 0.021643, + "hf_subset": "he", + "languages": [ + "heb-Hebr" + ] + }, + { + "accuracy": 0.030103, + "f1": 0.014043, + "f1_weighted": 0.01471, + "scores_per_experiment": [ + { + "accuracy": 0.074274, + "f1": 0.024934, + "f1_weighted": 0.02603 + }, + { + "accuracy": 0.056567, + "f1": 0.008031, + "f1_weighted": 0.010501 + }, + { + "accuracy": 0.015248, + "f1": 0.013303, + "f1_weighted": 0.01358 + }, + { + "accuracy": 0.014265, + "f1": 0.015179, + "f1_weighted": 0.01685 + }, + { + "accuracy": 0.063945, + "f1": 0.0085, + "f1_weighted": 0.015104 + }, + { + "accuracy": 0.025086, + "f1": 0.020337, + "f1_weighted": 0.019289 + }, + { + "accuracy": 0.009838, + "f1": 0.007469, + "f1_weighted": 0.008536 + }, + { + "accuracy": 0.019183, + "f1": 0.016696, + "f1_weighted": 0.014788 + }, + { + "accuracy": 0.01574, + "f1": 0.016562, + "f1_weighted": 0.015861 + }, + { + "accuracy": 0.006886, + "f1": 0.009416, + "f1_weighted": 0.006561 + } + ], + "main_score": 0.030103, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ] + }, + { + "accuracy": 0.023217, + "f1": 0.008707, + "f1_weighted": 0.014061, + "scores_per_experiment": [ + { + "accuracy": 0.026562, + "f1": 0.008703, + "f1_weighted": 0.024356 + }, + { + "accuracy": 0.01574, + "f1": 0.012241, + "f1_weighted": 0.014067 + }, + { + "accuracy": 0.017708, + "f1": 0.006636, + "f1_weighted": 0.019658 + }, + { + "accuracy": 0.012297, + "f1": 0.007729, + "f1_weighted": 0.010213 + }, + { + "accuracy": 0.020659, + "f1": 0.011524, + "f1_weighted": 0.008783 + }, + { + "accuracy": 0.023119, + "f1": 0.01156, + "f1_weighted": 0.01501 + }, + { + "accuracy": 0.008854, + "f1": 0.004822, + "f1_weighted": 0.00647 + }, + { + "accuracy": 0.009838, + "f1": 0.004949, + "f1_weighted": 0.004995 + }, + { + "accuracy": 0.061485, + "f1": 0.010374, + "f1_weighted": 0.021431 + }, + { + "accuracy": 0.035908, + "f1": 0.008531, + "f1_weighted": 0.015626 + } + ], + "main_score": 0.023217, + "hf_subset": "te", + "languages": [ + "tel-Telu" + ] + }, + { + "accuracy": 0.026709, + "f1": 0.013969, + "f1_weighted": 0.014669, + "scores_per_experiment": [ + { + "accuracy": 0.018692, + "f1": 0.012689, + "f1_weighted": 0.012871 + }, + { + "accuracy": 0.021643, + "f1": 0.023068, + "f1_weighted": 0.018808 + }, + { + "accuracy": 0.028037, + "f1": 0.025336, + "f1_weighted": 0.023963 + }, + { + "accuracy": 0.0182, + "f1": 0.016522, + "f1_weighted": 0.011279 + }, + { + "accuracy": 0.025578, + "f1": 0.012631, + "f1_weighted": 0.00909 + }, + { + "accuracy": 0.031972, + "f1": 0.008968, + "f1_weighted": 0.01637 + }, + { + "accuracy": 0.020167, + "f1": 0.011273, + "f1_weighted": 0.013003 + }, + { + "accuracy": 0.01574, + "f1": 0.007381, + "f1_weighted": 0.009625 + }, + { + "accuracy": 0.061977, + "f1": 0.015023, + "f1_weighted": 0.022829 + }, + { + "accuracy": 0.025086, + "f1": 0.0068, + "f1_weighted": 0.008849 + } + ], + "main_score": 0.026709, + "hf_subset": "hi", + "languages": [ + "hin-Deva" + ] + }, + { + "accuracy": 0.028136, + "f1": 0.017296, + "f1_weighted": 0.017829, + "scores_per_experiment": [ + { + "accuracy": 0.045253, + "f1": 0.02217, + "f1_weighted": 0.034513 + }, + { + "accuracy": 0.015248, + "f1": 0.016696, + "f1_weighted": 0.008779 + }, + { + "accuracy": 0.028037, + "f1": 0.017501, + "f1_weighted": 0.023531 + }, + { + "accuracy": 0.023119, + "f1": 0.017023, + "f1_weighted": 0.017687 + }, + { + "accuracy": 0.021643, + "f1": 0.012217, + "f1_weighted": 0.011323 + }, + { + "accuracy": 0.019675, + "f1": 0.015742, + "f1_weighted": 0.013295 + }, + { + "accuracy": 0.014265, + "f1": 0.010975, + "f1_weighted": 0.006639 + }, + { + "accuracy": 0.021643, + "f1": 0.027401, + "f1_weighted": 0.027067 + }, + { + "accuracy": 0.070831, + "f1": 0.024275, + "f1_weighted": 0.027977 + }, + { + "accuracy": 0.021643, + "f1": 0.008964, + "f1_weighted": 0.007478 + } + ], + "main_score": 0.028136, + "hf_subset": "ur", + "languages": [ + "urd-Arab" + ] + }, + { + "accuracy": 0.467634, + "f1": 0.448577, + "f1_weighted": 0.449585, + "scores_per_experiment": [ + { + "accuracy": 0.474176, + "f1": 0.463202, + "f1_weighted": 0.459916 + }, + { + "accuracy": 0.478111, + "f1": 0.460854, + "f1_weighted": 0.468506 + }, + { + "accuracy": 0.442204, + "f1": 0.428146, + "f1_weighted": 0.415742 + }, + { + "accuracy": 0.461879, + "f1": 0.428439, + "f1_weighted": 0.447919 + }, + { + "accuracy": 0.496803, + "f1": 0.452681, + "f1_weighted": 0.485841 + }, + { + "accuracy": 0.479587, + "f1": 0.469698, + "f1_weighted": 0.464324 + }, + { + "accuracy": 0.458436, + "f1": 0.454586, + "f1_weighted": 0.436229 + }, + { + "accuracy": 0.461879, + "f1": 0.440243, + "f1_weighted": 0.446688 + }, + { + "accuracy": 0.460403, + "f1": 0.455907, + "f1_weighted": 0.436496 + }, + { + "accuracy": 0.462863, + "f1": 0.432012, + "f1_weighted": 0.434194 + } + ], + "main_score": 0.467634, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ] + }, + { + "accuracy": 0.388539, + "f1": 0.367346, + "f1_weighted": 0.377656, + "scores_per_experiment": [ + { + "accuracy": 0.413674, + "f1": 0.394721, + "f1_weighted": 0.394925 + }, + { + "accuracy": 0.393015, + "f1": 0.370544, + "f1_weighted": 0.388916 + }, + { + "accuracy": 0.374816, + "f1": 0.360356, + "f1_weighted": 0.366976 + }, + { + "accuracy": 0.393999, + "f1": 0.362613, + "f1_weighted": 0.388964 + }, + { + "accuracy": 0.385145, + "f1": 0.349182, + "f1_weighted": 0.374632 + }, + { + "accuracy": 0.406788, + "f1": 0.377741, + "f1_weighted": 0.392331 + }, + { + "accuracy": 0.376291, + "f1": 0.360715, + "f1_weighted": 0.358867 + }, + { + "accuracy": 0.405804, + "f1": 0.366095, + "f1_weighted": 0.395609 + }, + { + "accuracy": 0.37088, + "f1": 0.375139, + "f1_weighted": 0.356778 + }, + { + "accuracy": 0.364978, + "f1": 0.356353, + "f1_weighted": 0.358566 + } + ], + "main_score": 0.388539, + "hf_subset": "id", + "languages": [ + "ind-Latn" + ] + }, + { + "accuracy": 0.041121, + "f1": 0.016712, + "f1_weighted": 0.015319, + "scores_per_experiment": [ + { + "accuracy": 0.058042, + "f1": 0.022077, + "f1_weighted": 0.018726 + }, + { + "accuracy": 0.055091, + "f1": 0.013241, + "f1_weighted": 0.020158 + }, + { + "accuracy": 0.032956, + "f1": 0.014603, + "f1_weighted": 0.011204 + }, + { + "accuracy": 0.034924, + "f1": 0.022747, + "f1_weighted": 0.016693 + }, + { + "accuracy": 0.054107, + "f1": 0.021242, + "f1_weighted": 0.018782 + }, + { + "accuracy": 0.053123, + "f1": 0.02471, + "f1_weighted": 0.02864 + }, + { + "accuracy": 0.037383, + "f1": 0.01391, + "f1_weighted": 0.007959 + }, + { + "accuracy": 0.030989, + "f1": 0.01312, + "f1_weighted": 0.009637 + }, + { + "accuracy": 0.016232, + "f1": 0.011016, + "f1_weighted": 0.005942 + }, + { + "accuracy": 0.038367, + "f1": 0.010455, + "f1_weighted": 0.015449 + } + ], + "main_score": 0.041121, + "hf_subset": "th", + "languages": [ + "tha-Thai" + ] + }, + { + "accuracy": 0.345155, + "f1": 0.306621, + "f1_weighted": 0.327131, + "scores_per_experiment": [ + { + "accuracy": 0.350713, + "f1": 0.312923, + "f1_weighted": 0.330327 + }, + { + "accuracy": 0.345303, + "f1": 0.297833, + "f1_weighted": 0.331608 + }, + { + "accuracy": 0.336449, + "f1": 0.318114, + "f1_weighted": 0.314497 + }, + { + "accuracy": 0.341367, + "f1": 0.302912, + "f1_weighted": 0.315515 + }, + { + "accuracy": 0.335465, + "f1": 0.290752, + "f1_weighted": 0.314101 + }, + { + "accuracy": 0.334481, + "f1": 0.300535, + "f1_weighted": 0.320384 + }, + { + "accuracy": 0.367929, + "f1": 0.319702, + "f1_weighted": 0.353944 + }, + { + "accuracy": 0.330054, + "f1": 0.289191, + "f1_weighted": 0.311388 + }, + { + "accuracy": 0.351205, + "f1": 0.314609, + "f1_weighted": 0.326467 + }, + { + "accuracy": 0.358583, + "f1": 0.319641, + "f1_weighted": 0.353078 + } + ], + "main_score": 0.345155, + "hf_subset": "fi", + "languages": [ + "fin-Latn" + ] + }, + { + "accuracy": 0.353173, + "f1": 0.328696, + "f1_weighted": 0.336571, + "scores_per_experiment": [ + { + "accuracy": 0.348746, + "f1": 0.326759, + "f1_weighted": 0.329594 + }, + { + "accuracy": 0.34727, + "f1": 0.326897, + "f1_weighted": 0.328993 + }, + { + "accuracy": 0.344319, + "f1": 0.333343, + "f1_weighted": 0.325853 + }, + { + "accuracy": 0.359075, + "f1": 0.334128, + "f1_weighted": 0.346572 + }, + { + "accuracy": 0.362518, + "f1": 0.313419, + "f1_weighted": 0.343847 + }, + { + "accuracy": 0.371372, + "f1": 0.357241, + "f1_weighted": 0.367078 + }, + { + "accuracy": 0.353665, + "f1": 0.339211, + "f1_weighted": 0.336909 + }, + { + "accuracy": 0.357108, + "f1": 0.307093, + "f1_weighted": 0.335177 + }, + { + "accuracy": 0.317265, + "f1": 0.305526, + "f1_weighted": 0.290889 + }, + { + "accuracy": 0.370389, + "f1": 0.343343, + "f1_weighted": 0.360797 + } + ], + "main_score": 0.353173, + "hf_subset": "ms", + "languages": [ + "msa-Latn" + ] + }, + { + "accuracy": 0.049041, + "f1": 0.011467, + "f1_weighted": 0.013988, + "scores_per_experiment": [ + { + "accuracy": 0.042794, + "f1": 0.011728, + "f1_weighted": 0.010695 + }, + { + "accuracy": 0.033448, + "f1": 0.004598, + "f1_weighted": 0.004448 + }, + { + "accuracy": 0.051648, + "f1": 0.010436, + "f1_weighted": 0.011428 + }, + { + "accuracy": 0.04427, + "f1": 0.012032, + "f1_weighted": 0.014377 + }, + { + "accuracy": 0.058534, + "f1": 0.016737, + "f1_weighted": 0.023813 + }, + { + "accuracy": 0.030497, + "f1": 0.01131, + "f1_weighted": 0.009441 + }, + { + "accuracy": 0.039351, + "f1": 0.010138, + "f1_weighted": 0.010014 + }, + { + "accuracy": 0.080669, + "f1": 0.011649, + "f1_weighted": 0.022601 + }, + { + "accuracy": 0.065912, + "f1": 0.014954, + "f1_weighted": 0.024185 + }, + { + "accuracy": 0.043286, + "f1": 0.011086, + "f1_weighted": 0.008875 + } + ], + "main_score": 0.049041, + "hf_subset": "km", + "languages": [ + "khm-Khmr" + ] + }, + { + "accuracy": 0.021003, + "f1": 0.008387, + "f1_weighted": 0.008966, + "scores_per_experiment": [ + { + "accuracy": 0.054107, + "f1": 0.01502, + "f1_weighted": 0.019169 + }, + { + "accuracy": 0.004427, + "f1": 0.00036, + "f1_weighted": 6e-05 + }, + { + "accuracy": 0.013773, + "f1": 0.013251, + "f1_weighted": 0.012992 + }, + { + "accuracy": 0.008854, + "f1": 0.003446, + "f1_weighted": 0.006659 + }, + { + "accuracy": 0.019183, + "f1": 0.012502, + "f1_weighted": 0.014438 + }, + { + "accuracy": 0.017708, + "f1": 0.010244, + "f1_weighted": 0.009 + }, + { + "accuracy": 0.005903, + "f1": 0.002845, + "f1_weighted": 0.001727 + }, + { + "accuracy": 0.008854, + "f1": 0.007941, + "f1_weighted": 0.006055 + }, + { + "accuracy": 0.058042, + "f1": 0.012168, + "f1_weighted": 0.01536 + }, + { + "accuracy": 0.019183, + "f1": 0.006095, + "f1_weighted": 0.004202 + } + ], + "main_score": 0.021003, + "hf_subset": "am", + "languages": [ + "amh-Ethi" + ] + } + ], + "test": [ + { + "accuracy": 0.351917, + "f1": 0.325894, + "f1_weighted": 0.332519, + "scores_per_experiment": [ + { + "accuracy": 0.359112, + "f1": 0.332917, + "f1_weighted": 0.332907 + }, + { + "accuracy": 0.345999, + "f1": 0.32688, + "f1_weighted": 0.330699 + }, + { + "accuracy": 0.356086, + "f1": 0.330494, + "f1_weighted": 0.338575 + }, + { + "accuracy": 0.341627, + "f1": 0.317164, + "f1_weighted": 0.315135 + }, + { + "accuracy": 0.372562, + "f1": 0.325386, + "f1_weighted": 0.352036 + }, + { + "accuracy": 0.339274, + "f1": 0.317353, + "f1_weighted": 0.325086 + }, + { + "accuracy": 0.350706, + "f1": 0.327902, + "f1_weighted": 0.335078 + }, + { + "accuracy": 0.355414, + "f1": 0.319474, + "f1_weighted": 0.33928 + }, + { + "accuracy": 0.35037, + "f1": 0.32828, + "f1_weighted": 0.325014 + }, + { + "accuracy": 0.348016, + "f1": 0.333086, + "f1_weighted": 0.331378 + } + ], + "main_score": 0.351917, + "hf_subset": "tr", + "languages": [ + "tur-Latn" + ] + }, + { + "accuracy": 0.026866, + "f1": 0.009948, + "f1_weighted": 0.010129, + "scores_per_experiment": [ + { + "accuracy": 0.016812, + "f1": 0.008763, + "f1_weighted": 0.005016 + }, + { + "accuracy": 0.030599, + "f1": 0.012003, + "f1_weighted": 0.016082 + }, + { + "accuracy": 0.02959, + "f1": 0.007985, + "f1_weighted": 0.01116 + }, + { + "accuracy": 0.011432, + "f1": 0.003001, + "f1_weighted": 0.00126 + }, + { + "accuracy": 0.031607, + "f1": 0.01132, + "f1_weighted": 0.011481 + }, + { + "accuracy": 0.029926, + "f1": 0.011286, + "f1_weighted": 0.009905 + }, + { + "accuracy": 0.032616, + "f1": 0.013817, + "f1_weighted": 0.015172 + }, + { + "accuracy": 0.022865, + "f1": 0.01252, + "f1_weighted": 0.009521 + }, + { + "accuracy": 0.025219, + "f1": 0.008938, + "f1_weighted": 0.009546 + }, + { + "accuracy": 0.037996, + "f1": 0.009846, + "f1_weighted": 0.012149 + } + ], + "main_score": 0.026866, + "hf_subset": "ka", + "languages": [ + "kat-Geor" + ] + }, + { + "accuracy": 0.028278, + "f1": 0.006204, + "f1_weighted": 0.010983, + "scores_per_experiment": [ + { + "accuracy": 0.0538, + "f1": 0.004082, + "f1_weighted": 0.013772 + }, + { + "accuracy": 0.024546, + "f1": 0.006634, + "f1_weighted": 0.01127 + }, + { + "accuracy": 0.015467, + "f1": 0.005912, + "f1_weighted": 0.007676 + }, + { + "accuracy": 0.018494, + "f1": 0.006622, + "f1_weighted": 0.008757 + }, + { + "accuracy": 0.025891, + "f1": 0.011556, + "f1_weighted": 0.017082 + }, + { + "accuracy": 0.015804, + "f1": 0.002503, + "f1_weighted": 0.003036 + }, + { + "accuracy": 0.017149, + "f1": 0.004646, + "f1_weighted": 0.010094 + }, + { + "accuracy": 0.012441, + "f1": 0.003812, + "f1_weighted": 0.008208 + }, + { + "accuracy": 0.071621, + "f1": 0.011481, + "f1_weighted": 0.025188 + }, + { + "accuracy": 0.027572, + "f1": 0.004797, + "f1_weighted": 0.004751 + } + ], + "main_score": 0.028278, + "hf_subset": "ko", + "languages": [ + "kor-Kore" + ] + }, + { + "accuracy": 0.449462, + "f1": 0.444992, + "f1_weighted": 0.429648, + "scores_per_experiment": [ + { + "accuracy": 0.456288, + "f1": 0.443737, + "f1_weighted": 0.434338 + }, + { + "accuracy": 0.452925, + "f1": 0.454858, + "f1_weighted": 0.440004 + }, + { + "accuracy": 0.448218, + "f1": 0.437397, + "f1_weighted": 0.42455 + }, + { + "accuracy": 0.452925, + "f1": 0.444947, + "f1_weighted": 0.435176 + }, + { + "accuracy": 0.46772, + "f1": 0.452398, + "f1_weighted": 0.452284 + }, + { + "accuracy": 0.436113, + "f1": 0.438555, + "f1_weighted": 0.405655 + }, + { + "accuracy": 0.464022, + "f1": 0.46759, + "f1_weighted": 0.447673 + }, + { + "accuracy": 0.444855, + "f1": 0.433111, + "f1_weighted": 0.419606 + }, + { + "accuracy": 0.425353, + "f1": 0.434174, + "f1_weighted": 0.402486 + }, + { + "accuracy": 0.4462, + "f1": 0.443156, + "f1_weighted": 0.434712 + } + ], + "main_score": 0.449462, + "hf_subset": "it", + "languages": [ + "ita-Latn" + ] + }, + { + "accuracy": 0.171116, + "f1": 0.156157, + "f1_weighted": 0.158394, + "scores_per_experiment": [ + { + "accuracy": 0.16577, + "f1": 0.156779, + "f1_weighted": 0.148959 + }, + { + "accuracy": 0.146604, + "f1": 0.142783, + "f1_weighted": 0.136503 + }, + { + "accuracy": 0.17115, + "f1": 0.165148, + "f1_weighted": 0.170306 + }, + { + "accuracy": 0.180565, + "f1": 0.159617, + "f1_weighted": 0.174212 + }, + { + "accuracy": 0.205783, + "f1": 0.164731, + "f1_weighted": 0.196103 + }, + { + "accuracy": 0.150303, + "f1": 0.146123, + "f1_weighted": 0.129156 + }, + { + "accuracy": 0.166779, + "f1": 0.154747, + "f1_weighted": 0.153538 + }, + { + "accuracy": 0.197041, + "f1": 0.162296, + "f1_weighted": 0.181019 + }, + { + "accuracy": 0.158036, + "f1": 0.153591, + "f1_weighted": 0.13467 + }, + { + "accuracy": 0.169132, + "f1": 0.155754, + "f1_weighted": 0.159475 + } + ], + "main_score": 0.171116, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + }, + { + "accuracy": 0.037525, + "f1": 0.008203, + "f1_weighted": 0.012489, + "scores_per_experiment": [ + { + "accuracy": 0.064223, + "f1": 0.00728, + "f1_weighted": 0.016663 + }, + { + "accuracy": 0.01076, + "f1": 0.006301, + "f1_weighted": 0.003692 + }, + { + "accuracy": 0.010424, + "f1": 0.00755, + "f1_weighted": 0.005963 + }, + { + "accuracy": 0.042703, + "f1": 0.012526, + "f1_weighted": 0.019187 + }, + { + "accuracy": 0.070612, + "f1": 0.010473, + "f1_weighted": 0.020177 + }, + { + "accuracy": 0.009751, + "f1": 0.005618, + "f1_weighted": 0.00472 + }, + { + "accuracy": 0.011432, + "f1": 0.008301, + "f1_weighted": 0.007709 + }, + { + "accuracy": 0.067586, + "f1": 0.010272, + "f1_weighted": 0.022163 + }, + { + "accuracy": 0.065568, + "f1": 0.009467, + "f1_weighted": 0.019133 + }, + { + "accuracy": 0.022192, + "f1": 0.004246, + "f1_weighted": 0.005483 + } + ], + "main_score": 0.037525, + "hf_subset": "fa", + "languages": [ + "fas-Arab" + ] + }, + { + "accuracy": 0.164694, + "f1": 0.144353, + "f1_weighted": 0.14374, + "scores_per_experiment": [ + { + "accuracy": 0.183591, + "f1": 0.153634, + "f1_weighted": 0.161032 + }, + { + "accuracy": 0.158036, + "f1": 0.133682, + "f1_weighted": 0.147338 + }, + { + "accuracy": 0.165434, + "f1": 0.149956, + "f1_weighted": 0.149152 + }, + { + "accuracy": 0.161399, + "f1": 0.138458, + "f1_weighted": 0.140792 + }, + { + "accuracy": 0.164425, + "f1": 0.143933, + "f1_weighted": 0.143723 + }, + { + "accuracy": 0.15232, + "f1": 0.134982, + "f1_weighted": 0.132927 + }, + { + "accuracy": 0.18191, + "f1": 0.162491, + "f1_weighted": 0.160544 + }, + { + "accuracy": 0.172495, + "f1": 0.150014, + "f1_weighted": 0.143414 + }, + { + "accuracy": 0.167451, + "f1": 0.151897, + "f1_weighted": 0.148057 + }, + { + "accuracy": 0.139879, + "f1": 0.124479, + "f1_weighted": 0.110419 + } + ], + "main_score": 0.164694, + "hf_subset": "mn", + "languages": [ + "mon-Cyrl" + ] + }, + { + "accuracy": 0.026059, + "f1": 0.014293, + "f1_weighted": 0.013698, + "scores_per_experiment": [ + { + "accuracy": 0.019839, + "f1": 0.01559, + "f1_weighted": 0.01289 + }, + { + "accuracy": 0.011432, + "f1": 0.004981, + "f1_weighted": 0.002148 + }, + { + "accuracy": 0.016812, + "f1": 0.009561, + "f1_weighted": 0.007384 + }, + { + "accuracy": 0.027236, + "f1": 0.021147, + "f1_weighted": 0.0221 + }, + { + "accuracy": 0.033289, + "f1": 0.012687, + "f1_weighted": 0.015588 + }, + { + "accuracy": 0.034297, + "f1": 0.014449, + "f1_weighted": 0.015526 + }, + { + "accuracy": 0.01614, + "f1": 0.005695, + "f1_weighted": 0.005486 + }, + { + "accuracy": 0.015131, + "f1": 0.018351, + "f1_weighted": 0.01493 + }, + { + "accuracy": 0.06187, + "f1": 0.02261, + "f1_weighted": 0.025438 + }, + { + "accuracy": 0.024546, + "f1": 0.017858, + "f1_weighted": 0.015494 + } + ], + "main_score": 0.026059, + "hf_subset": "kn", + "languages": [ + "kan-Knda" + ] + }, + { + "accuracy": 0.032313, + "f1": 0.00949, + "f1_weighted": 0.013142, + "scores_per_experiment": [ + { + "accuracy": 0.063215, + "f1": 0.004595, + "f1_weighted": 0.013592 + }, + { + "accuracy": 0.016812, + "f1": 0.00793, + "f1_weighted": 0.008091 + }, + { + "accuracy": 0.017821, + "f1": 0.009791, + "f1_weighted": 0.009056 + }, + { + "accuracy": 0.017821, + "f1": 0.006682, + "f1_weighted": 0.005674 + }, + { + "accuracy": 0.025555, + "f1": 0.009238, + "f1_weighted": 0.01105 + }, + { + "accuracy": 0.019502, + "f1": 0.00711, + "f1_weighted": 0.007747 + }, + { + "accuracy": 0.016812, + "f1": 0.007456, + "f1_weighted": 0.005611 + }, + { + "accuracy": 0.066577, + "f1": 0.017075, + "f1_weighted": 0.034565 + }, + { + "accuracy": 0.054472, + "f1": 0.014363, + "f1_weighted": 0.026507 + }, + { + "accuracy": 0.024546, + "f1": 0.010664, + "f1_weighted": 0.009522 + } + ], + "main_score": 0.032313, + "hf_subset": "hy", + "languages": [ + "hye-Armn" + ] + }, + { + "accuracy": 0.0461, + "f1": 0.027388, + "f1_weighted": 0.03316, + "scores_per_experiment": [ + { + "accuracy": 0.072629, + "f1": 0.044166, + "f1_weighted": 0.066574 + }, + { + "accuracy": 0.030935, + "f1": 0.016623, + "f1_weighted": 0.017097 + }, + { + "accuracy": 0.046402, + "f1": 0.034752, + "f1_weighted": 0.039465 + }, + { + "accuracy": 0.030935, + "f1": 0.019166, + "f1_weighted": 0.013957 + }, + { + "accuracy": 0.024882, + "f1": 0.013127, + "f1_weighted": 0.013792 + }, + { + "accuracy": 0.057498, + "f1": 0.039304, + "f1_weighted": 0.045198 + }, + { + "accuracy": 0.036315, + "f1": 0.026705, + "f1_weighted": 0.031441 + }, + { + "accuracy": 0.048756, + "f1": 0.028535, + "f1_weighted": 0.040617 + }, + { + "accuracy": 0.086079, + "f1": 0.031506, + "f1_weighted": 0.050549 + }, + { + "accuracy": 0.026564, + "f1": 0.019997, + "f1_weighted": 0.012913 + } + ], + "main_score": 0.0461, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ] + }, + { + "accuracy": 0.448285, + "f1": 0.418769, + "f1_weighted": 0.435154, + "scores_per_experiment": [ + { + "accuracy": 0.460323, + "f1": 0.434, + "f1_weighted": 0.457771 + }, + { + "accuracy": 0.479489, + "f1": 0.44692, + "f1_weighted": 0.474131 + }, + { + "accuracy": 0.435777, + "f1": 0.411586, + "f1_weighted": 0.423091 + }, + { + "accuracy": 0.46234, + "f1": 0.425099, + "f1_weighted": 0.457586 + }, + { + "accuracy": 0.470074, + "f1": 0.419563, + "f1_weighted": 0.455326 + }, + { + "accuracy": 0.421991, + "f1": 0.404526, + "f1_weighted": 0.408266 + }, + { + "accuracy": 0.445864, + "f1": 0.418489, + "f1_weighted": 0.437604 + }, + { + "accuracy": 0.460995, + "f1": 0.431181, + "f1_weighted": 0.441339 + }, + { + "accuracy": 0.431406, + "f1": 0.405933, + "f1_weighted": 0.397911 + }, + { + "accuracy": 0.414593, + "f1": 0.390392, + "f1_weighted": 0.398515 + } + ], + "main_score": 0.448285, + "hf_subset": "ro", + "languages": [ + "ron-Latn" + ] + }, + { + "accuracy": 0.391863, + "f1": 0.370802, + "f1_weighted": 0.377744, + "scores_per_experiment": [ + { + "accuracy": 0.3961, + "f1": 0.378624, + "f1_weighted": 0.386072 + }, + { + "accuracy": 0.378615, + "f1": 0.362894, + "f1_weighted": 0.370038 + }, + { + "accuracy": 0.393746, + "f1": 0.37383, + "f1_weighted": 0.380239 + }, + { + "accuracy": 0.396772, + "f1": 0.369708, + "f1_weighted": 0.379651 + }, + { + "accuracy": 0.405178, + "f1": 0.381962, + "f1_weighted": 0.389871 + }, + { + "accuracy": 0.396772, + "f1": 0.379556, + "f1_weighted": 0.383944 + }, + { + "accuracy": 0.378951, + "f1": 0.354131, + "f1_weighted": 0.366222 + }, + { + "accuracy": 0.408204, + "f1": 0.388731, + "f1_weighted": 0.39591 + }, + { + "accuracy": 0.365165, + "f1": 0.348355, + "f1_weighted": 0.34289 + }, + { + "accuracy": 0.399126, + "f1": 0.370228, + "f1_weighted": 0.382601 + } + ], + "main_score": 0.391863, + "hf_subset": "nb", + "languages": [ + "nob-Latn" + ] + }, + { + "accuracy": 0.370343, + "f1": 0.352054, + "f1_weighted": 0.356317, + "scores_per_experiment": [ + { + "accuracy": 0.378278, + "f1": 0.358999, + "f1_weighted": 0.358885 + }, + { + "accuracy": 0.350706, + "f1": 0.3423, + "f1_weighted": 0.342705 + }, + { + "accuracy": 0.352724, + "f1": 0.340254, + "f1_weighted": 0.331737 + }, + { + "accuracy": 0.380296, + "f1": 0.341239, + "f1_weighted": 0.361294 + }, + { + "accuracy": 0.382986, + "f1": 0.35724, + "f1_weighted": 0.372599 + }, + { + "accuracy": 0.368527, + "f1": 0.364979, + "f1_weighted": 0.361295 + }, + { + "accuracy": 0.372898, + "f1": 0.344112, + "f1_weighted": 0.362619 + }, + { + "accuracy": 0.380968, + "f1": 0.366991, + "f1_weighted": 0.363725 + }, + { + "accuracy": 0.355414, + "f1": 0.344465, + "f1_weighted": 0.329451 + }, + { + "accuracy": 0.380632, + "f1": 0.359961, + "f1_weighted": 0.378859 + } + ], + "main_score": 0.370343, + "hf_subset": "sv", + "languages": [ + "swe-Latn" + ] + }, + { + "accuracy": 0.361634, + "f1": 0.338098, + "f1_weighted": 0.356112, + "scores_per_experiment": [ + { + "accuracy": 0.35306, + "f1": 0.344904, + "f1_weighted": 0.346774 + }, + { + "accuracy": 0.370208, + "f1": 0.35661, + "f1_weighted": 0.3642 + }, + { + "accuracy": 0.360794, + "f1": 0.336728, + "f1_weighted": 0.35903 + }, + { + "accuracy": 0.37458, + "f1": 0.345399, + "f1_weighted": 0.373031 + }, + { + "accuracy": 0.366846, + "f1": 0.322372, + "f1_weighted": 0.360279 + }, + { + "accuracy": 0.359785, + "f1": 0.325546, + "f1_weighted": 0.360337 + }, + { + "accuracy": 0.351379, + "f1": 0.325632, + "f1_weighted": 0.342472 + }, + { + "accuracy": 0.355414, + "f1": 0.341843, + "f1_weighted": 0.346764 + }, + { + "accuracy": 0.362475, + "f1": 0.342192, + "f1_weighted": 0.34939 + }, + { + "accuracy": 0.361802, + "f1": 0.339754, + "f1_weighted": 0.358843 + } + ], + "main_score": 0.361634, + "hf_subset": "pl", + "languages": [ + "pol-Latn" + ] + }, + { + "accuracy": 0.031103, + "f1": 0.021005, + "f1_weighted": 0.026847, + "scores_per_experiment": [ + { + "accuracy": 0.036315, + "f1": 0.032427, + "f1_weighted": 0.0315 + }, + { + "accuracy": 0.031944, + "f1": 0.01194, + "f1_weighted": 0.020797 + }, + { + "accuracy": 0.029254, + "f1": 0.023004, + "f1_weighted": 0.031623 + }, + { + "accuracy": 0.039677, + "f1": 0.022481, + "f1_weighted": 0.033437 + }, + { + "accuracy": 0.036651, + "f1": 0.024294, + "f1_weighted": 0.027663 + }, + { + "accuracy": 0.033289, + "f1": 0.024764, + "f1_weighted": 0.034063 + }, + { + "accuracy": 0.0269, + "f1": 0.015327, + "f1_weighted": 0.02901 + }, + { + "accuracy": 0.012777, + "f1": 0.013445, + "f1_weighted": 0.005254 + }, + { + "accuracy": 0.035306, + "f1": 0.017227, + "f1_weighted": 0.024914 + }, + { + "accuracy": 0.028917, + "f1": 0.025145, + "f1_weighted": 0.030209 + } + ], + "main_score": 0.031103, + "hf_subset": "zh-TW", + "languages": [ + "cmo-Hant" + ] + }, + { + "accuracy": 0.377942, + "f1": 0.352819, + "f1_weighted": 0.353735, + "scores_per_experiment": [ + { + "accuracy": 0.367182, + "f1": 0.35381, + "f1_weighted": 0.346707 + }, + { + "accuracy": 0.399462, + "f1": 0.377272, + "f1_weighted": 0.380847 + }, + { + "accuracy": 0.406187, + "f1": 0.376232, + "f1_weighted": 0.389618 + }, + { + "accuracy": 0.385003, + "f1": 0.346036, + "f1_weighted": 0.356338 + }, + { + "accuracy": 0.37727, + "f1": 0.328095, + "f1_weighted": 0.355215 + }, + { + "accuracy": 0.373571, + "f1": 0.354009, + "f1_weighted": 0.344763 + }, + { + "accuracy": 0.400134, + "f1": 0.370849, + "f1_weighted": 0.373533 + }, + { + "accuracy": 0.38803, + "f1": 0.359864, + "f1_weighted": 0.361454 + }, + { + "accuracy": 0.323134, + "f1": 0.317892, + "f1_weighted": 0.296146 + }, + { + "accuracy": 0.359449, + "f1": 0.344132, + "f1_weighted": 0.332726 + } + ], + "main_score": 0.377942, + "hf_subset": "tl", + "languages": [ + "tgl-Latn" + ] + }, + { + "accuracy": 0.030397, + "f1": 0.006938, + "f1_weighted": 0.012131, + "scores_per_experiment": [ + { + "accuracy": 0.062878, + "f1": 0.00777, + "f1_weighted": 0.015891 + }, + { + "accuracy": 0.008406, + "f1": 0.003799, + "f1_weighted": 0.001926 + }, + { + "accuracy": 0.016476, + "f1": 0.005704, + "f1_weighted": 0.007674 + }, + { + "accuracy": 0.022865, + "f1": 0.007772, + "f1_weighted": 0.013103 + }, + { + "accuracy": 0.022529, + "f1": 0.005969, + "f1_weighted": 0.011768 + }, + { + "accuracy": 0.028917, + "f1": 0.009197, + "f1_weighted": 0.012717 + }, + { + "accuracy": 0.019502, + "f1": 0.006678, + "f1_weighted": 0.013566 + }, + { + "accuracy": 0.012441, + "f1": 0.007439, + "f1_weighted": 0.010603 + }, + { + "accuracy": 0.080027, + "f1": 0.010759, + "f1_weighted": 0.029017 + }, + { + "accuracy": 0.029926, + "f1": 0.004288, + "f1_weighted": 0.005042 + } + ], + "main_score": 0.030397, + "hf_subset": "ml", + "languages": [ + "mal-Mlym" + ] + }, + { + "accuracy": 0.441493, + "f1": 0.419658, + "f1_weighted": 0.429956, + "scores_per_experiment": [ + { + "accuracy": 0.427034, + "f1": 0.41304, + "f1_weighted": 0.408513 + }, + { + "accuracy": 0.455279, + "f1": 0.414781, + "f1_weighted": 0.452132 + }, + { + "accuracy": 0.436785, + "f1": 0.416651, + "f1_weighted": 0.422242 + }, + { + "accuracy": 0.466375, + "f1": 0.434361, + "f1_weighted": 0.461196 + }, + { + "accuracy": 0.474781, + "f1": 0.436668, + "f1_weighted": 0.463675 + }, + { + "accuracy": 0.433423, + "f1": 0.418991, + "f1_weighted": 0.417179 + }, + { + "accuracy": 0.416947, + "f1": 0.404938, + "f1_weighted": 0.405628 + }, + { + "accuracy": 0.43813, + "f1": 0.405949, + "f1_weighted": 0.421847 + }, + { + "accuracy": 0.425017, + "f1": 0.424688, + "f1_weighted": 0.409909 + }, + { + "accuracy": 0.441157, + "f1": 0.426511, + "f1_weighted": 0.437239 + } + ], + "main_score": 0.441493, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ] + }, + { + "accuracy": 0.671587, + "f1": 0.646134, + "f1_weighted": 0.658315, + "scores_per_experiment": [ + { + "accuracy": 0.681237, + "f1": 0.648435, + "f1_weighted": 0.668358 + }, + { + "accuracy": 0.694687, + "f1": 0.674848, + "f1_weighted": 0.688771 + }, + { + "accuracy": 0.66577, + "f1": 0.638846, + "f1_weighted": 0.650845 + }, + { + "accuracy": 0.688299, + "f1": 0.653998, + "f1_weighted": 0.680187 + }, + { + "accuracy": 0.662744, + "f1": 0.625388, + "f1_weighted": 0.642611 + }, + { + "accuracy": 0.648958, + "f1": 0.637008, + "f1_weighted": 0.633678 + }, + { + "accuracy": 0.663416, + "f1": 0.64599, + "f1_weighted": 0.649092 + }, + { + "accuracy": 0.664425, + "f1": 0.639689, + "f1_weighted": 0.654751 + }, + { + "accuracy": 0.6577, + "f1": 0.637076, + "f1_weighted": 0.636141 + }, + { + "accuracy": 0.688635, + "f1": 0.660064, + "f1_weighted": 0.678712 + } + ], + "main_score": 0.671587, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.356355, + "f1": 0.332249, + "f1_weighted": 0.345939, + "scores_per_experiment": [ + { + "accuracy": 0.386012, + "f1": 0.364943, + "f1_weighted": 0.373996 + }, + { + "accuracy": 0.351715, + "f1": 0.330994, + "f1_weighted": 0.347545 + }, + { + "accuracy": 0.342972, + "f1": 0.313057, + "f1_weighted": 0.334077 + }, + { + "accuracy": 0.366174, + "f1": 0.344661, + "f1_weighted": 0.355962 + }, + { + "accuracy": 0.368863, + "f1": 0.330113, + "f1_weighted": 0.354843 + }, + { + "accuracy": 0.33154, + "f1": 0.324125, + "f1_weighted": 0.326533 + }, + { + "accuracy": 0.354405, + "f1": 0.334496, + "f1_weighted": 0.339506 + }, + { + "accuracy": 0.359449, + "f1": 0.327312, + "f1_weighted": 0.343607 + }, + { + "accuracy": 0.355077, + "f1": 0.3339, + "f1_weighted": 0.344388 + }, + { + "accuracy": 0.347344, + "f1": 0.318892, + "f1_weighted": 0.338929 + } + ], + "main_score": 0.356355, + "hf_subset": "cy", + "languages": [ + "cym-Latn" + ] + }, + { + "accuracy": 0.462105, + "f1": 0.426033, + "f1_weighted": 0.45157, + "scores_per_experiment": [ + { + "accuracy": 0.463349, + "f1": 0.426767, + "f1_weighted": 0.462064 + }, + { + "accuracy": 0.488904, + "f1": 0.443229, + "f1_weighted": 0.483735 + }, + { + "accuracy": 0.472764, + "f1": 0.429143, + "f1_weighted": 0.455665 + }, + { + "accuracy": 0.4731, + "f1": 0.430254, + "f1_weighted": 0.451315 + }, + { + "accuracy": 0.45696, + "f1": 0.410826, + "f1_weighted": 0.446189 + }, + { + "accuracy": 0.456624, + "f1": 0.424796, + "f1_weighted": 0.453152 + }, + { + "accuracy": 0.452925, + "f1": 0.414412, + "f1_weighted": 0.443257 + }, + { + "accuracy": 0.474781, + "f1": 0.435084, + "f1_weighted": 0.462554 + }, + { + "accuracy": 0.435104, + "f1": 0.41507, + "f1_weighted": 0.417762 + }, + { + "accuracy": 0.446537, + "f1": 0.430751, + "f1_weighted": 0.440003 + } + ], + "main_score": 0.462105, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ] + }, + { + "accuracy": 0.340148, + "f1": 0.32042, + "f1_weighted": 0.324847, + "scores_per_experiment": [ + { + "accuracy": 0.349361, + "f1": 0.333273, + "f1_weighted": 0.332194 + }, + { + "accuracy": 0.349361, + "f1": 0.328293, + "f1_weighted": 0.341102 + }, + { + "accuracy": 0.339946, + "f1": 0.31866, + "f1_weighted": 0.322941 + }, + { + "accuracy": 0.339274, + "f1": 0.322717, + "f1_weighted": 0.329954 + }, + { + "accuracy": 0.34768, + "f1": 0.315391, + "f1_weighted": 0.336091 + }, + { + "accuracy": 0.338265, + "f1": 0.317889, + "f1_weighted": 0.313101 + }, + { + "accuracy": 0.341627, + "f1": 0.325124, + "f1_weighted": 0.322736 + }, + { + "accuracy": 0.354069, + "f1": 0.32262, + "f1_weighted": 0.338855 + }, + { + "accuracy": 0.330195, + "f1": 0.314253, + "f1_weighted": 0.311827 + }, + { + "accuracy": 0.311701, + "f1": 0.305985, + "f1_weighted": 0.299665 + } + ], + "main_score": 0.340148, + "hf_subset": "jv", + "languages": [ + "jav-Latn" + ] + }, + { + "accuracy": 0.379354, + "f1": 0.348406, + "f1_weighted": 0.368068, + "scores_per_experiment": [ + { + "accuracy": 0.400807, + "f1": 0.372621, + "f1_weighted": 0.395059 + }, + { + "accuracy": 0.379287, + "f1": 0.344219, + "f1_weighted": 0.372808 + }, + { + "accuracy": 0.361466, + "f1": 0.341554, + "f1_weighted": 0.349664 + }, + { + "accuracy": 0.374243, + "f1": 0.32856, + "f1_weighted": 0.359929 + }, + { + "accuracy": 0.387021, + "f1": 0.337431, + "f1_weighted": 0.379065 + }, + { + "accuracy": 0.380968, + "f1": 0.367298, + "f1_weighted": 0.377152 + }, + { + "accuracy": 0.368191, + "f1": 0.349527, + "f1_weighted": 0.347767 + }, + { + "accuracy": 0.394418, + "f1": 0.364762, + "f1_weighted": 0.382533 + }, + { + "accuracy": 0.359449, + "f1": 0.333259, + "f1_weighted": 0.337563 + }, + { + "accuracy": 0.387693, + "f1": 0.344825, + "f1_weighted": 0.379141 + } + ], + "main_score": 0.379354, + "hf_subset": "af", + "languages": [ + "afr-Latn" + ] + }, + { + "accuracy": 0.026496, + "f1": 0.010828, + "f1_weighted": 0.013366, + "scores_per_experiment": [ + { + "accuracy": 0.067249, + "f1": 0.012445, + "f1_weighted": 0.024342 + }, + { + "accuracy": 0.016476, + "f1": 0.006006, + "f1_weighted": 0.008521 + }, + { + "accuracy": 0.028581, + "f1": 0.021191, + "f1_weighted": 0.019475 + }, + { + "accuracy": 0.017149, + "f1": 0.006341, + "f1_weighted": 0.008125 + }, + { + "accuracy": 0.013114, + "f1": 0.003935, + "f1_weighted": 0.003168 + }, + { + "accuracy": 0.017821, + "f1": 0.009602, + "f1_weighted": 0.010578 + }, + { + "accuracy": 0.022192, + "f1": 0.009796, + "f1_weighted": 0.01208 + }, + { + "accuracy": 0.04035, + "f1": 0.012777, + "f1_weighted": 0.017605 + }, + { + "accuracy": 0.027572, + "f1": 0.019046, + "f1_weighted": 0.025098 + }, + { + "accuracy": 0.014459, + "f1": 0.007142, + "f1_weighted": 0.004671 + } + ], + "main_score": 0.026496, + "hf_subset": "bn", + "languages": [ + "ben-Beng" + ] + }, + { + "accuracy": 0.510726, + "f1": 0.487621, + "f1_weighted": 0.499369, + "scores_per_experiment": [ + { + "accuracy": 0.512777, + "f1": 0.495488, + "f1_weighted": 0.504305 + }, + { + "accuracy": 0.540013, + "f1": 0.51398, + "f1_weighted": 0.536515 + }, + { + "accuracy": 0.490921, + "f1": 0.469495, + "f1_weighted": 0.483779 + }, + { + "accuracy": 0.518157, + "f1": 0.487632, + "f1_weighted": 0.507368 + }, + { + "accuracy": 0.52421, + "f1": 0.483993, + "f1_weighted": 0.512473 + }, + { + "accuracy": 0.476463, + "f1": 0.47059, + "f1_weighted": 0.45233 + }, + { + "accuracy": 0.519839, + "f1": 0.501172, + "f1_weighted": 0.512655 + }, + { + "accuracy": 0.51883, + "f1": 0.486125, + "f1_weighted": 0.507927 + }, + { + "accuracy": 0.491594, + "f1": 0.479055, + "f1_weighted": 0.470497 + }, + { + "accuracy": 0.514459, + "f1": 0.48868, + "f1_weighted": 0.505844 + } + ], + "main_score": 0.510726, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ] + }, + { + "accuracy": 0.044418, + "f1": 0.012019, + "f1_weighted": 0.017136, + "scores_per_experiment": [ + { + "accuracy": 0.091459, + "f1": 0.019569, + "f1_weighted": 0.031913 + }, + { + "accuracy": 0.029926, + "f1": 0.014535, + "f1_weighted": 0.016609 + }, + { + "accuracy": 0.017821, + "f1": 0.00602, + "f1_weighted": 0.008664 + }, + { + "accuracy": 0.028581, + "f1": 0.004719, + "f1_weighted": 0.006275 + }, + { + "accuracy": 0.074983, + "f1": 0.011546, + "f1_weighted": 0.024655 + }, + { + "accuracy": 0.03228, + "f1": 0.014485, + "f1_weighted": 0.015555 + }, + { + "accuracy": 0.046066, + "f1": 0.01195, + "f1_weighted": 0.017281 + }, + { + "accuracy": 0.015804, + "f1": 0.007397, + "f1_weighted": 0.00881 + }, + { + "accuracy": 0.073974, + "f1": 0.013724, + "f1_weighted": 0.022721 + }, + { + "accuracy": 0.033289, + "f1": 0.016242, + "f1_weighted": 0.018875 + } + ], + "main_score": 0.044418, + "hf_subset": "my", + "languages": [ + "mya-Mymr" + ] + }, + { + "accuracy": 0.405044, + "f1": 0.37566, + "f1_weighted": 0.38565, + "scores_per_experiment": [ + { + "accuracy": 0.421654, + "f1": 0.393689, + "f1_weighted": 0.408497 + }, + { + "accuracy": 0.416274, + "f1": 0.387165, + "f1_weighted": 0.407158 + }, + { + "accuracy": 0.384667, + "f1": 0.354597, + "f1_weighted": 0.359928 + }, + { + "accuracy": 0.395763, + "f1": 0.37463, + "f1_weighted": 0.373986 + }, + { + "accuracy": 0.415602, + "f1": 0.371079, + "f1_weighted": 0.394375 + }, + { + "accuracy": 0.394418, + "f1": 0.365467, + "f1_weighted": 0.36655 + }, + { + "accuracy": 0.395091, + "f1": 0.368695, + "f1_weighted": 0.369394 + }, + { + "accuracy": 0.430397, + "f1": 0.384843, + "f1_weighted": 0.409893 + }, + { + "accuracy": 0.400471, + "f1": 0.386944, + "f1_weighted": 0.381152 + }, + { + "accuracy": 0.3961, + "f1": 0.369494, + "f1_weighted": 0.385569 + } + ], + "main_score": 0.405044, + "hf_subset": "nl", + "languages": [ + "nld-Latn" + ] + }, + { + "accuracy": 0.337525, + "f1": 0.311291, + "f1_weighted": 0.319081, + "scores_per_experiment": [ + { + "accuracy": 0.354405, + "f1": 0.32428, + "f1_weighted": 0.339846 + }, + { + "accuracy": 0.31809, + "f1": 0.288673, + "f1_weighted": 0.296448 + }, + { + "accuracy": 0.336584, + "f1": 0.318851, + "f1_weighted": 0.310873 + }, + { + "accuracy": 0.32616, + "f1": 0.305166, + "f1_weighted": 0.309999 + }, + { + "accuracy": 0.343981, + "f1": 0.289247, + "f1_weighted": 0.328276 + }, + { + "accuracy": 0.347007, + "f1": 0.322961, + "f1_weighted": 0.326437 + }, + { + "accuracy": 0.320444, + "f1": 0.300428, + "f1_weighted": 0.298666 + }, + { + "accuracy": 0.344654, + "f1": 0.317654, + "f1_weighted": 0.32761 + }, + { + "accuracy": 0.325824, + "f1": 0.31588, + "f1_weighted": 0.302828 + }, + { + "accuracy": 0.358104, + "f1": 0.329769, + "f1_weighted": 0.349823 + } + ], + "main_score": 0.337525, + "hf_subset": "hu", + "languages": [ + "hun-Latn" + ] + }, + { + "accuracy": 0.383255, + "f1": 0.365683, + "f1_weighted": 0.368304, + "scores_per_experiment": [ + { + "accuracy": 0.429388, + "f1": 0.400982, + "f1_weighted": 0.417441 + }, + { + "accuracy": 0.395427, + "f1": 0.38598, + "f1_weighted": 0.380545 + }, + { + "accuracy": 0.365501, + "f1": 0.351977, + "f1_weighted": 0.349052 + }, + { + "accuracy": 0.390047, + "f1": 0.376535, + "f1_weighted": 0.380627 + }, + { + "accuracy": 0.382986, + "f1": 0.346132, + "f1_weighted": 0.363135 + }, + { + "accuracy": 0.35575, + "f1": 0.337882, + "f1_weighted": 0.348712 + }, + { + "accuracy": 0.377606, + "f1": 0.361989, + "f1_weighted": 0.364461 + }, + { + "accuracy": 0.390047, + "f1": 0.382306, + "f1_weighted": 0.371118 + }, + { + "accuracy": 0.371553, + "f1": 0.359161, + "f1_weighted": 0.348505 + }, + { + "accuracy": 0.374243, + "f1": 0.35389, + "f1_weighted": 0.359445 + } + ], + "main_score": 0.383255, + "hf_subset": "lv", + "languages": [ + "lav-Latn" + ] + }, + { + "accuracy": 0.417149, + "f1": 0.396717, + "f1_weighted": 0.406137, + "scores_per_experiment": [ + { + "accuracy": 0.434768, + "f1": 0.405946, + "f1_weighted": 0.427646 + }, + { + "accuracy": 0.441829, + "f1": 0.41456, + "f1_weighted": 0.439724 + }, + { + "accuracy": 0.405178, + "f1": 0.385957, + "f1_weighted": 0.39554 + }, + { + "accuracy": 0.405178, + "f1": 0.384119, + "f1_weighted": 0.391481 + }, + { + "accuracy": 0.423672, + "f1": 0.3946, + "f1_weighted": 0.414149 + }, + { + "accuracy": 0.407868, + "f1": 0.406494, + "f1_weighted": 0.388037 + }, + { + "accuracy": 0.416611, + "f1": 0.395869, + "f1_weighted": 0.404553 + }, + { + "accuracy": 0.434095, + "f1": 0.422375, + "f1_weighted": 0.427619 + }, + { + "accuracy": 0.392737, + "f1": 0.370109, + "f1_weighted": 0.370716 + }, + { + "accuracy": 0.409549, + "f1": 0.387144, + "f1_weighted": 0.401905 + } + ], + "main_score": 0.417149, + "hf_subset": "da", + "languages": [ + "dan-Latn" + ] + }, + { + "accuracy": 0.3154, + "f1": 0.301933, + "f1_weighted": 0.303532, + "scores_per_experiment": [ + { + "accuracy": 0.32885, + "f1": 0.318308, + "f1_weighted": 0.313122 + }, + { + "accuracy": 0.302959, + "f1": 0.291139, + "f1_weighted": 0.292982 + }, + { + "accuracy": 0.306658, + "f1": 0.290987, + "f1_weighted": 0.292816 + }, + { + "accuracy": 0.333894, + "f1": 0.319096, + "f1_weighted": 0.326228 + }, + { + "accuracy": 0.335911, + "f1": 0.313905, + "f1_weighted": 0.333772 + }, + { + "accuracy": 0.290854, + "f1": 0.286515, + "f1_weighted": 0.277215 + }, + { + "accuracy": 0.324479, + "f1": 0.308949, + "f1_weighted": 0.312522 + }, + { + "accuracy": 0.311029, + "f1": 0.295108, + "f1_weighted": 0.295106 + }, + { + "accuracy": 0.302623, + "f1": 0.287898, + "f1_weighted": 0.28217 + }, + { + "accuracy": 0.316745, + "f1": 0.307419, + "f1_weighted": 0.309382 + } + ], + "main_score": 0.3154, + "hf_subset": "is", + "languages": [ + "isl-Latn" + ] + }, + { + "accuracy": 0.097276, + "f1": 0.053028, + "f1_weighted": 0.075101, + "scores_per_experiment": [ + { + "accuracy": 0.096167, + "f1": 0.059027, + "f1_weighted": 0.081414 + }, + { + "accuracy": 0.094822, + "f1": 0.049526, + "f1_weighted": 0.067183 + }, + { + "accuracy": 0.087088, + "f1": 0.056085, + "f1_weighted": 0.07645 + }, + { + "accuracy": 0.113988, + "f1": 0.05834, + "f1_weighted": 0.07868 + }, + { + "accuracy": 0.095494, + "f1": 0.05855, + "f1_weighted": 0.080144 + }, + { + "accuracy": 0.098521, + "f1": 0.053451, + "f1_weighted": 0.084865 + }, + { + "accuracy": 0.084062, + "f1": 0.043511, + "f1_weighted": 0.067294 + }, + { + "accuracy": 0.082044, + "f1": 0.045207, + "f1_weighted": 0.065742 + }, + { + "accuracy": 0.113988, + "f1": 0.04541, + "f1_weighted": 0.066282 + }, + { + "accuracy": 0.10659, + "f1": 0.061171, + "f1_weighted": 0.082951 + } + ], + "main_score": 0.097276, + "hf_subset": "el", + "languages": [ + "ell-Grek" + ] + }, + { + "accuracy": 0.370377, + "f1": 0.349854, + "f1_weighted": 0.355963, + "scores_per_experiment": [ + { + "accuracy": 0.389711, + "f1": 0.356491, + "f1_weighted": 0.373094 + }, + { + "accuracy": 0.369872, + "f1": 0.344836, + "f1_weighted": 0.362099 + }, + { + "accuracy": 0.354405, + "f1": 0.358726, + "f1_weighted": 0.34078 + }, + { + "accuracy": 0.374916, + "f1": 0.340741, + "f1_weighted": 0.358264 + }, + { + "accuracy": 0.39879, + "f1": 0.364765, + "f1_weighted": 0.388912 + }, + { + "accuracy": 0.347007, + "f1": 0.342076, + "f1_weighted": 0.332813 + }, + { + "accuracy": 0.381977, + "f1": 0.356365, + "f1_weighted": 0.36671 + }, + { + "accuracy": 0.366174, + "f1": 0.341997, + "f1_weighted": 0.346634 + }, + { + "accuracy": 0.358776, + "f1": 0.333133, + "f1_weighted": 0.335392 + }, + { + "accuracy": 0.362139, + "f1": 0.359407, + "f1_weighted": 0.354932 + } + ], + "main_score": 0.370377, + "hf_subset": "sl", + "languages": [ + "slv-Latn" + ] + }, + { + "accuracy": 0.327808, + "f1": 0.316457, + "f1_weighted": 0.317372, + "scores_per_experiment": [ + { + "accuracy": 0.303295, + "f1": 0.305944, + "f1_weighted": 0.285086 + }, + { + "accuracy": 0.345662, + "f1": 0.326614, + "f1_weighted": 0.34108 + }, + { + "accuracy": 0.308675, + "f1": 0.311951, + "f1_weighted": 0.295819 + }, + { + "accuracy": 0.350706, + "f1": 0.325313, + "f1_weighted": 0.343219 + }, + { + "accuracy": 0.342636, + "f1": 0.31576, + "f1_weighted": 0.33792 + }, + { + "accuracy": 0.36651, + "f1": 0.345593, + "f1_weighted": 0.360202 + }, + { + "accuracy": 0.316409, + "f1": 0.310553, + "f1_weighted": 0.31292 + }, + { + "accuracy": 0.316409, + "f1": 0.306433, + "f1_weighted": 0.296351 + }, + { + "accuracy": 0.311365, + "f1": 0.306037, + "f1_weighted": 0.283922 + }, + { + "accuracy": 0.316409, + "f1": 0.310368, + "f1_weighted": 0.317204 + } + ], + "main_score": 0.327808, + "hf_subset": "az", + "languages": [ + "aze-Latn" + ] + }, + { + "accuracy": 0.362071, + "f1": 0.347916, + "f1_weighted": 0.345887, + "scores_per_experiment": [ + { + "accuracy": 0.36113, + "f1": 0.356734, + "f1_weighted": 0.341385 + }, + { + "accuracy": 0.35575, + "f1": 0.359127, + "f1_weighted": 0.351033 + }, + { + "accuracy": 0.338265, + "f1": 0.33589, + "f1_weighted": 0.326563 + }, + { + "accuracy": 0.372562, + "f1": 0.350171, + "f1_weighted": 0.361706 + }, + { + "accuracy": 0.376597, + "f1": 0.352224, + "f1_weighted": 0.35714 + }, + { + "accuracy": 0.375925, + "f1": 0.348587, + "f1_weighted": 0.360855 + }, + { + "accuracy": 0.353396, + "f1": 0.349514, + "f1_weighted": 0.33209 + }, + { + "accuracy": 0.381641, + "f1": 0.343216, + "f1_weighted": 0.364445 + }, + { + "accuracy": 0.325488, + "f1": 0.321928, + "f1_weighted": 0.294687 + }, + { + "accuracy": 0.37996, + "f1": 0.36177, + "f1_weighted": 0.368969 + } + ], + "main_score": 0.362071, + "hf_subset": "sw", + "languages": [ + "swa-Latn" + ] + }, + { + "accuracy": 0.269401, + "f1": 0.245598, + "f1_weighted": 0.241294, + "scores_per_experiment": [ + { + "accuracy": 0.275387, + "f1": 0.270111, + "f1_weighted": 0.25386 + }, + { + "accuracy": 0.267317, + "f1": 0.250234, + "f1_weighted": 0.242166 + }, + { + "accuracy": 0.27505, + "f1": 0.246589, + "f1_weighted": 0.256782 + }, + { + "accuracy": 0.249832, + "f1": 0.227107, + "f1_weighted": 0.215518 + }, + { + "accuracy": 0.257902, + "f1": 0.225174, + "f1_weighted": 0.229151 + }, + { + "accuracy": 0.269334, + "f1": 0.249841, + "f1_weighted": 0.22426 + }, + { + "accuracy": 0.270679, + "f1": 0.252839, + "f1_weighted": 0.242409 + }, + { + "accuracy": 0.277404, + "f1": 0.244785, + "f1_weighted": 0.25692 + }, + { + "accuracy": 0.302959, + "f1": 0.264623, + "f1_weighted": 0.269889 + }, + { + "accuracy": 0.248151, + "f1": 0.224675, + "f1_weighted": 0.221987 + } + ], + "main_score": 0.269401, + "hf_subset": "vi", + "languages": [ + "vie-Latn" + ] + }, + { + "accuracy": 0.015434, + "f1": 0.004763, + "f1_weighted": 0.006541, + "scores_per_experiment": [ + { + "accuracy": 0.015131, + "f1": 0.006183, + "f1_weighted": 0.008844 + }, + { + "accuracy": 0.00807, + "f1": 0.004057, + "f1_weighted": 0.002321 + }, + { + "accuracy": 0.017821, + "f1": 0.004204, + "f1_weighted": 0.012941 + }, + { + "accuracy": 0.013786, + "f1": 0.004129, + "f1_weighted": 0.004643 + }, + { + "accuracy": 0.017149, + "f1": 0.002981, + "f1_weighted": 0.001969 + }, + { + "accuracy": 0.006725, + "f1": 0.00167, + "f1_weighted": 0.005001 + }, + { + "accuracy": 0.01614, + "f1": 0.008938, + "f1_weighted": 0.007089 + }, + { + "accuracy": 0.01076, + "f1": 0.004373, + "f1_weighted": 0.005461 + }, + { + "accuracy": 0.023537, + "f1": 0.004917, + "f1_weighted": 0.012526 + }, + { + "accuracy": 0.025219, + "f1": 0.006183, + "f1_weighted": 0.004613 + } + ], + "main_score": 0.015434, + "hf_subset": "ta", + "languages": [ + "tam-Taml" + ] + }, + { + "accuracy": 0.011197, + "f1": 0.001876, + "f1_weighted": 0.001193, + "scores_per_experiment": [ + { + "accuracy": 0.019166, + "f1": 0.001362, + "f1_weighted": 0.000896 + }, + { + "accuracy": 0.004371, + "f1": 0.002417, + "f1_weighted": 0.001169 + }, + { + "accuracy": 0.015131, + "f1": 0.002743, + "f1_weighted": 0.00151 + }, + { + "accuracy": 0.006725, + "f1": 0.000962, + "f1_weighted": 0.000612 + }, + { + "accuracy": 0.012441, + "f1": 0.00172, + "f1_weighted": 0.000854 + }, + { + "accuracy": 0.005044, + "f1": 0.002287, + "f1_weighted": 0.001537 + }, + { + "accuracy": 0.006725, + "f1": 0.000649, + "f1_weighted": 0.000194 + }, + { + "accuracy": 0.008406, + "f1": 0.003348, + "f1_weighted": 0.002863 + }, + { + "accuracy": 0.02421, + "f1": 0.001237, + "f1_weighted": 0.001256 + }, + { + "accuracy": 0.009751, + "f1": 0.002034, + "f1_weighted": 0.001042 + } + ], + "main_score": 0.011197, + "hf_subset": "zh-CN", + "languages": [ + "cmo-Hans" + ] + }, + { + "accuracy": 0.404842, + "f1": 0.375484, + "f1_weighted": 0.391294, + "scores_per_experiment": [ + { + "accuracy": 0.411231, + "f1": 0.385912, + "f1_weighted": 0.401363 + }, + { + "accuracy": 0.400471, + "f1": 0.379219, + "f1_weighted": 0.39472 + }, + { + "accuracy": 0.422327, + "f1": 0.377495, + "f1_weighted": 0.406742 + }, + { + "accuracy": 0.396772, + "f1": 0.362508, + "f1_weighted": 0.378666 + }, + { + "accuracy": 0.420982, + "f1": 0.385884, + "f1_weighted": 0.405559 + }, + { + "accuracy": 0.407532, + "f1": 0.375857, + "f1_weighted": 0.390264 + }, + { + "accuracy": 0.381641, + "f1": 0.363402, + "f1_weighted": 0.378227 + }, + { + "accuracy": 0.422999, + "f1": 0.383113, + "f1_weighted": 0.41042 + }, + { + "accuracy": 0.39072, + "f1": 0.361384, + "f1_weighted": 0.37091 + }, + { + "accuracy": 0.393746, + "f1": 0.38007, + "f1_weighted": 0.376071 + } + ], + "main_score": 0.404842, + "hf_subset": "sq", + "languages": [ + "sqi-Latn" + ] + }, + { + "accuracy": 0.023638, + "f1": 0.006977, + "f1_weighted": 0.007109, + "scores_per_experiment": [ + { + "accuracy": 0.010424, + "f1": 0.003783, + "f1_weighted": 0.002176 + }, + { + "accuracy": 0.008406, + "f1": 0.004548, + "f1_weighted": 0.002289 + }, + { + "accuracy": 0.007397, + "f1": 0.004731, + "f1_weighted": 0.002538 + }, + { + "accuracy": 0.01614, + "f1": 0.008025, + "f1_weighted": 0.007337 + }, + { + "accuracy": 0.063551, + "f1": 0.005686, + "f1_weighted": 0.016722 + }, + { + "accuracy": 0.011096, + "f1": 0.005956, + "f1_weighted": 0.00392 + }, + { + "accuracy": 0.015467, + "f1": 0.002655, + "f1_weighted": 0.001967 + }, + { + "accuracy": 0.009751, + "f1": 0.010181, + "f1_weighted": 0.003927 + }, + { + "accuracy": 0.068594, + "f1": 0.014458, + "f1_weighted": 0.022802 + }, + { + "accuracy": 0.025555, + "f1": 0.009748, + "f1_weighted": 0.007415 + } + ], + "main_score": 0.023638, + "hf_subset": "he", + "languages": [ + "heb-Hebr" + ] + }, + { + "accuracy": 0.031036, + "f1": 0.018235, + "f1_weighted": 0.015921, + "scores_per_experiment": [ + { + "accuracy": 0.067249, + "f1": 0.027422, + "f1_weighted": 0.022833 + }, + { + "accuracy": 0.062878, + "f1": 0.009156, + "f1_weighted": 0.01389 + }, + { + "accuracy": 0.015467, + "f1": 0.016056, + "f1_weighted": 0.013448 + }, + { + "accuracy": 0.012777, + "f1": 0.017282, + "f1_weighted": 0.01381 + }, + { + "accuracy": 0.067922, + "f1": 0.009597, + "f1_weighted": 0.016954 + }, + { + "accuracy": 0.022529, + "f1": 0.023311, + "f1_weighted": 0.019492 + }, + { + "accuracy": 0.01345, + "f1": 0.013987, + "f1_weighted": 0.0111 + }, + { + "accuracy": 0.019166, + "f1": 0.024484, + "f1_weighted": 0.019906 + }, + { + "accuracy": 0.018157, + "f1": 0.025793, + "f1_weighted": 0.020429 + }, + { + "accuracy": 0.01076, + "f1": 0.015261, + "f1_weighted": 0.007343 + } + ], + "main_score": 0.031036, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ] + }, + { + "accuracy": 0.025185, + "f1": 0.009124, + "f1_weighted": 0.014325, + "scores_per_experiment": [ + { + "accuracy": 0.030599, + "f1": 0.009563, + "f1_weighted": 0.025833 + }, + { + "accuracy": 0.017485, + "f1": 0.011061, + "f1_weighted": 0.012941 + }, + { + "accuracy": 0.020511, + "f1": 0.007656, + "f1_weighted": 0.02112 + }, + { + "accuracy": 0.011432, + "f1": 0.007839, + "f1_weighted": 0.009626 + }, + { + "accuracy": 0.019502, + "f1": 0.012641, + "f1_weighted": 0.008656 + }, + { + "accuracy": 0.020511, + "f1": 0.01292, + "f1_weighted": 0.014961 + }, + { + "accuracy": 0.010087, + "f1": 0.004644, + "f1_weighted": 0.005334 + }, + { + "accuracy": 0.012105, + "f1": 0.004887, + "f1_weighted": 0.005796 + }, + { + "accuracy": 0.068931, + "f1": 0.01022, + "f1_weighted": 0.02388 + }, + { + "accuracy": 0.040686, + "f1": 0.009804, + "f1_weighted": 0.015104 + } + ], + "main_score": 0.025185, + "hf_subset": "te", + "languages": [ + "tel-Telu" + ] + }, + { + "accuracy": 0.029052, + "f1": 0.015303, + "f1_weighted": 0.016223, + "scores_per_experiment": [ + { + "accuracy": 0.021184, + "f1": 0.015028, + "f1_weighted": 0.015155 + }, + { + "accuracy": 0.02152, + "f1": 0.018019, + "f1_weighted": 0.016332 + }, + { + "accuracy": 0.031271, + "f1": 0.028646, + "f1_weighted": 0.027703 + }, + { + "accuracy": 0.019502, + "f1": 0.01436, + "f1_weighted": 0.0116 + }, + { + "accuracy": 0.024882, + "f1": 0.014957, + "f1_weighted": 0.010588 + }, + { + "accuracy": 0.032616, + "f1": 0.015803, + "f1_weighted": 0.019574 + }, + { + "accuracy": 0.022529, + "f1": 0.011582, + "f1_weighted": 0.014269 + }, + { + "accuracy": 0.019166, + "f1": 0.008679, + "f1_weighted": 0.010489 + }, + { + "accuracy": 0.071957, + "f1": 0.01342, + "f1_weighted": 0.026165 + }, + { + "accuracy": 0.025891, + "f1": 0.012534, + "f1_weighted": 0.010351 + } + ], + "main_score": 0.029052, + "hf_subset": "hi", + "languages": [ + "hin-Deva" + ] + }, + { + "accuracy": 0.027505, + "f1": 0.015648, + "f1_weighted": 0.016959, + "scores_per_experiment": [ + { + "accuracy": 0.047747, + "f1": 0.021848, + "f1_weighted": 0.037262 + }, + { + "accuracy": 0.013114, + "f1": 0.014001, + "f1_weighted": 0.00638 + }, + { + "accuracy": 0.026227, + "f1": 0.016863, + "f1_weighted": 0.022744 + }, + { + "accuracy": 0.021856, + "f1": 0.016137, + "f1_weighted": 0.017515 + }, + { + "accuracy": 0.01883, + "f1": 0.015426, + "f1_weighted": 0.012895 + }, + { + "accuracy": 0.02152, + "f1": 0.009651, + "f1_weighted": 0.013209 + }, + { + "accuracy": 0.014459, + "f1": 0.005248, + "f1_weighted": 0.005397 + }, + { + "accuracy": 0.016812, + "f1": 0.02057, + "f1_weighted": 0.019058 + }, + { + "accuracy": 0.071621, + "f1": 0.023737, + "f1_weighted": 0.027209 + }, + { + "accuracy": 0.022865, + "f1": 0.013004, + "f1_weighted": 0.007925 + } + ], + "main_score": 0.027505, + "hf_subset": "ur", + "languages": [ + "urd-Arab" + ] + }, + { + "accuracy": 0.463013, + "f1": 0.449193, + "f1_weighted": 0.445964, + "scores_per_experiment": [ + { + "accuracy": 0.47848, + "f1": 0.469017, + "f1_weighted": 0.468722 + }, + { + "accuracy": 0.477135, + "f1": 0.469099, + "f1_weighted": 0.461195 + }, + { + "accuracy": 0.434095, + "f1": 0.427007, + "f1_weighted": 0.409561 + }, + { + "accuracy": 0.455952, + "f1": 0.438426, + "f1_weighted": 0.441615 + }, + { + "accuracy": 0.490249, + "f1": 0.451244, + "f1_weighted": 0.481996 + }, + { + "accuracy": 0.471755, + "f1": 0.456911, + "f1_weighted": 0.462231 + }, + { + "accuracy": 0.451917, + "f1": 0.445482, + "f1_weighted": 0.429747 + }, + { + "accuracy": 0.464358, + "f1": 0.452058, + "f1_weighted": 0.449155 + }, + { + "accuracy": 0.449899, + "f1": 0.44032, + "f1_weighted": 0.424755 + }, + { + "accuracy": 0.456288, + "f1": 0.44237, + "f1_weighted": 0.430659 + } + ], + "main_score": 0.463013, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ] + }, + { + "accuracy": 0.398352, + "f1": 0.386953, + "f1_weighted": 0.381742, + "scores_per_experiment": [ + { + "accuracy": 0.421654, + "f1": 0.414236, + "f1_weighted": 0.403219 + }, + { + "accuracy": 0.394082, + "f1": 0.383491, + "f1_weighted": 0.378632 + }, + { + "accuracy": 0.3961, + "f1": 0.382518, + "f1_weighted": 0.379931 + }, + { + "accuracy": 0.411567, + "f1": 0.395854, + "f1_weighted": 0.402181 + }, + { + "accuracy": 0.392401, + "f1": 0.373866, + "f1_weighted": 0.376722 + }, + { + "accuracy": 0.410894, + "f1": 0.394277, + "f1_weighted": 0.386612 + }, + { + "accuracy": 0.38534, + "f1": 0.381, + "f1_weighted": 0.36262 + }, + { + "accuracy": 0.412912, + "f1": 0.386683, + "f1_weighted": 0.397338 + }, + { + "accuracy": 0.378951, + "f1": 0.378336, + "f1_weighted": 0.361328 + }, + { + "accuracy": 0.379623, + "f1": 0.379272, + "f1_weighted": 0.36884 + } + ], + "main_score": 0.398352, + "hf_subset": "id", + "languages": [ + "ind-Latn" + ] + }, + { + "accuracy": 0.040215, + "f1": 0.015984, + "f1_weighted": 0.015684, + "scores_per_experiment": [ + { + "accuracy": 0.055817, + "f1": 0.017948, + "f1_weighted": 0.01487 + }, + { + "accuracy": 0.055817, + "f1": 0.010886, + "f1_weighted": 0.023459 + }, + { + "accuracy": 0.027572, + "f1": 0.015676, + "f1_weighted": 0.010251 + }, + { + "accuracy": 0.02959, + "f1": 0.012969, + "f1_weighted": 0.011599 + }, + { + "accuracy": 0.051446, + "f1": 0.020367, + "f1_weighted": 0.0181 + }, + { + "accuracy": 0.047747, + "f1": 0.017144, + "f1_weighted": 0.026368 + }, + { + "accuracy": 0.03766, + "f1": 0.013779, + "f1_weighted": 0.009325 + }, + { + "accuracy": 0.029254, + "f1": 0.014533, + "f1_weighted": 0.009792 + }, + { + "accuracy": 0.022192, + "f1": 0.017476, + "f1_weighted": 0.010533 + }, + { + "accuracy": 0.045057, + "f1": 0.019063, + "f1_weighted": 0.022548 + } + ], + "main_score": 0.040215, + "hf_subset": "th", + "languages": [ + "tha-Thai" + ] + }, + { + "accuracy": 0.356826, + "f1": 0.32398, + "f1_weighted": 0.341082, + "scores_per_experiment": [ + { + "accuracy": 0.370881, + "f1": 0.336953, + "f1_weighted": 0.359182 + }, + { + "accuracy": 0.366846, + "f1": 0.327065, + "f1_weighted": 0.355077 + }, + { + "accuracy": 0.338937, + "f1": 0.312663, + "f1_weighted": 0.313987 + }, + { + "accuracy": 0.338937, + "f1": 0.311258, + "f1_weighted": 0.31334 + }, + { + "accuracy": 0.363147, + "f1": 0.321598, + "f1_weighted": 0.346493 + }, + { + "accuracy": 0.341627, + "f1": 0.309558, + "f1_weighted": 0.323056 + }, + { + "accuracy": 0.364829, + "f1": 0.332417, + "f1_weighted": 0.355239 + }, + { + "accuracy": 0.364492, + "f1": 0.323176, + "f1_weighted": 0.348037 + }, + { + "accuracy": 0.339274, + "f1": 0.317428, + "f1_weighted": 0.318964 + }, + { + "accuracy": 0.379287, + "f1": 0.347688, + "f1_weighted": 0.37744 + } + ], + "main_score": 0.356826, + "hf_subset": "fi", + "languages": [ + "fin-Latn" + ] + }, + { + "accuracy": 0.363887, + "f1": 0.339244, + "f1_weighted": 0.345264, + "scores_per_experiment": [ + { + "accuracy": 0.354741, + "f1": 0.327649, + "f1_weighted": 0.338067 + }, + { + "accuracy": 0.370881, + "f1": 0.35012, + "f1_weighted": 0.344781 + }, + { + "accuracy": 0.360457, + "f1": 0.343599, + "f1_weighted": 0.343988 + }, + { + "accuracy": 0.370208, + "f1": 0.339506, + "f1_weighted": 0.352459 + }, + { + "accuracy": 0.362811, + "f1": 0.321685, + "f1_weighted": 0.341672 + }, + { + "accuracy": 0.358104, + "f1": 0.346496, + "f1_weighted": 0.351645 + }, + { + "accuracy": 0.372562, + "f1": 0.351953, + "f1_weighted": 0.354021 + }, + { + "accuracy": 0.37996, + "f1": 0.342866, + "f1_weighted": 0.361531 + }, + { + "accuracy": 0.332549, + "f1": 0.313963, + "f1_weighted": 0.300719 + }, + { + "accuracy": 0.376597, + "f1": 0.354604, + "f1_weighted": 0.36376 + } + ], + "main_score": 0.363887, + "hf_subset": "ms", + "languages": [ + "msa-Latn" + ] + }, + { + "accuracy": 0.057028, + "f1": 0.015832, + "f1_weighted": 0.018924, + "scores_per_experiment": [ + { + "accuracy": 0.049092, + "f1": 0.02047, + "f1_weighted": 0.015362 + }, + { + "accuracy": 0.045057, + "f1": 0.007266, + "f1_weighted": 0.008067 + }, + { + "accuracy": 0.054472, + "f1": 0.0092, + "f1_weighted": 0.011728 + }, + { + "accuracy": 0.061533, + "f1": 0.019568, + "f1_weighted": 0.023388 + }, + { + "accuracy": 0.071284, + "f1": 0.021281, + "f1_weighted": 0.032233 + }, + { + "accuracy": 0.037323, + "f1": 0.022067, + "f1_weighted": 0.018125 + }, + { + "accuracy": 0.047411, + "f1": 0.012342, + "f1_weighted": 0.013957 + }, + { + "accuracy": 0.084062, + "f1": 0.013375, + "f1_weighted": 0.024739 + }, + { + "accuracy": 0.073974, + "f1": 0.020936, + "f1_weighted": 0.030271 + }, + { + "accuracy": 0.046066, + "f1": 0.011813, + "f1_weighted": 0.011371 + } + ], + "main_score": 0.057028, + "hf_subset": "km", + "languages": [ + "khm-Khmr" + ] + }, + { + "accuracy": 0.022159, + "f1": 0.009117, + "f1_weighted": 0.009469, + "scores_per_experiment": [ + { + "accuracy": 0.0538, + "f1": 0.01079, + "f1_weighted": 0.015919 + }, + { + "accuracy": 0.005716, + "f1": 0.005889, + "f1_weighted": 0.002067 + }, + { + "accuracy": 0.011769, + "f1": 0.012417, + "f1_weighted": 0.008942 + }, + { + "accuracy": 0.010087, + "f1": 0.006011, + "f1_weighted": 0.008398 + }, + { + "accuracy": 0.023537, + "f1": 0.013413, + "f1_weighted": 0.018698 + }, + { + "accuracy": 0.015804, + "f1": 0.011926, + "f1_weighted": 0.00914 + }, + { + "accuracy": 0.00538, + "f1": 0.000853, + "f1_weighted": 0.000209 + }, + { + "accuracy": 0.006389, + "f1": 0.005486, + "f1_weighted": 0.003897 + }, + { + "accuracy": 0.063551, + "f1": 0.009314, + "f1_weighted": 0.017678 + }, + { + "accuracy": 0.025555, + "f1": 0.015071, + "f1_weighted": 0.009742 + } + ], + "main_score": 0.022159, + "hf_subset": "am", + "languages": [ + "amh-Ethi" + ] + } + ] + }, + "evaluation_time": 1627.0867643356323, + "kg_co2_emissions": null } \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/MassiveScenarioClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/MassiveScenarioClassification.json index e71ef3516b..a910294198 100644 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/MassiveScenarioClassification.json +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/MassiveScenarioClassification.json @@ -1,15 +1,6337 @@ { - "test": { - "en": { - "accuracy": 0.7539677202420982, - "accuracy_stderr": 0.020623815205131072, - "f1": 0.7412165626022825, - "f1_stderr": 0.018963346396895613, - "main_score": 0.7539677202420982 - }, - "evaluation_time": 22.56 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "MassiveScenarioClassification", - "dataset_revision": "7d571f92784cd94a019292a1f45445077d0ef634" + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "task_name": "MassiveScenarioClassification", + "mteb_version": "1.34.14", + "scores": { + "validation": [ + { + "accuracy": 0.397344, + "f1": 0.396715, + "f1_weighted": 0.378752, + "scores_per_experiment": [ + { + "accuracy": 0.387113, + "f1": 0.387201, + "f1_weighted": 0.360848 + }, + { + "accuracy": 0.397934, + "f1": 0.396514, + "f1_weighted": 0.381836 + }, + { + "accuracy": 0.429907, + "f1": 0.421628, + "f1_weighted": 0.421278 + }, + { + "accuracy": 0.372356, + "f1": 0.368352, + "f1_weighted": 0.350804 + }, + { + "accuracy": 0.409739, + "f1": 0.408874, + "f1_weighted": 0.39104 + }, + { + "accuracy": 0.394983, + "f1": 0.40266, + "f1_weighted": 0.375949 + }, + { + "accuracy": 0.395967, + "f1": 0.387728, + "f1_weighted": 0.37992 + }, + { + "accuracy": 0.416626, + "f1": 0.415923, + "f1_weighted": 0.393828 + }, + { + "accuracy": 0.362027, + "f1": 0.373514, + "f1_weighted": 0.339769 + }, + { + "accuracy": 0.406788, + "f1": 0.404753, + "f1_weighted": 0.392252 + } + ], + "main_score": 0.397344, + "hf_subset": "tr", + "languages": [ + "tur-Latn" + ] + }, + { + "accuracy": 0.070635, + "f1": 0.026991, + "f1_weighted": 0.02939, + "scores_per_experiment": [ + { + "accuracy": 0.058042, + "f1": 0.020296, + "f1_weighted": 0.01987 + }, + { + "accuracy": 0.073291, + "f1": 0.034379, + "f1_weighted": 0.037212 + }, + { + "accuracy": 0.062469, + "f1": 0.024498, + "f1_weighted": 0.020395 + }, + { + "accuracy": 0.061485, + "f1": 0.026444, + "f1_weighted": 0.025934 + }, + { + "accuracy": 0.090507, + "f1": 0.035841, + "f1_weighted": 0.043881 + }, + { + "accuracy": 0.065912, + "f1": 0.031448, + "f1_weighted": 0.029788 + }, + { + "accuracy": 0.074766, + "f1": 0.020902, + "f1_weighted": 0.023024 + }, + { + "accuracy": 0.097393, + "f1": 0.034123, + "f1_weighted": 0.05463 + }, + { + "accuracy": 0.031481, + "f1": 0.0108, + "f1_weighted": 0.004825 + }, + { + "accuracy": 0.090999, + "f1": 0.031177, + "f1_weighted": 0.034344 + } + ], + "main_score": 0.070635, + "hf_subset": "ka", + "languages": [ + "kat-Geor" + ] + }, + { + "accuracy": 0.066454, + "f1": 0.023658, + "f1_weighted": 0.025602, + "scores_per_experiment": [ + { + "accuracy": 0.051648, + "f1": 0.018547, + "f1_weighted": 0.017965 + }, + { + "accuracy": 0.07575, + "f1": 0.021071, + "f1_weighted": 0.029746 + }, + { + "accuracy": 0.07575, + "f1": 0.033097, + "f1_weighted": 0.040016 + }, + { + "accuracy": 0.068372, + "f1": 0.02081, + "f1_weighted": 0.02898 + }, + { + "accuracy": 0.07575, + "f1": 0.025521, + "f1_weighted": 0.029305 + }, + { + "accuracy": 0.071815, + "f1": 0.022869, + "f1_weighted": 0.022792 + }, + { + "accuracy": 0.045253, + "f1": 0.02088, + "f1_weighted": 0.013076 + }, + { + "accuracy": 0.078701, + "f1": 0.02804, + "f1_weighted": 0.027237 + }, + { + "accuracy": 0.065912, + "f1": 0.026992, + "f1_weighted": 0.029944 + }, + { + "accuracy": 0.055583, + "f1": 0.018757, + "f1_weighted": 0.016962 + } + ], + "main_score": 0.066454, + "hf_subset": "ko", + "languages": [ + "kor-Kore" + ] + }, + { + "accuracy": 0.527545, + "f1": 0.509086, + "f1_weighted": 0.51746, + "scores_per_experiment": [ + { + "accuracy": 0.531727, + "f1": 0.515434, + "f1_weighted": 0.516862 + }, + { + "accuracy": 0.525332, + "f1": 0.515583, + "f1_weighted": 0.517091 + }, + { + "accuracy": 0.563699, + "f1": 0.538839, + "f1_weighted": 0.559858 + }, + { + "accuracy": 0.549434, + "f1": 0.52744, + "f1_weighted": 0.539302 + }, + { + "accuracy": 0.542056, + "f1": 0.5183, + "f1_weighted": 0.538513 + }, + { + "accuracy": 0.498278, + "f1": 0.471886, + "f1_weighted": 0.488541 + }, + { + "accuracy": 0.528775, + "f1": 0.50578, + "f1_weighted": 0.521936 + }, + { + "accuracy": 0.531235, + "f1": 0.518723, + "f1_weighted": 0.515159 + }, + { + "accuracy": 0.484998, + "f1": 0.474765, + "f1_weighted": 0.468476 + }, + { + "accuracy": 0.519921, + "f1": 0.50411, + "f1_weighted": 0.508858 + } + ], + "main_score": 0.527545, + "hf_subset": "it", + "languages": [ + "ita-Latn" + ] + }, + { + "accuracy": 0.229808, + "f1": 0.224224, + "f1_weighted": 0.213901, + "scores_per_experiment": [ + { + "accuracy": 0.204624, + "f1": 0.20803, + "f1_weighted": 0.181329 + }, + { + "accuracy": 0.222823, + "f1": 0.227214, + "f1_weighted": 0.210205 + }, + { + "accuracy": 0.263158, + "f1": 0.24731, + "f1_weighted": 0.254586 + }, + { + "accuracy": 0.242991, + "f1": 0.225492, + "f1_weighted": 0.236339 + }, + { + "accuracy": 0.244466, + "f1": 0.233697, + "f1_weighted": 0.219458 + }, + { + "accuracy": 0.204624, + "f1": 0.195116, + "f1_weighted": 0.168291 + }, + { + "accuracy": 0.233153, + "f1": 0.224614, + "f1_weighted": 0.217402 + }, + { + "accuracy": 0.246926, + "f1": 0.246311, + "f1_weighted": 0.244785 + }, + { + "accuracy": 0.213478, + "f1": 0.214642, + "f1_weighted": 0.202249 + }, + { + "accuracy": 0.22184, + "f1": 0.21981, + "f1_weighted": 0.20437 + } + ], + "main_score": 0.229808, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + }, + { + "accuracy": 0.063207, + "f1": 0.027118, + "f1_weighted": 0.029052, + "scores_per_experiment": [ + { + "accuracy": 0.073291, + "f1": 0.029121, + "f1_weighted": 0.045778 + }, + { + "accuracy": 0.070339, + "f1": 0.038975, + "f1_weighted": 0.039393 + }, + { + "accuracy": 0.043778, + "f1": 0.013297, + "f1_weighted": 0.008956 + }, + { + "accuracy": 0.040826, + "f1": 0.016702, + "f1_weighted": 0.017346 + }, + { + "accuracy": 0.050172, + "f1": 0.027234, + "f1_weighted": 0.018524 + }, + { + "accuracy": 0.050664, + "f1": 0.022904, + "f1_weighted": 0.02103 + }, + { + "accuracy": 0.048205, + "f1": 0.017337, + "f1_weighted": 0.015118 + }, + { + "accuracy": 0.151992, + "f1": 0.053909, + "f1_weighted": 0.074936 + }, + { + "accuracy": 0.035908, + "f1": 0.024547, + "f1_weighted": 0.018938 + }, + { + "accuracy": 0.066896, + "f1": 0.027158, + "f1_weighted": 0.030502 + } + ], + "main_score": 0.063207, + "hf_subset": "fa", + "languages": [ + "fas-Arab" + ] + }, + { + "accuracy": 0.20664, + "f1": 0.184312, + "f1_weighted": 0.19312, + "scores_per_experiment": [ + { + "accuracy": 0.230202, + "f1": 0.197805, + "f1_weighted": 0.21446 + }, + { + "accuracy": 0.196262, + "f1": 0.168414, + "f1_weighted": 0.183582 + }, + { + "accuracy": 0.242007, + "f1": 0.213655, + "f1_weighted": 0.241131 + }, + { + "accuracy": 0.243974, + "f1": 0.205325, + "f1_weighted": 0.228711 + }, + { + "accuracy": 0.217413, + "f1": 0.199761, + "f1_weighted": 0.199117 + }, + { + "accuracy": 0.192818, + "f1": 0.182378, + "f1_weighted": 0.170267 + }, + { + "accuracy": 0.194786, + "f1": 0.184794, + "f1_weighted": 0.194407 + }, + { + "accuracy": 0.188883, + "f1": 0.169868, + "f1_weighted": 0.178815 + }, + { + "accuracy": 0.191835, + "f1": 0.16733, + "f1_weighted": 0.174184 + }, + { + "accuracy": 0.168224, + "f1": 0.153792, + "f1_weighted": 0.146523 + } + ], + "main_score": 0.20664, + "hf_subset": "mn", + "languages": [ + "mon-Cyrl" + ] + }, + { + "accuracy": 0.087998, + "f1": 0.048584, + "f1_weighted": 0.046856, + "scores_per_experiment": [ + { + "accuracy": 0.137727, + "f1": 0.058997, + "f1_weighted": 0.08359 + }, + { + "accuracy": 0.093458, + "f1": 0.044904, + "f1_weighted": 0.037735 + }, + { + "accuracy": 0.062469, + "f1": 0.048729, + "f1_weighted": 0.037097 + }, + { + "accuracy": 0.078701, + "f1": 0.043122, + "f1_weighted": 0.038257 + }, + { + "accuracy": 0.04181, + "f1": 0.040133, + "f1_weighted": 0.024417 + }, + { + "accuracy": 0.062961, + "f1": 0.055936, + "f1_weighted": 0.048247 + }, + { + "accuracy": 0.095425, + "f1": 0.051422, + "f1_weighted": 0.047465 + }, + { + "accuracy": 0.148057, + "f1": 0.060109, + "f1_weighted": 0.073373 + }, + { + "accuracy": 0.073291, + "f1": 0.038721, + "f1_weighted": 0.039764 + }, + { + "accuracy": 0.08608, + "f1": 0.043769, + "f1_weighted": 0.038611 + } + ], + "main_score": 0.087998, + "hf_subset": "kn", + "languages": [ + "kan-Knda" + ] + }, + { + "accuracy": 0.087703, + "f1": 0.041108, + "f1_weighted": 0.04537, + "scores_per_experiment": [ + { + "accuracy": 0.115593, + "f1": 0.046984, + "f1_weighted": 0.058968 + }, + { + "accuracy": 0.058042, + "f1": 0.033739, + "f1_weighted": 0.037282 + }, + { + "accuracy": 0.068372, + "f1": 0.027631, + "f1_weighted": 0.026008 + }, + { + "accuracy": 0.04968, + "f1": 0.039243, + "f1_weighted": 0.025767 + }, + { + "accuracy": 0.068864, + "f1": 0.039383, + "f1_weighted": 0.032487 + }, + { + "accuracy": 0.074766, + "f1": 0.042034, + "f1_weighted": 0.037265 + }, + { + "accuracy": 0.114609, + "f1": 0.049533, + "f1_weighted": 0.081049 + }, + { + "accuracy": 0.141171, + "f1": 0.046311, + "f1_weighted": 0.062189 + }, + { + "accuracy": 0.116576, + "f1": 0.04803, + "f1_weighted": 0.063113 + }, + { + "accuracy": 0.069356, + "f1": 0.038194, + "f1_weighted": 0.02957 + } + ], + "main_score": 0.087703, + "hf_subset": "hy", + "languages": [ + "hye-Armn" + ] + }, + { + "accuracy": 0.114412, + "f1": 0.067793, + "f1_weighted": 0.092004, + "scores_per_experiment": [ + { + "accuracy": 0.100836, + "f1": 0.052993, + "f1_weighted": 0.073429 + }, + { + "accuracy": 0.10182, + "f1": 0.054303, + "f1_weighted": 0.068077 + }, + { + "accuracy": 0.10969, + "f1": 0.072433, + "f1_weighted": 0.102507 + }, + { + "accuracy": 0.13576, + "f1": 0.088745, + "f1_weighted": 0.116523 + }, + { + "accuracy": 0.08362, + "f1": 0.064319, + "f1_weighted": 0.066581 + }, + { + "accuracy": 0.110674, + "f1": 0.06956, + "f1_weighted": 0.112966 + }, + { + "accuracy": 0.105755, + "f1": 0.065786, + "f1_weighted": 0.094119 + }, + { + "accuracy": 0.188883, + "f1": 0.090772, + "f1_weighted": 0.148739 + }, + { + "accuracy": 0.137236, + "f1": 0.074593, + "f1_weighted": 0.088887 + }, + { + "accuracy": 0.069848, + "f1": 0.044429, + "f1_weighted": 0.048212 + } + ], + "main_score": 0.114412, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ] + }, + { + "accuracy": 0.537137, + "f1": 0.522901, + "f1_weighted": 0.523395, + "scores_per_experiment": [ + { + "accuracy": 0.540089, + "f1": 0.531947, + "f1_weighted": 0.529733 + }, + { + "accuracy": 0.528775, + "f1": 0.515747, + "f1_weighted": 0.50648 + }, + { + "accuracy": 0.5273, + "f1": 0.518635, + "f1_weighted": 0.519569 + }, + { + "accuracy": 0.555337, + "f1": 0.541926, + "f1_weighted": 0.546963 + }, + { + "accuracy": 0.549926, + "f1": 0.52877, + "f1_weighted": 0.532426 + }, + { + "accuracy": 0.537629, + "f1": 0.524697, + "f1_weighted": 0.528316 + }, + { + "accuracy": 0.529759, + "f1": 0.504996, + "f1_weighted": 0.511099 + }, + { + "accuracy": 0.546975, + "f1": 0.532333, + "f1_weighted": 0.534113 + }, + { + "accuracy": 0.539105, + "f1": 0.525075, + "f1_weighted": 0.523344 + }, + { + "accuracy": 0.516478, + "f1": 0.504884, + "f1_weighted": 0.501906 + } + ], + "main_score": 0.537137, + "hf_subset": "ro", + "languages": [ + "ron-Latn" + ] + }, + { + "accuracy": 0.462814, + "f1": 0.43758, + "f1_weighted": 0.457154, + "scores_per_experiment": [ + { + "accuracy": 0.464338, + "f1": 0.432292, + "f1_weighted": 0.46378 + }, + { + "accuracy": 0.442204, + "f1": 0.423467, + "f1_weighted": 0.434918 + }, + { + "accuracy": 0.477127, + "f1": 0.45872, + "f1_weighted": 0.473714 + }, + { + "accuracy": 0.481554, + "f1": 0.448863, + "f1_weighted": 0.483285 + }, + { + "accuracy": 0.476144, + "f1": 0.443943, + "f1_weighted": 0.468349 + }, + { + "accuracy": 0.461879, + "f1": 0.438649, + "f1_weighted": 0.456183 + }, + { + "accuracy": 0.474668, + "f1": 0.443887, + "f1_weighted": 0.472086 + }, + { + "accuracy": 0.492868, + "f1": 0.472997, + "f1_weighted": 0.489954 + }, + { + "accuracy": 0.407772, + "f1": 0.390705, + "f1_weighted": 0.389996 + }, + { + "accuracy": 0.449582, + "f1": 0.42228, + "f1_weighted": 0.439271 + } + ], + "main_score": 0.462814, + "hf_subset": "nb", + "languages": [ + "nob-Latn" + ] + }, + { + "accuracy": 0.45819, + "f1": 0.429273, + "f1_weighted": 0.449742, + "scores_per_experiment": [ + { + "accuracy": 0.470241, + "f1": 0.444127, + "f1_weighted": 0.458677 + }, + { + "accuracy": 0.445155, + "f1": 0.412701, + "f1_weighted": 0.44009 + }, + { + "accuracy": 0.471717, + "f1": 0.440036, + "f1_weighted": 0.471412 + }, + { + "accuracy": 0.455485, + "f1": 0.411951, + "f1_weighted": 0.450066 + }, + { + "accuracy": 0.452041, + "f1": 0.418168, + "f1_weighted": 0.438831 + }, + { + "accuracy": 0.460895, + "f1": 0.432581, + "f1_weighted": 0.449282 + }, + { + "accuracy": 0.474176, + "f1": 0.437233, + "f1_weighted": 0.462475 + }, + { + "accuracy": 0.487949, + "f1": 0.462863, + "f1_weighted": 0.482756 + }, + { + "accuracy": 0.424496, + "f1": 0.410607, + "f1_weighted": 0.407986 + }, + { + "accuracy": 0.439744, + "f1": 0.422459, + "f1_weighted": 0.43585 + } + ], + "main_score": 0.45819, + "hf_subset": "sv", + "languages": [ + "swe-Latn" + ] + }, + { + "accuracy": 0.427004, + "f1": 0.405258, + "f1_weighted": 0.423375, + "scores_per_experiment": [ + { + "accuracy": 0.40482, + "f1": 0.397091, + "f1_weighted": 0.405925 + }, + { + "accuracy": 0.420561, + "f1": 0.398264, + "f1_weighted": 0.412832 + }, + { + "accuracy": 0.429415, + "f1": 0.404385, + "f1_weighted": 0.422136 + }, + { + "accuracy": 0.410723, + "f1": 0.397139, + "f1_weighted": 0.409382 + }, + { + "accuracy": 0.443187, + "f1": 0.406944, + "f1_weighted": 0.437498 + }, + { + "accuracy": 0.450566, + "f1": 0.417995, + "f1_weighted": 0.451389 + }, + { + "accuracy": 0.439252, + "f1": 0.403015, + "f1_weighted": 0.431867 + }, + { + "accuracy": 0.434333, + "f1": 0.425129, + "f1_weighted": 0.438081 + }, + { + "accuracy": 0.435317, + "f1": 0.413677, + "f1_weighted": 0.424565 + }, + { + "accuracy": 0.401869, + "f1": 0.388942, + "f1_weighted": 0.400071 + } + ], + "main_score": 0.427004, + "hf_subset": "pl", + "languages": [ + "pol-Latn" + ] + }, + { + "accuracy": 0.099951, + "f1": 0.03991, + "f1_weighted": 0.048796, + "scores_per_experiment": [ + { + "accuracy": 0.067388, + "f1": 0.019486, + "f1_weighted": 0.017951 + }, + { + "accuracy": 0.088539, + "f1": 0.033158, + "f1_weighted": 0.040291 + }, + { + "accuracy": 0.087555, + "f1": 0.033532, + "f1_weighted": 0.041958 + }, + { + "accuracy": 0.090507, + "f1": 0.040843, + "f1_weighted": 0.050048 + }, + { + "accuracy": 0.096901, + "f1": 0.055393, + "f1_weighted": 0.060113 + }, + { + "accuracy": 0.094934, + "f1": 0.039009, + "f1_weighted": 0.047954 + }, + { + "accuracy": 0.090507, + "f1": 0.042984, + "f1_weighted": 0.052542 + }, + { + "accuracy": 0.09149, + "f1": 0.028486, + "f1_weighted": 0.036883 + }, + { + "accuracy": 0.141171, + "f1": 0.057483, + "f1_weighted": 0.070985 + }, + { + "accuracy": 0.150516, + "f1": 0.048723, + "f1_weighted": 0.069239 + } + ], + "main_score": 0.099951, + "hf_subset": "zh-TW", + "languages": [ + "cmo-Hant" + ] + }, + { + "accuracy": 0.473242, + "f1": 0.453518, + "f1_weighted": 0.457801, + "scores_per_experiment": [ + { + "accuracy": 0.461387, + "f1": 0.446705, + "f1_weighted": 0.438362 + }, + { + "accuracy": 0.47516, + "f1": 0.467256, + "f1_weighted": 0.462574 + }, + { + "accuracy": 0.513035, + "f1": 0.49091, + "f1_weighted": 0.501862 + }, + { + "accuracy": 0.451549, + "f1": 0.432011, + "f1_weighted": 0.435138 + }, + { + "accuracy": 0.497787, + "f1": 0.460235, + "f1_weighted": 0.486644 + }, + { + "accuracy": 0.484506, + "f1": 0.455688, + "f1_weighted": 0.466796 + }, + { + "accuracy": 0.489916, + "f1": 0.470056, + "f1_weighted": 0.479401 + }, + { + "accuracy": 0.454993, + "f1": 0.437741, + "f1_weighted": 0.439753 + }, + { + "accuracy": 0.439252, + "f1": 0.430235, + "f1_weighted": 0.414907 + }, + { + "accuracy": 0.46483, + "f1": 0.444348, + "f1_weighted": 0.452569 + } + ], + "main_score": 0.473242, + "hf_subset": "tl", + "languages": [ + "tgl-Latn" + ] + }, + { + "accuracy": 0.077324, + "f1": 0.028655, + "f1_weighted": 0.036845, + "scores_per_experiment": [ + { + "accuracy": 0.121495, + "f1": 0.041375, + "f1_weighted": 0.070575 + }, + { + "accuracy": 0.066896, + "f1": 0.020436, + "f1_weighted": 0.020439 + }, + { + "accuracy": 0.055583, + "f1": 0.01949, + "f1_weighted": 0.026789 + }, + { + "accuracy": 0.072307, + "f1": 0.030645, + "f1_weighted": 0.025015 + }, + { + "accuracy": 0.060502, + "f1": 0.026877, + "f1_weighted": 0.027857 + }, + { + "accuracy": 0.088539, + "f1": 0.027571, + "f1_weighted": 0.036983 + }, + { + "accuracy": 0.074766, + "f1": 0.033126, + "f1_weighted": 0.046395 + }, + { + "accuracy": 0.139203, + "f1": 0.052042, + "f1_weighted": 0.087155 + }, + { + "accuracy": 0.032464, + "f1": 0.011641, + "f1_weighted": 0.009698 + }, + { + "accuracy": 0.061485, + "f1": 0.023347, + "f1_weighted": 0.017543 + } + ], + "main_score": 0.077324, + "hf_subset": "ml", + "languages": [ + "mal-Mlym" + ] + }, + { + "accuracy": 0.525086, + "f1": 0.51364, + "f1_weighted": 0.511988, + "scores_per_experiment": [ + { + "accuracy": 0.520413, + "f1": 0.51029, + "f1_weighted": 0.502167 + }, + { + "accuracy": 0.551894, + "f1": 0.527289, + "f1_weighted": 0.543168 + }, + { + "accuracy": 0.546483, + "f1": 0.527691, + "f1_weighted": 0.533987 + }, + { + "accuracy": 0.525824, + "f1": 0.518794, + "f1_weighted": 0.514332 + }, + { + "accuracy": 0.518938, + "f1": 0.512698, + "f1_weighted": 0.511946 + }, + { + "accuracy": 0.523856, + "f1": 0.518763, + "f1_weighted": 0.5126 + }, + { + "accuracy": 0.49877, + "f1": 0.487733, + "f1_weighted": 0.483612 + }, + { + "accuracy": 0.528775, + "f1": 0.514594, + "f1_weighted": 0.512631 + }, + { + "accuracy": 0.533694, + "f1": 0.527587, + "f1_weighted": 0.518406 + }, + { + "accuracy": 0.502213, + "f1": 0.490957, + "f1_weighted": 0.487031 + } + ], + "main_score": 0.525086, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ] + }, + { + "accuracy": 0.743925, + "f1": 0.732846, + "f1_weighted": 0.740496, + "scores_per_experiment": [ + { + "accuracy": 0.750615, + "f1": 0.739999, + "f1_weighted": 0.749062 + }, + { + "accuracy": 0.750123, + "f1": 0.74775, + "f1_weighted": 0.746792 + }, + { + "accuracy": 0.759961, + "f1": 0.74573, + "f1_weighted": 0.754927 + }, + { + "accuracy": 0.737826, + "f1": 0.719911, + "f1_weighted": 0.731973 + }, + { + "accuracy": 0.748647, + "f1": 0.732473, + "f1_weighted": 0.74604 + }, + { + "accuracy": 0.725529, + "f1": 0.719822, + "f1_weighted": 0.719673 + }, + { + "accuracy": 0.739793, + "f1": 0.726882, + "f1_weighted": 0.736018 + }, + { + "accuracy": 0.743237, + "f1": 0.731017, + "f1_weighted": 0.745161 + }, + { + "accuracy": 0.757993, + "f1": 0.745901, + "f1_weighted": 0.752086 + }, + { + "accuracy": 0.725529, + "f1": 0.71897, + "f1_weighted": 0.72323 + } + ], + "main_score": 0.743925, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.392917, + "f1": 0.3664, + "f1_weighted": 0.384322, + "scores_per_experiment": [ + { + "accuracy": 0.396458, + "f1": 0.376505, + "f1_weighted": 0.3873 + }, + { + "accuracy": 0.385637, + "f1": 0.357069, + "f1_weighted": 0.377876 + }, + { + "accuracy": 0.424004, + "f1": 0.392957, + "f1_weighted": 0.410796 + }, + { + "accuracy": 0.377767, + "f1": 0.352461, + "f1_weighted": 0.370089 + }, + { + "accuracy": 0.406788, + "f1": 0.374434, + "f1_weighted": 0.394953 + }, + { + "accuracy": 0.37334, + "f1": 0.344731, + "f1_weighted": 0.358105 + }, + { + "accuracy": 0.414658, + "f1": 0.383262, + "f1_weighted": 0.41337 + }, + { + "accuracy": 0.394491, + "f1": 0.376906, + "f1_weighted": 0.394615 + }, + { + "accuracy": 0.376783, + "f1": 0.349902, + "f1_weighted": 0.369206 + }, + { + "accuracy": 0.379242, + "f1": 0.355776, + "f1_weighted": 0.366913 + } + ], + "main_score": 0.392917, + "hf_subset": "cy", + "languages": [ + "cym-Latn" + ] + }, + { + "accuracy": 0.577668, + "f1": 0.554231, + "f1_weighted": 0.57184, + "scores_per_experiment": [ + { + "accuracy": 0.594688, + "f1": 0.567337, + "f1_weighted": 0.588465 + }, + { + "accuracy": 0.59518, + "f1": 0.576696, + "f1_weighted": 0.593716 + }, + { + "accuracy": 0.580423, + "f1": 0.5564, + "f1_weighted": 0.569852 + }, + { + "accuracy": 0.568618, + "f1": 0.541723, + "f1_weighted": 0.567443 + }, + { + "accuracy": 0.583866, + "f1": 0.555098, + "f1_weighted": 0.573509 + }, + { + "accuracy": 0.566158, + "f1": 0.54315, + "f1_weighted": 0.554601 + }, + { + "accuracy": 0.568126, + "f1": 0.549671, + "f1_weighted": 0.568971 + }, + { + "accuracy": 0.580915, + "f1": 0.570067, + "f1_weighted": 0.579196 + }, + { + "accuracy": 0.57698, + "f1": 0.552301, + "f1_weighted": 0.568833 + }, + { + "accuracy": 0.561731, + "f1": 0.529864, + "f1_weighted": 0.553813 + } + ], + "main_score": 0.577668, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ] + }, + { + "accuracy": 0.391687, + "f1": 0.373728, + "f1_weighted": 0.380842, + "scores_per_experiment": [ + { + "accuracy": 0.382194, + "f1": 0.369117, + "f1_weighted": 0.373412 + }, + { + "accuracy": 0.385637, + "f1": 0.364994, + "f1_weighted": 0.382973 + }, + { + "accuracy": 0.410723, + "f1": 0.376152, + "f1_weighted": 0.406321 + }, + { + "accuracy": 0.385637, + "f1": 0.371764, + "f1_weighted": 0.379466 + }, + { + "accuracy": 0.38908, + "f1": 0.357951, + "f1_weighted": 0.353202 + }, + { + "accuracy": 0.391048, + "f1": 0.378157, + "f1_weighted": 0.38343 + }, + { + "accuracy": 0.383178, + "f1": 0.377619, + "f1_weighted": 0.384619 + }, + { + "accuracy": 0.426463, + "f1": 0.406693, + "f1_weighted": 0.414319 + }, + { + "accuracy": 0.334973, + "f1": 0.328716, + "f1_weighted": 0.31839 + }, + { + "accuracy": 0.427939, + "f1": 0.406122, + "f1_weighted": 0.412287 + } + ], + "main_score": 0.391687, + "hf_subset": "jv", + "languages": [ + "jav-Latn" + ] + }, + { + "accuracy": 0.45937, + "f1": 0.437887, + "f1_weighted": 0.4577, + "scores_per_experiment": [ + { + "accuracy": 0.458928, + "f1": 0.436156, + "f1_weighted": 0.459459 + }, + { + "accuracy": 0.476144, + "f1": 0.44635, + "f1_weighted": 0.479808 + }, + { + "accuracy": 0.476144, + "f1": 0.456506, + "f1_weighted": 0.471681 + }, + { + "accuracy": 0.443679, + "f1": 0.429238, + "f1_weighted": 0.446247 + }, + { + "accuracy": 0.468273, + "f1": 0.42823, + "f1_weighted": 0.450626 + }, + { + "accuracy": 0.453517, + "f1": 0.438571, + "f1_weighted": 0.455173 + }, + { + "accuracy": 0.473192, + "f1": 0.449593, + "f1_weighted": 0.473093 + }, + { + "accuracy": 0.466798, + "f1": 0.452749, + "f1_weighted": 0.46526 + }, + { + "accuracy": 0.43335, + "f1": 0.412603, + "f1_weighted": 0.432334 + }, + { + "accuracy": 0.443679, + "f1": 0.428871, + "f1_weighted": 0.44332 + } + ], + "main_score": 0.45937, + "hf_subset": "af", + "languages": [ + "afr-Latn" + ] + }, + { + "accuracy": 0.085883, + "f1": 0.034132, + "f1_weighted": 0.04267, + "scores_per_experiment": [ + { + "accuracy": 0.133792, + "f1": 0.037955, + "f1_weighted": 0.068181 + }, + { + "accuracy": 0.1515, + "f1": 0.044479, + "f1_weighted": 0.075961 + }, + { + "accuracy": 0.045745, + "f1": 0.029386, + "f1_weighted": 0.027261 + }, + { + "accuracy": 0.057059, + "f1": 0.029727, + "f1_weighted": 0.035161 + }, + { + "accuracy": 0.162814, + "f1": 0.049251, + "f1_weighted": 0.083394 + }, + { + "accuracy": 0.099361, + "f1": 0.042367, + "f1_weighted": 0.046778 + }, + { + "accuracy": 0.055091, + "f1": 0.025198, + "f1_weighted": 0.022735 + }, + { + "accuracy": 0.069356, + "f1": 0.036742, + "f1_weighted": 0.030538 + }, + { + "accuracy": 0.035908, + "f1": 0.023536, + "f1_weighted": 0.016122 + }, + { + "accuracy": 0.048205, + "f1": 0.022679, + "f1_weighted": 0.020572 + } + ], + "main_score": 0.085883, + "hf_subset": "bn", + "languages": [ + "ben-Beng" + ] + }, + { + "accuracy": 0.615347, + "f1": 0.5982, + "f1_weighted": 0.609261, + "scores_per_experiment": [ + { + "accuracy": 0.628136, + "f1": 0.617736, + "f1_weighted": 0.620312 + }, + { + "accuracy": 0.627152, + "f1": 0.611763, + "f1_weighted": 0.625528 + }, + { + "accuracy": 0.638465, + "f1": 0.613328, + "f1_weighted": 0.627538 + }, + { + "accuracy": 0.619774, + "f1": 0.594123, + "f1_weighted": 0.618873 + }, + { + "accuracy": 0.609936, + "f1": 0.597076, + "f1_weighted": 0.598497 + }, + { + "accuracy": 0.597639, + "f1": 0.587352, + "f1_weighted": 0.595245 + }, + { + "accuracy": 0.612395, + "f1": 0.583572, + "f1_weighted": 0.610942 + }, + { + "accuracy": 0.626168, + "f1": 0.610964, + "f1_weighted": 0.628408 + }, + { + "accuracy": 0.587801, + "f1": 0.57925, + "f1_weighted": 0.57082 + }, + { + "accuracy": 0.606001, + "f1": 0.586832, + "f1_weighted": 0.596447 + } + ], + "main_score": 0.615347, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ] + }, + { + "accuracy": 0.098082, + "f1": 0.047025, + "f1_weighted": 0.051061, + "scores_per_experiment": [ + { + "accuracy": 0.134776, + "f1": 0.048162, + "f1_weighted": 0.060223 + }, + { + "accuracy": 0.098869, + "f1": 0.043543, + "f1_weighted": 0.037276 + }, + { + "accuracy": 0.091982, + "f1": 0.044166, + "f1_weighted": 0.04525 + }, + { + "accuracy": 0.063945, + "f1": 0.02723, + "f1_weighted": 0.033832 + }, + { + "accuracy": 0.103296, + "f1": 0.055347, + "f1_weighted": 0.05236 + }, + { + "accuracy": 0.071815, + "f1": 0.039588, + "f1_weighted": 0.027915 + }, + { + "accuracy": 0.11756, + "f1": 0.062153, + "f1_weighted": 0.093863 + }, + { + "accuracy": 0.098377, + "f1": 0.046247, + "f1_weighted": 0.039969 + }, + { + "accuracy": 0.098377, + "f1": 0.052443, + "f1_weighted": 0.073428 + }, + { + "accuracy": 0.10182, + "f1": 0.051369, + "f1_weighted": 0.046492 + } + ], + "main_score": 0.098082, + "hf_subset": "my", + "languages": [ + "mya-Mymr" + ] + }, + { + "accuracy": 0.500295, + "f1": 0.473817, + "f1_weighted": 0.49647, + "scores_per_experiment": [ + { + "accuracy": 0.510084, + "f1": 0.479751, + "f1_weighted": 0.507935 + }, + { + "accuracy": 0.511559, + "f1": 0.477473, + "f1_weighted": 0.505008 + }, + { + "accuracy": 0.508116, + "f1": 0.473659, + "f1_weighted": 0.500712 + }, + { + "accuracy": 0.512051, + "f1": 0.4821, + "f1_weighted": 0.507128 + }, + { + "accuracy": 0.508116, + "f1": 0.475571, + "f1_weighted": 0.50452 + }, + { + "accuracy": 0.496311, + "f1": 0.47558, + "f1_weighted": 0.489415 + }, + { + "accuracy": 0.500246, + "f1": 0.470593, + "f1_weighted": 0.495094 + }, + { + "accuracy": 0.479587, + "f1": 0.464359, + "f1_weighted": 0.476506 + }, + { + "accuracy": 0.500246, + "f1": 0.481545, + "f1_weighted": 0.499044 + }, + { + "accuracy": 0.476636, + "f1": 0.457535, + "f1_weighted": 0.479342 + } + ], + "main_score": 0.500295, + "hf_subset": "nl", + "languages": [ + "nld-Latn" + ] + }, + { + "accuracy": 0.388982, + "f1": 0.371749, + "f1_weighted": 0.379098, + "scores_per_experiment": [ + { + "accuracy": 0.395475, + "f1": 0.381563, + "f1_weighted": 0.38218 + }, + { + "accuracy": 0.373832, + "f1": 0.357393, + "f1_weighted": 0.369346 + }, + { + "accuracy": 0.38908, + "f1": 0.372926, + "f1_weighted": 0.387124 + }, + { + "accuracy": 0.388096, + "f1": 0.371028, + "f1_weighted": 0.380375 + }, + { + "accuracy": 0.386129, + "f1": 0.362615, + "f1_weighted": 0.378375 + }, + { + "accuracy": 0.40728, + "f1": 0.389282, + "f1_weighted": 0.403083 + }, + { + "accuracy": 0.375799, + "f1": 0.365823, + "f1_weighted": 0.363691 + }, + { + "accuracy": 0.39941, + "f1": 0.383238, + "f1_weighted": 0.392055 + }, + { + "accuracy": 0.382686, + "f1": 0.358991, + "f1_weighted": 0.361514 + }, + { + "accuracy": 0.392031, + "f1": 0.37463, + "f1_weighted": 0.373238 + } + ], + "main_score": 0.388982, + "hf_subset": "hu", + "languages": [ + "hun-Latn" + ] + }, + { + "accuracy": 0.400984, + "f1": 0.376105, + "f1_weighted": 0.392515, + "scores_per_experiment": [ + { + "accuracy": 0.394983, + "f1": 0.37507, + "f1_weighted": 0.382245 + }, + { + "accuracy": 0.395967, + "f1": 0.368583, + "f1_weighted": 0.387781 + }, + { + "accuracy": 0.435317, + "f1": 0.399405, + "f1_weighted": 0.418941 + }, + { + "accuracy": 0.371864, + "f1": 0.348452, + "f1_weighted": 0.376408 + }, + { + "accuracy": 0.422528, + "f1": 0.392676, + "f1_weighted": 0.41057 + }, + { + "accuracy": 0.362027, + "f1": 0.347722, + "f1_weighted": 0.347411 + }, + { + "accuracy": 0.43335, + "f1": 0.405921, + "f1_weighted": 0.433905 + }, + { + "accuracy": 0.417609, + "f1": 0.39344, + "f1_weighted": 0.407066 + }, + { + "accuracy": 0.360059, + "f1": 0.343969, + "f1_weighted": 0.349558 + }, + { + "accuracy": 0.416134, + "f1": 0.385809, + "f1_weighted": 0.411266 + } + ], + "main_score": 0.400984, + "hf_subset": "lv", + "languages": [ + "lav-Latn" + ] + }, + { + "accuracy": 0.493655, + "f1": 0.471355, + "f1_weighted": 0.484854, + "scores_per_experiment": [ + { + "accuracy": 0.484998, + "f1": 0.464416, + "f1_weighted": 0.480466 + }, + { + "accuracy": 0.489916, + "f1": 0.474018, + "f1_weighted": 0.476867 + }, + { + "accuracy": 0.496311, + "f1": 0.486258, + "f1_weighted": 0.499648 + }, + { + "accuracy": 0.495819, + "f1": 0.465019, + "f1_weighted": 0.48904 + }, + { + "accuracy": 0.507624, + "f1": 0.471775, + "f1_weighted": 0.488791 + }, + { + "accuracy": 0.477619, + "f1": 0.462561, + "f1_weighted": 0.466559 + }, + { + "accuracy": 0.498278, + "f1": 0.472426, + "f1_weighted": 0.491293 + }, + { + "accuracy": 0.515002, + "f1": 0.495528, + "f1_weighted": 0.50804 + }, + { + "accuracy": 0.488933, + "f1": 0.464956, + "f1_weighted": 0.477238 + }, + { + "accuracy": 0.482046, + "f1": 0.456591, + "f1_weighted": 0.470601 + } + ], + "main_score": 0.493655, + "hf_subset": "da", + "languages": [ + "dan-Latn" + ] + }, + { + "accuracy": 0.409739, + "f1": 0.381541, + "f1_weighted": 0.40516, + "scores_per_experiment": [ + { + "accuracy": 0.399902, + "f1": 0.378766, + "f1_weighted": 0.395853 + }, + { + "accuracy": 0.41515, + "f1": 0.386468, + "f1_weighted": 0.41764 + }, + { + "accuracy": 0.429907, + "f1": 0.393458, + "f1_weighted": 0.425511 + }, + { + "accuracy": 0.416134, + "f1": 0.38475, + "f1_weighted": 0.412042 + }, + { + "accuracy": 0.398918, + "f1": 0.380202, + "f1_weighted": 0.391103 + }, + { + "accuracy": 0.414658, + "f1": 0.378603, + "f1_weighted": 0.409058 + }, + { + "accuracy": 0.429907, + "f1": 0.398054, + "f1_weighted": 0.416503 + }, + { + "accuracy": 0.427447, + "f1": 0.408771, + "f1_weighted": 0.437576 + }, + { + "accuracy": 0.385637, + "f1": 0.353008, + "f1_weighted": 0.37317 + }, + { + "accuracy": 0.379734, + "f1": 0.353327, + "f1_weighted": 0.373147 + } + ], + "main_score": 0.409739, + "hf_subset": "is", + "languages": [ + "isl-Latn" + ] + }, + { + "accuracy": 0.155239, + "f1": 0.114939, + "f1_weighted": 0.120863, + "scores_per_experiment": [ + { + "accuracy": 0.194786, + "f1": 0.136565, + "f1_weighted": 0.16303 + }, + { + "accuracy": 0.126906, + "f1": 0.097406, + "f1_weighted": 0.091729 + }, + { + "accuracy": 0.147073, + "f1": 0.111389, + "f1_weighted": 0.107253 + }, + { + "accuracy": 0.137727, + "f1": 0.111269, + "f1_weighted": 0.106891 + }, + { + "accuracy": 0.136744, + "f1": 0.114193, + "f1_weighted": 0.109818 + }, + { + "accuracy": 0.162814, + "f1": 0.122065, + "f1_weighted": 0.116245 + }, + { + "accuracy": 0.12002, + "f1": 0.084223, + "f1_weighted": 0.088883 + }, + { + "accuracy": 0.21397, + "f1": 0.124389, + "f1_weighted": 0.158798 + }, + { + "accuracy": 0.15937, + "f1": 0.128116, + "f1_weighted": 0.13372 + }, + { + "accuracy": 0.152976, + "f1": 0.119771, + "f1_weighted": 0.132267 + } + ], + "main_score": 0.155239, + "hf_subset": "el", + "languages": [ + "ell-Grek" + ] + }, + { + "accuracy": 0.406542, + "f1": 0.392922, + "f1_weighted": 0.393011, + "scores_per_experiment": [ + { + "accuracy": 0.405804, + "f1": 0.399539, + "f1_weighted": 0.384735 + }, + { + "accuracy": 0.405804, + "f1": 0.388745, + "f1_weighted": 0.396304 + }, + { + "accuracy": 0.428923, + "f1": 0.405644, + "f1_weighted": 0.418801 + }, + { + "accuracy": 0.381702, + "f1": 0.366541, + "f1_weighted": 0.365235 + }, + { + "accuracy": 0.416134, + "f1": 0.398953, + "f1_weighted": 0.398751 + }, + { + "accuracy": 0.363502, + "f1": 0.34979, + "f1_weighted": 0.327018 + }, + { + "accuracy": 0.43089, + "f1": 0.413029, + "f1_weighted": 0.422078 + }, + { + "accuracy": 0.42548, + "f1": 0.41332, + "f1_weighted": 0.420863 + }, + { + "accuracy": 0.39695, + "f1": 0.397423, + "f1_weighted": 0.392387 + }, + { + "accuracy": 0.410231, + "f1": 0.396234, + "f1_weighted": 0.403942 + } + ], + "main_score": 0.406542, + "hf_subset": "sl", + "languages": [ + "slv-Latn" + ] + }, + { + "accuracy": 0.38303, + "f1": 0.362693, + "f1_weighted": 0.371382, + "scores_per_experiment": [ + { + "accuracy": 0.39695, + "f1": 0.370485, + "f1_weighted": 0.384075 + }, + { + "accuracy": 0.38121, + "f1": 0.371143, + "f1_weighted": 0.370926 + }, + { + "accuracy": 0.393507, + "f1": 0.364153, + "f1_weighted": 0.377605 + }, + { + "accuracy": 0.387113, + "f1": 0.356012, + "f1_weighted": 0.379142 + }, + { + "accuracy": 0.38121, + "f1": 0.353216, + "f1_weighted": 0.373789 + }, + { + "accuracy": 0.394491, + "f1": 0.367053, + "f1_weighted": 0.37439 + }, + { + "accuracy": 0.39154, + "f1": 0.37508, + "f1_weighted": 0.381751 + }, + { + "accuracy": 0.380226, + "f1": 0.363737, + "f1_weighted": 0.37112 + }, + { + "accuracy": 0.342843, + "f1": 0.332276, + "f1_weighted": 0.323616 + }, + { + "accuracy": 0.38121, + "f1": 0.373778, + "f1_weighted": 0.377403 + } + ], + "main_score": 0.38303, + "hf_subset": "az", + "languages": [ + "aze-Latn" + ] + }, + { + "accuracy": 0.428628, + "f1": 0.395952, + "f1_weighted": 0.412981, + "scores_per_experiment": [ + { + "accuracy": 0.440236, + "f1": 0.413964, + "f1_weighted": 0.437659 + }, + { + "accuracy": 0.424004, + "f1": 0.393859, + "f1_weighted": 0.408422 + }, + { + "accuracy": 0.447122, + "f1": 0.410096, + "f1_weighted": 0.434295 + }, + { + "accuracy": 0.431874, + "f1": 0.386905, + "f1_weighted": 0.414581 + }, + { + "accuracy": 0.422528, + "f1": 0.385136, + "f1_weighted": 0.398207 + }, + { + "accuracy": 0.418593, + "f1": 0.372894, + "f1_weighted": 0.393903 + }, + { + "accuracy": 0.446631, + "f1": 0.415555, + "f1_weighted": 0.432322 + }, + { + "accuracy": 0.454993, + "f1": 0.421503, + "f1_weighted": 0.440764 + }, + { + "accuracy": 0.387605, + "f1": 0.370506, + "f1_weighted": 0.373324 + }, + { + "accuracy": 0.412691, + "f1": 0.389099, + "f1_weighted": 0.396333 + } + ], + "main_score": 0.428628, + "hf_subset": "sw", + "languages": [ + "swa-Latn" + ] + }, + { + "accuracy": 0.32912, + "f1": 0.308561, + "f1_weighted": 0.31548, + "scores_per_experiment": [ + { + "accuracy": 0.328578, + "f1": 0.306293, + "f1_weighted": 0.303784 + }, + { + "accuracy": 0.318249, + "f1": 0.300526, + "f1_weighted": 0.299817 + }, + { + "accuracy": 0.357108, + "f1": 0.329804, + "f1_weighted": 0.354196 + }, + { + "accuracy": 0.320216, + "f1": 0.290898, + "f1_weighted": 0.304342 + }, + { + "accuracy": 0.329562, + "f1": 0.313584, + "f1_weighted": 0.313752 + }, + { + "accuracy": 0.304968, + "f1": 0.290078, + "f1_weighted": 0.299545 + }, + { + "accuracy": 0.329562, + "f1": 0.315141, + "f1_weighted": 0.32263 + }, + { + "accuracy": 0.315298, + "f1": 0.303835, + "f1_weighted": 0.292836 + }, + { + "accuracy": 0.351205, + "f1": 0.322093, + "f1_weighted": 0.337643 + }, + { + "accuracy": 0.336449, + "f1": 0.313354, + "f1_weighted": 0.326252 + } + ], + "main_score": 0.32912, + "hf_subset": "vi", + "languages": [ + "vie-Latn" + ] + }, + { + "accuracy": 0.077373, + "f1": 0.022775, + "f1_weighted": 0.029558, + "scores_per_experiment": [ + { + "accuracy": 0.132809, + "f1": 0.027746, + "f1_weighted": 0.058497 + }, + { + "accuracy": 0.061977, + "f1": 0.019918, + "f1_weighted": 0.016684 + }, + { + "accuracy": 0.039351, + "f1": 0.025217, + "f1_weighted": 0.019856 + }, + { + "accuracy": 0.062469, + "f1": 0.018367, + "f1_weighted": 0.016805 + }, + { + "accuracy": 0.077226, + "f1": 0.020351, + "f1_weighted": 0.018587 + }, + { + "accuracy": 0.060502, + "f1": 0.01448, + "f1_weighted": 0.016685 + }, + { + "accuracy": 0.047221, + "f1": 0.017694, + "f1_weighted": 0.013698 + }, + { + "accuracy": 0.12789, + "f1": 0.028789, + "f1_weighted": 0.05187 + }, + { + "accuracy": 0.104771, + "f1": 0.031248, + "f1_weighted": 0.060039 + }, + { + "accuracy": 0.059518, + "f1": 0.023944, + "f1_weighted": 0.022858 + } + ], + "main_score": 0.077373, + "hf_subset": "ta", + "languages": [ + "tam-Taml" + ] + }, + { + "accuracy": 0.049877, + "f1": 0.009221, + "f1_weighted": 0.011705, + "scores_per_experiment": [ + { + "accuracy": 0.02607, + "f1": 0.008607, + "f1_weighted": 0.00807 + }, + { + "accuracy": 0.021151, + "f1": 0.007903, + "f1_weighted": 0.007588 + }, + { + "accuracy": 0.106247, + "f1": 0.014498, + "f1_weighted": 0.024195 + }, + { + "accuracy": 0.123955, + "f1": 0.01718, + "f1_weighted": 0.0333 + }, + { + "accuracy": 0.039843, + "f1": 0.007459, + "f1_weighted": 0.010713 + }, + { + "accuracy": 0.038859, + "f1": 0.008493, + "f1_weighted": 0.008514 + }, + { + "accuracy": 0.031481, + "f1": 0.003391, + "f1_weighted": 0.001922 + }, + { + "accuracy": 0.035908, + "f1": 0.007855, + "f1_weighted": 0.007024 + }, + { + "accuracy": 0.038367, + "f1": 0.008379, + "f1_weighted": 0.007458 + }, + { + "accuracy": 0.036891, + "f1": 0.008445, + "f1_weighted": 0.008267 + } + ], + "main_score": 0.049877, + "hf_subset": "zh-CN", + "languages": [ + "cmo-Hans" + ] + }, + { + "accuracy": 0.488342, + "f1": 0.452383, + "f1_weighted": 0.480631, + "scores_per_experiment": [ + { + "accuracy": 0.523364, + "f1": 0.484007, + "f1_weighted": 0.517764 + }, + { + "accuracy": 0.491884, + "f1": 0.458533, + "f1_weighted": 0.492595 + }, + { + "accuracy": 0.500246, + "f1": 0.458609, + "f1_weighted": 0.499079 + }, + { + "accuracy": 0.508116, + "f1": 0.465192, + "f1_weighted": 0.504042 + }, + { + "accuracy": 0.463847, + "f1": 0.427384, + "f1_weighted": 0.444125 + }, + { + "accuracy": 0.477619, + "f1": 0.441619, + "f1_weighted": 0.467428 + }, + { + "accuracy": 0.481062, + "f1": 0.437857, + "f1_weighted": 0.464701 + }, + { + "accuracy": 0.498278, + "f1": 0.48175, + "f1_weighted": 0.499181 + }, + { + "accuracy": 0.441712, + "f1": 0.415065, + "f1_weighted": 0.433996 + }, + { + "accuracy": 0.497295, + "f1": 0.45381, + "f1_weighted": 0.483399 + } + ], + "main_score": 0.488342, + "hf_subset": "sq", + "languages": [ + "sqi-Latn" + ] + }, + { + "accuracy": 0.084801, + "f1": 0.023475, + "f1_weighted": 0.035453, + "scores_per_experiment": [ + { + "accuracy": 0.110674, + "f1": 0.027196, + "f1_weighted": 0.048806 + }, + { + "accuracy": 0.040334, + "f1": 0.015851, + "f1_weighted": 0.015505 + }, + { + "accuracy": 0.041318, + "f1": 0.014933, + "f1_weighted": 0.017132 + }, + { + "accuracy": 0.066404, + "f1": 0.019838, + "f1_weighted": 0.019384 + }, + { + "accuracy": 0.079685, + "f1": 0.024754, + "f1_weighted": 0.03299 + }, + { + "accuracy": 0.055583, + "f1": 0.013704, + "f1_weighted": 0.016312 + }, + { + "accuracy": 0.138219, + "f1": 0.033138, + "f1_weighted": 0.066571 + }, + { + "accuracy": 0.134776, + "f1": 0.029477, + "f1_weighted": 0.055273 + }, + { + "accuracy": 0.111166, + "f1": 0.024822, + "f1_weighted": 0.046122 + }, + { + "accuracy": 0.069848, + "f1": 0.031039, + "f1_weighted": 0.036434 + } + ], + "main_score": 0.084801, + "hf_subset": "he", + "languages": [ + "heb-Hebr" + ] + }, + { + "accuracy": 0.057157, + "f1": 0.03179, + "f1_weighted": 0.023074, + "scores_per_experiment": [ + { + "accuracy": 0.049188, + "f1": 0.049965, + "f1_weighted": 0.038782 + }, + { + "accuracy": 0.05755, + "f1": 0.029101, + "f1_weighted": 0.021394 + }, + { + "accuracy": 0.067388, + "f1": 0.027797, + "f1_weighted": 0.01858 + }, + { + "accuracy": 0.079685, + "f1": 0.034438, + "f1_weighted": 0.026137 + }, + { + "accuracy": 0.038367, + "f1": 0.022974, + "f1_weighted": 0.015505 + }, + { + "accuracy": 0.045253, + "f1": 0.033288, + "f1_weighted": 0.025449 + }, + { + "accuracy": 0.036891, + "f1": 0.022262, + "f1_weighted": 0.012584 + }, + { + "accuracy": 0.079685, + "f1": 0.030278, + "f1_weighted": 0.023872 + }, + { + "accuracy": 0.042302, + "f1": 0.0265, + "f1_weighted": 0.01769 + }, + { + "accuracy": 0.075258, + "f1": 0.041296, + "f1_weighted": 0.03075 + } + ], + "main_score": 0.057157, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ] + }, + { + "accuracy": 0.065175, + "f1": 0.030341, + "f1_weighted": 0.032124, + "scores_per_experiment": [ + { + "accuracy": 0.131825, + "f1": 0.030764, + "f1_weighted": 0.048021 + }, + { + "accuracy": 0.06001, + "f1": 0.029026, + "f1_weighted": 0.028104 + }, + { + "accuracy": 0.071815, + "f1": 0.035775, + "f1_weighted": 0.034589 + }, + { + "accuracy": 0.061485, + "f1": 0.031604, + "f1_weighted": 0.032354 + }, + { + "accuracy": 0.030989, + "f1": 0.017813, + "f1_weighted": 0.009406 + }, + { + "accuracy": 0.05755, + "f1": 0.034463, + "f1_weighted": 0.038353 + }, + { + "accuracy": 0.066896, + "f1": 0.04717, + "f1_weighted": 0.059758 + }, + { + "accuracy": 0.071815, + "f1": 0.029289, + "f1_weighted": 0.029004 + }, + { + "accuracy": 0.043286, + "f1": 0.026619, + "f1_weighted": 0.024295 + }, + { + "accuracy": 0.056075, + "f1": 0.020883, + "f1_weighted": 0.017355 + } + ], + "main_score": 0.065175, + "hf_subset": "te", + "languages": [ + "tel-Telu" + ] + }, + { + "accuracy": 0.079242, + "f1": 0.038528, + "f1_weighted": 0.040676, + "scores_per_experiment": [ + { + "accuracy": 0.079193, + "f1": 0.048585, + "f1_weighted": 0.058967 + }, + { + "accuracy": 0.05755, + "f1": 0.038364, + "f1_weighted": 0.03017 + }, + { + "accuracy": 0.05755, + "f1": 0.035007, + "f1_weighted": 0.029752 + }, + { + "accuracy": 0.040826, + "f1": 0.0163, + "f1_weighted": 0.008271 + }, + { + "accuracy": 0.133301, + "f1": 0.049291, + "f1_weighted": 0.06477 + }, + { + "accuracy": 0.068372, + "f1": 0.031739, + "f1_weighted": 0.034374 + }, + { + "accuracy": 0.050172, + "f1": 0.036521, + "f1_weighted": 0.026232 + }, + { + "accuracy": 0.136744, + "f1": 0.055353, + "f1_weighted": 0.069251 + }, + { + "accuracy": 0.120512, + "f1": 0.049825, + "f1_weighted": 0.070741 + }, + { + "accuracy": 0.048205, + "f1": 0.024292, + "f1_weighted": 0.014233 + } + ], + "main_score": 0.079242, + "hf_subset": "hi", + "languages": [ + "hin-Deva" + ] + }, + { + "accuracy": 0.098328, + "f1": 0.052095, + "f1_weighted": 0.057511, + "scores_per_experiment": [ + { + "accuracy": 0.138219, + "f1": 0.048455, + "f1_weighted": 0.061988 + }, + { + "accuracy": 0.060502, + "f1": 0.040133, + "f1_weighted": 0.040671 + }, + { + "accuracy": 0.086572, + "f1": 0.054511, + "f1_weighted": 0.044387 + }, + { + "accuracy": 0.144122, + "f1": 0.059379, + "f1_weighted": 0.094926 + }, + { + "accuracy": 0.067388, + "f1": 0.044918, + "f1_weighted": 0.031775 + }, + { + "accuracy": 0.074766, + "f1": 0.048808, + "f1_weighted": 0.039014 + }, + { + "accuracy": 0.069848, + "f1": 0.050703, + "f1_weighted": 0.048358 + }, + { + "accuracy": 0.12002, + "f1": 0.074176, + "f1_weighted": 0.089986 + }, + { + "accuracy": 0.122479, + "f1": 0.049312, + "f1_weighted": 0.06099 + }, + { + "accuracy": 0.099361, + "f1": 0.050552, + "f1_weighted": 0.063014 + } + ], + "main_score": 0.098328, + "hf_subset": "ur", + "languages": [ + "urd-Arab" + ] + }, + { + "accuracy": 0.540777, + "f1": 0.526159, + "f1_weighted": 0.531621, + "scores_per_experiment": [ + { + "accuracy": 0.569602, + "f1": 0.554762, + "f1_weighted": 0.567059 + }, + { + "accuracy": 0.520413, + "f1": 0.513019, + "f1_weighted": 0.51312 + }, + { + "accuracy": 0.552386, + "f1": 0.526119, + "f1_weighted": 0.544143 + }, + { + "accuracy": 0.508608, + "f1": 0.499916, + "f1_weighted": 0.494005 + }, + { + "accuracy": 0.55878, + "f1": 0.54146, + "f1_weighted": 0.551841 + }, + { + "accuracy": 0.530743, + "f1": 0.518892, + "f1_weighted": 0.517368 + }, + { + "accuracy": 0.535662, + "f1": 0.519926, + "f1_weighted": 0.520335 + }, + { + "accuracy": 0.555829, + "f1": 0.542236, + "f1_weighted": 0.552378 + }, + { + "accuracy": 0.551894, + "f1": 0.543489, + "f1_weighted": 0.540033 + }, + { + "accuracy": 0.523856, + "f1": 0.501767, + "f1_weighted": 0.51593 + } + ], + "main_score": 0.540777, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ] + }, + { + "accuracy": 0.445401, + "f1": 0.434655, + "f1_weighted": 0.435694, + "scores_per_experiment": [ + { + "accuracy": 0.437777, + "f1": 0.421468, + "f1_weighted": 0.431789 + }, + { + "accuracy": 0.432366, + "f1": 0.427339, + "f1_weighted": 0.426435 + }, + { + "accuracy": 0.481062, + "f1": 0.461889, + "f1_weighted": 0.473875 + }, + { + "accuracy": 0.451058, + "f1": 0.443115, + "f1_weighted": 0.441931 + }, + { + "accuracy": 0.453517, + "f1": 0.444923, + "f1_weighted": 0.444781 + }, + { + "accuracy": 0.460403, + "f1": 0.44332, + "f1_weighted": 0.449307 + }, + { + "accuracy": 0.42548, + "f1": 0.419039, + "f1_weighted": 0.415602 + }, + { + "accuracy": 0.460403, + "f1": 0.452125, + "f1_weighted": 0.449393 + }, + { + "accuracy": 0.413674, + "f1": 0.423643, + "f1_weighted": 0.397899 + }, + { + "accuracy": 0.438269, + "f1": 0.409688, + "f1_weighted": 0.42593 + } + ], + "main_score": 0.445401, + "hf_subset": "id", + "languages": [ + "ind-Latn" + ] + }, + { + "accuracy": 0.081505, + "f1": 0.049829, + "f1_weighted": 0.044112, + "scores_per_experiment": [ + { + "accuracy": 0.050664, + "f1": 0.034306, + "f1_weighted": 0.020854 + }, + { + "accuracy": 0.112641, + "f1": 0.066508, + "f1_weighted": 0.071259 + }, + { + "accuracy": 0.063945, + "f1": 0.044096, + "f1_weighted": 0.0326 + }, + { + "accuracy": 0.114117, + "f1": 0.059157, + "f1_weighted": 0.065229 + }, + { + "accuracy": 0.111658, + "f1": 0.060013, + "f1_weighted": 0.055823 + }, + { + "accuracy": 0.067388, + "f1": 0.05611, + "f1_weighted": 0.045246 + }, + { + "accuracy": 0.076734, + "f1": 0.050994, + "f1_weighted": 0.040007 + }, + { + "accuracy": 0.081653, + "f1": 0.051025, + "f1_weighted": 0.042534 + }, + { + "accuracy": 0.054599, + "f1": 0.036058, + "f1_weighted": 0.02937 + }, + { + "accuracy": 0.081653, + "f1": 0.040025, + "f1_weighted": 0.038199 + } + ], + "main_score": 0.081505, + "hf_subset": "th", + "languages": [ + "tha-Thai" + ] + }, + { + "accuracy": 0.393212, + "f1": 0.360129, + "f1_weighted": 0.388375, + "scores_per_experiment": [ + { + "accuracy": 0.360059, + "f1": 0.34604, + "f1_weighted": 0.354433 + }, + { + "accuracy": 0.387113, + "f1": 0.355372, + "f1_weighted": 0.37319 + }, + { + "accuracy": 0.414166, + "f1": 0.38041, + "f1_weighted": 0.413779 + }, + { + "accuracy": 0.392523, + "f1": 0.348248, + "f1_weighted": 0.380181 + }, + { + "accuracy": 0.39154, + "f1": 0.352947, + "f1_weighted": 0.392089 + }, + { + "accuracy": 0.394983, + "f1": 0.369765, + "f1_weighted": 0.38453 + }, + { + "accuracy": 0.416134, + "f1": 0.381486, + "f1_weighted": 0.418719 + }, + { + "accuracy": 0.40482, + "f1": 0.377325, + "f1_weighted": 0.402659 + }, + { + "accuracy": 0.379242, + "f1": 0.333726, + "f1_weighted": 0.380496 + }, + { + "accuracy": 0.39154, + "f1": 0.355969, + "f1_weighted": 0.383673 + } + ], + "main_score": 0.393212, + "hf_subset": "fi", + "languages": [ + "fin-Latn" + ] + }, + { + "accuracy": 0.448205, + "f1": 0.410897, + "f1_weighted": 0.428443, + "scores_per_experiment": [ + { + "accuracy": 0.468273, + "f1": 0.429368, + "f1_weighted": 0.451578 + }, + { + "accuracy": 0.470733, + "f1": 0.429697, + "f1_weighted": 0.454991 + }, + { + "accuracy": 0.459911, + "f1": 0.418991, + "f1_weighted": 0.436475 + }, + { + "accuracy": 0.431874, + "f1": 0.391809, + "f1_weighted": 0.409707 + }, + { + "accuracy": 0.470241, + "f1": 0.419496, + "f1_weighted": 0.446973 + }, + { + "accuracy": 0.434333, + "f1": 0.401487, + "f1_weighted": 0.414197 + }, + { + "accuracy": 0.462863, + "f1": 0.422439, + "f1_weighted": 0.452437 + }, + { + "accuracy": 0.466798, + "f1": 0.432712, + "f1_weighted": 0.44765 + }, + { + "accuracy": 0.388588, + "f1": 0.367373, + "f1_weighted": 0.366233 + }, + { + "accuracy": 0.428431, + "f1": 0.395594, + "f1_weighted": 0.404192 + } + ], + "main_score": 0.448205, + "hf_subset": "ms", + "languages": [ + "msa-Latn" + ] + }, + { + "accuracy": 0.097737, + "f1": 0.03583, + "f1_weighted": 0.047886, + "scores_per_experiment": [ + { + "accuracy": 0.138219, + "f1": 0.04137, + "f1_weighted": 0.068478 + }, + { + "accuracy": 0.100344, + "f1": 0.037142, + "f1_weighted": 0.046492 + }, + { + "accuracy": 0.054107, + "f1": 0.030403, + "f1_weighted": 0.025632 + }, + { + "accuracy": 0.093458, + "f1": 0.034925, + "f1_weighted": 0.039743 + }, + { + "accuracy": 0.060994, + "f1": 0.027407, + "f1_weighted": 0.033315 + }, + { + "accuracy": 0.059518, + "f1": 0.034332, + "f1_weighted": 0.0337 + }, + { + "accuracy": 0.160354, + "f1": 0.040016, + "f1_weighted": 0.077257 + }, + { + "accuracy": 0.147565, + "f1": 0.045134, + "f1_weighted": 0.06938 + }, + { + "accuracy": 0.050664, + "f1": 0.024172, + "f1_weighted": 0.026588 + }, + { + "accuracy": 0.11215, + "f1": 0.043401, + "f1_weighted": 0.058275 + } + ], + "main_score": 0.097737, + "hf_subset": "km", + "languages": [ + "khm-Khmr" + ] + }, + { + "accuracy": 0.080964, + "f1": 0.031088, + "f1_weighted": 0.036662, + "scores_per_experiment": [ + { + "accuracy": 0.12789, + "f1": 0.035082, + "f1_weighted": 0.050817 + }, + { + "accuracy": 0.047221, + "f1": 0.015945, + "f1_weighted": 0.01103 + }, + { + "accuracy": 0.062469, + "f1": 0.03456, + "f1_weighted": 0.034426 + }, + { + "accuracy": 0.063453, + "f1": 0.030564, + "f1_weighted": 0.026481 + }, + { + "accuracy": 0.090015, + "f1": 0.044167, + "f1_weighted": 0.037863 + }, + { + "accuracy": 0.08362, + "f1": 0.03456, + "f1_weighted": 0.03351 + }, + { + "accuracy": 0.056567, + "f1": 0.024932, + "f1_weighted": 0.03642 + }, + { + "accuracy": 0.123463, + "f1": 0.040791, + "f1_weighted": 0.063143 + }, + { + "accuracy": 0.110674, + "f1": 0.032876, + "f1_weighted": 0.061225 + }, + { + "accuracy": 0.04427, + "f1": 0.017404, + "f1_weighted": 0.011702 + } + ], + "main_score": 0.080964, + "hf_subset": "am", + "languages": [ + "amh-Ethi" + ] + } + ], + "test": [ + { + "accuracy": 0.405649, + "f1": 0.400625, + "f1_weighted": 0.38625, + "scores_per_experiment": [ + { + "accuracy": 0.406187, + "f1": 0.40581, + "f1_weighted": 0.376404 + }, + { + "accuracy": 0.401816, + "f1": 0.401697, + "f1_weighted": 0.382933 + }, + { + "accuracy": 0.441829, + "f1": 0.426183, + "f1_weighted": 0.426891 + }, + { + "accuracy": 0.397781, + "f1": 0.384517, + "f1_weighted": 0.378298 + }, + { + "accuracy": 0.406523, + "f1": 0.395073, + "f1_weighted": 0.388387 + }, + { + "accuracy": 0.395763, + "f1": 0.395906, + "f1_weighted": 0.380194 + }, + { + "accuracy": 0.417283, + "f1": 0.406203, + "f1_weighted": 0.39646 + }, + { + "accuracy": 0.425353, + "f1": 0.424404, + "f1_weighted": 0.40474 + }, + { + "accuracy": 0.351379, + "f1": 0.357212, + "f1_weighted": 0.335135 + }, + { + "accuracy": 0.412576, + "f1": 0.409245, + "f1_weighted": 0.393058 + } + ], + "main_score": 0.405649, + "hf_subset": "tr", + "languages": [ + "tur-Latn" + ] + }, + { + "accuracy": 0.075286, + "f1": 0.029325, + "f1_weighted": 0.032592, + "scores_per_experiment": [ + { + "accuracy": 0.065905, + "f1": 0.026831, + "f1_weighted": 0.024956 + }, + { + "accuracy": 0.078009, + "f1": 0.03575, + "f1_weighted": 0.038103 + }, + { + "accuracy": 0.064896, + "f1": 0.027413, + "f1_weighted": 0.025378 + }, + { + "accuracy": 0.055817, + "f1": 0.026311, + "f1_weighted": 0.023776 + }, + { + "accuracy": 0.1039, + "f1": 0.040452, + "f1_weighted": 0.049927 + }, + { + "accuracy": 0.065905, + "f1": 0.030292, + "f1_weighted": 0.028071 + }, + { + "accuracy": 0.089442, + "f1": 0.026736, + "f1_weighted": 0.032442 + }, + { + "accuracy": 0.094149, + "f1": 0.033233, + "f1_weighted": 0.055642 + }, + { + "accuracy": 0.031271, + "f1": 0.012148, + "f1_weighted": 0.005785 + }, + { + "accuracy": 0.103564, + "f1": 0.034082, + "f1_weighted": 0.041839 + } + ], + "main_score": 0.075286, + "hf_subset": "ka", + "languages": [ + "kat-Geor" + ] + }, + { + "accuracy": 0.065871, + "f1": 0.024973, + "f1_weighted": 0.027215, + "scores_per_experiment": [ + { + "accuracy": 0.051446, + "f1": 0.018763, + "f1_weighted": 0.019255 + }, + { + "accuracy": 0.066577, + "f1": 0.01941, + "f1_weighted": 0.027328 + }, + { + "accuracy": 0.078009, + "f1": 0.031604, + "f1_weighted": 0.042244 + }, + { + "accuracy": 0.065905, + "f1": 0.019797, + "f1_weighted": 0.029262 + }, + { + "accuracy": 0.088769, + "f1": 0.029617, + "f1_weighted": 0.035507 + }, + { + "accuracy": 0.064896, + "f1": 0.024151, + "f1_weighted": 0.023292 + }, + { + "accuracy": 0.0538, + "f1": 0.028053, + "f1_weighted": 0.019671 + }, + { + "accuracy": 0.067586, + "f1": 0.028526, + "f1_weighted": 0.025407 + }, + { + "accuracy": 0.066913, + "f1": 0.029345, + "f1_weighted": 0.029335 + }, + { + "accuracy": 0.054808, + "f1": 0.020465, + "f1_weighted": 0.020846 + } + ], + "main_score": 0.065871, + "hf_subset": "ko", + "languages": [ + "kor-Kore" + ] + }, + { + "accuracy": 0.537189, + "f1": 0.509283, + "f1_weighted": 0.527945, + "scores_per_experiment": [ + { + "accuracy": 0.53497, + "f1": 0.515711, + "f1_weighted": 0.519644 + }, + { + "accuracy": 0.539677, + "f1": 0.518122, + "f1_weighted": 0.535229 + }, + { + "accuracy": 0.559852, + "f1": 0.523839, + "f1_weighted": 0.550963 + }, + { + "accuracy": 0.578682, + "f1": 0.538719, + "f1_weighted": 0.572666 + }, + { + "accuracy": 0.56187, + "f1": 0.523837, + "f1_weighted": 0.558253 + }, + { + "accuracy": 0.497646, + "f1": 0.465423, + "f1_weighted": 0.486485 + }, + { + "accuracy": 0.548083, + "f1": 0.510446, + "f1_weighted": 0.544893 + }, + { + "accuracy": 0.539341, + "f1": 0.521179, + "f1_weighted": 0.523239 + }, + { + "accuracy": 0.485541, + "f1": 0.471798, + "f1_weighted": 0.468517 + }, + { + "accuracy": 0.526227, + "f1": 0.503749, + "f1_weighted": 0.519557 + } + ], + "main_score": 0.537189, + "hf_subset": "it", + "languages": [ + "ita-Latn" + ] + }, + { + "accuracy": 0.231876, + "f1": 0.222369, + "f1_weighted": 0.217478, + "scores_per_experiment": [ + { + "accuracy": 0.193006, + "f1": 0.193802, + "f1_weighted": 0.168881 + }, + { + "accuracy": 0.239744, + "f1": 0.237175, + "f1_weighted": 0.226067 + }, + { + "accuracy": 0.271352, + "f1": 0.252502, + "f1_weighted": 0.264428 + }, + { + "accuracy": 0.233356, + "f1": 0.214618, + "f1_weighted": 0.227476 + }, + { + "accuracy": 0.256893, + "f1": 0.23649, + "f1_weighted": 0.233189 + }, + { + "accuracy": 0.208137, + "f1": 0.194064, + "f1_weighted": 0.173035 + }, + { + "accuracy": 0.235709, + "f1": 0.220324, + "f1_weighted": 0.224541 + }, + { + "accuracy": 0.254876, + "f1": 0.245925, + "f1_weighted": 0.255269 + }, + { + "accuracy": 0.207801, + "f1": 0.210312, + "f1_weighted": 0.200687 + }, + { + "accuracy": 0.217888, + "f1": 0.218475, + "f1_weighted": 0.201212 + } + ], + "main_score": 0.231876, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + }, + { + "accuracy": 0.064795, + "f1": 0.026863, + "f1_weighted": 0.031007, + "scores_per_experiment": [ + { + "accuracy": 0.087088, + "f1": 0.032904, + "f1_weighted": 0.05564 + }, + { + "accuracy": 0.060861, + "f1": 0.031002, + "f1_weighted": 0.03199 + }, + { + "accuracy": 0.04304, + "f1": 0.015625, + "f1_weighted": 0.009276 + }, + { + "accuracy": 0.042367, + "f1": 0.018127, + "f1_weighted": 0.018446 + }, + { + "accuracy": 0.055145, + "f1": 0.029097, + "f1_weighted": 0.022585 + }, + { + "accuracy": 0.050101, + "f1": 0.022106, + "f1_weighted": 0.021938 + }, + { + "accuracy": 0.046066, + "f1": 0.017812, + "f1_weighted": 0.017454 + }, + { + "accuracy": 0.148285, + "f1": 0.049087, + "f1_weighted": 0.075778 + }, + { + "accuracy": 0.037323, + "f1": 0.023366, + "f1_weighted": 0.020168 + }, + { + "accuracy": 0.077673, + "f1": 0.029501, + "f1_weighted": 0.0368 + } + ], + "main_score": 0.064795, + "hf_subset": "fa", + "languages": [ + "fas-Arab" + ] + }, + { + "accuracy": 0.215098, + "f1": 0.188507, + "f1_weighted": 0.204621, + "scores_per_experiment": [ + { + "accuracy": 0.238736, + "f1": 0.203802, + "f1_weighted": 0.224311 + }, + { + "accuracy": 0.196705, + "f1": 0.165789, + "f1_weighted": 0.18598 + }, + { + "accuracy": 0.248151, + "f1": 0.209988, + "f1_weighted": 0.248694 + }, + { + "accuracy": 0.23302, + "f1": 0.193479, + "f1_weighted": 0.216482 + }, + { + "accuracy": 0.235037, + "f1": 0.20525, + "f1_weighted": 0.223418 + }, + { + "accuracy": 0.190316, + "f1": 0.182554, + "f1_weighted": 0.167499 + }, + { + "accuracy": 0.223941, + "f1": 0.210119, + "f1_weighted": 0.229659 + }, + { + "accuracy": 0.202757, + "f1": 0.182043, + "f1_weighted": 0.19977 + }, + { + "accuracy": 0.204775, + "f1": 0.170232, + "f1_weighted": 0.190287 + }, + { + "accuracy": 0.177539, + "f1": 0.161819, + "f1_weighted": 0.160114 + } + ], + "main_score": 0.215098, + "hf_subset": "mn", + "languages": [ + "mon-Cyrl" + ] + }, + { + "accuracy": 0.085642, + "f1": 0.046249, + "f1_weighted": 0.045089, + "scores_per_experiment": [ + { + "accuracy": 0.125757, + "f1": 0.056944, + "f1_weighted": 0.076415 + }, + { + "accuracy": 0.096503, + "f1": 0.041821, + "f1_weighted": 0.03898 + }, + { + "accuracy": 0.047747, + "f1": 0.035484, + "f1_weighted": 0.025865 + }, + { + "accuracy": 0.077001, + "f1": 0.04541, + "f1_weighted": 0.041417 + }, + { + "accuracy": 0.040013, + "f1": 0.036852, + "f1_weighted": 0.021933 + }, + { + "accuracy": 0.055145, + "f1": 0.050643, + "f1_weighted": 0.042306 + }, + { + "accuracy": 0.103564, + "f1": 0.05185, + "f1_weighted": 0.049608 + }, + { + "accuracy": 0.147949, + "f1": 0.064212, + "f1_weighted": 0.076902 + }, + { + "accuracy": 0.065568, + "f1": 0.035922, + "f1_weighted": 0.03669 + }, + { + "accuracy": 0.097176, + "f1": 0.043352, + "f1_weighted": 0.040773 + } + ], + "main_score": 0.085642, + "hf_subset": "kn", + "languages": [ + "kan-Knda" + ] + }, + { + "accuracy": 0.092502, + "f1": 0.042756, + "f1_weighted": 0.048976, + "scores_per_experiment": [ + { + "accuracy": 0.115669, + "f1": 0.046147, + "f1_weighted": 0.060176 + }, + { + "accuracy": 0.062542, + "f1": 0.039587, + "f1_weighted": 0.04272 + }, + { + "accuracy": 0.076664, + "f1": 0.028371, + "f1_weighted": 0.02951 + }, + { + "accuracy": 0.050437, + "f1": 0.03984, + "f1_weighted": 0.02852 + }, + { + "accuracy": 0.084734, + "f1": 0.04366, + "f1_weighted": 0.04277 + }, + { + "accuracy": 0.075656, + "f1": 0.042859, + "f1_weighted": 0.039959 + }, + { + "accuracy": 0.11735, + "f1": 0.0531, + "f1_weighted": 0.082256 + }, + { + "accuracy": 0.14156, + "f1": 0.048371, + "f1_weighted": 0.065847 + }, + { + "accuracy": 0.115669, + "f1": 0.044084, + "f1_weighted": 0.061765 + }, + { + "accuracy": 0.084734, + "f1": 0.041541, + "f1_weighted": 0.036239 + } + ], + "main_score": 0.092502, + "hf_subset": "hy", + "languages": [ + "hye-Armn" + ] + }, + { + "accuracy": 0.119738, + "f1": 0.070108, + "f1_weighted": 0.101805, + "scores_per_experiment": [ + { + "accuracy": 0.096167, + "f1": 0.049729, + "f1_weighted": 0.065786 + }, + { + "accuracy": 0.095831, + "f1": 0.053036, + "f1_weighted": 0.068638 + }, + { + "accuracy": 0.113652, + "f1": 0.073547, + "f1_weighted": 0.113921 + }, + { + "accuracy": 0.136853, + "f1": 0.086454, + "f1_weighted": 0.125388 + }, + { + "accuracy": 0.091796, + "f1": 0.066375, + "f1_weighted": 0.076136 + }, + { + "accuracy": 0.126429, + "f1": 0.077996, + "f1_weighted": 0.132787 + }, + { + "accuracy": 0.119032, + "f1": 0.0711, + "f1_weighted": 0.111658 + }, + { + "accuracy": 0.192334, + "f1": 0.095196, + "f1_weighted": 0.160416 + }, + { + "accuracy": 0.132145, + "f1": 0.071817, + "f1_weighted": 0.094055 + }, + { + "accuracy": 0.093141, + "f1": 0.055827, + "f1_weighted": 0.069261 + } + ], + "main_score": 0.119738, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ] + }, + { + "accuracy": 0.53265, + "f1": 0.513544, + "f1_weighted": 0.519981, + "scores_per_experiment": [ + { + "accuracy": 0.524546, + "f1": 0.511129, + "f1_weighted": 0.511701 + }, + { + "accuracy": 0.54035, + "f1": 0.520621, + "f1_weighted": 0.518099 + }, + { + "accuracy": 0.52421, + "f1": 0.511152, + "f1_weighted": 0.514326 + }, + { + "accuracy": 0.558171, + "f1": 0.538894, + "f1_weighted": 0.556435 + }, + { + "accuracy": 0.553463, + "f1": 0.52691, + "f1_weighted": 0.539659 + }, + { + "accuracy": 0.531607, + "f1": 0.506796, + "f1_weighted": 0.523915 + }, + { + "accuracy": 0.528917, + "f1": 0.503089, + "f1_weighted": 0.507357 + }, + { + "accuracy": 0.526227, + "f1": 0.502588, + "f1_weighted": 0.513064 + }, + { + "accuracy": 0.523201, + "f1": 0.512467, + "f1_weighted": 0.513323 + }, + { + "accuracy": 0.515804, + "f1": 0.501795, + "f1_weighted": 0.501928 + } + ], + "main_score": 0.53265, + "hf_subset": "ro", + "languages": [ + "ron-Latn" + ] + }, + { + "accuracy": 0.471856, + "f1": 0.435399, + "f1_weighted": 0.467932, + "scores_per_experiment": [ + { + "accuracy": 0.47848, + "f1": 0.437959, + "f1_weighted": 0.480495 + }, + { + "accuracy": 0.45158, + "f1": 0.414232, + "f1_weighted": 0.444891 + }, + { + "accuracy": 0.480834, + "f1": 0.448813, + "f1_weighted": 0.479818 + }, + { + "accuracy": 0.501345, + "f1": 0.460928, + "f1_weighted": 0.504265 + }, + { + "accuracy": 0.47041, + "f1": 0.421674, + "f1_weighted": 0.463196 + }, + { + "accuracy": 0.466375, + "f1": 0.430431, + "f1_weighted": 0.465052 + }, + { + "accuracy": 0.504035, + "f1": 0.463127, + "f1_weighted": 0.501204 + }, + { + "accuracy": 0.494284, + "f1": 0.46231, + "f1_weighted": 0.495248 + }, + { + "accuracy": 0.417956, + "f1": 0.398212, + "f1_weighted": 0.403856 + }, + { + "accuracy": 0.453262, + "f1": 0.4163, + "f1_weighted": 0.441292 + } + ], + "main_score": 0.471856, + "hf_subset": "nb", + "languages": [ + "nob-Latn" + ] + }, + { + "accuracy": 0.465568, + "f1": 0.430253, + "f1_weighted": 0.460311, + "scores_per_experiment": [ + { + "accuracy": 0.473773, + "f1": 0.43646, + "f1_weighted": 0.462691 + }, + { + "accuracy": 0.460323, + "f1": 0.419892, + "f1_weighted": 0.460595 + }, + { + "accuracy": 0.487895, + "f1": 0.455147, + "f1_weighted": 0.486318 + }, + { + "accuracy": 0.472764, + "f1": 0.429131, + "f1_weighted": 0.474355 + }, + { + "accuracy": 0.465367, + "f1": 0.420165, + "f1_weighted": 0.45761 + }, + { + "accuracy": 0.446537, + "f1": 0.408707, + "f1_weighted": 0.436602 + }, + { + "accuracy": 0.475454, + "f1": 0.434789, + "f1_weighted": 0.467086 + }, + { + "accuracy": 0.493948, + "f1": 0.455617, + "f1_weighted": 0.490154 + }, + { + "accuracy": 0.43544, + "f1": 0.412399, + "f1_weighted": 0.424944 + }, + { + "accuracy": 0.444183, + "f1": 0.430224, + "f1_weighted": 0.442758 + } + ], + "main_score": 0.465568, + "hf_subset": "sv", + "languages": [ + "swe-Latn" + ] + }, + { + "accuracy": 0.439005, + "f1": 0.410035, + "f1_weighted": 0.435633, + "scores_per_experiment": [ + { + "accuracy": 0.422327, + "f1": 0.402761, + "f1_weighted": 0.424978 + }, + { + "accuracy": 0.437122, + "f1": 0.404456, + "f1_weighted": 0.426828 + }, + { + "accuracy": 0.437122, + "f1": 0.411943, + "f1_weighted": 0.42452 + }, + { + "accuracy": 0.436449, + "f1": 0.411824, + "f1_weighted": 0.435508 + }, + { + "accuracy": 0.457297, + "f1": 0.413875, + "f1_weighted": 0.452768 + }, + { + "accuracy": 0.455615, + "f1": 0.420709, + "f1_weighted": 0.457477 + }, + { + "accuracy": 0.452589, + "f1": 0.418742, + "f1_weighted": 0.451407 + }, + { + "accuracy": 0.439812, + "f1": 0.41958, + "f1_weighted": 0.443594 + }, + { + "accuracy": 0.442838, + "f1": 0.408275, + "f1_weighted": 0.433161 + }, + { + "accuracy": 0.408877, + "f1": 0.388182, + "f1_weighted": 0.406084 + } + ], + "main_score": 0.439005, + "hf_subset": "pl", + "languages": [ + "pol-Latn" + ] + }, + { + "accuracy": 0.105783, + "f1": 0.04225, + "f1_weighted": 0.058061, + "scores_per_experiment": [ + { + "accuracy": 0.055817, + "f1": 0.015187, + "f1_weighted": 0.012144 + }, + { + "accuracy": 0.098521, + "f1": 0.034534, + "f1_weighted": 0.050183 + }, + { + "accuracy": 0.089778, + "f1": 0.037594, + "f1_weighted": 0.053582 + }, + { + "accuracy": 0.114997, + "f1": 0.049796, + "f1_weighted": 0.067481 + }, + { + "accuracy": 0.115333, + "f1": 0.054987, + "f1_weighted": 0.070918 + }, + { + "accuracy": 0.099529, + "f1": 0.045292, + "f1_weighted": 0.061032 + }, + { + "accuracy": 0.090787, + "f1": 0.045263, + "f1_weighted": 0.060222 + }, + { + "accuracy": 0.093477, + "f1": 0.031728, + "f1_weighted": 0.045521 + }, + { + "accuracy": 0.137525, + "f1": 0.055336, + "f1_weighted": 0.076491 + }, + { + "accuracy": 0.162071, + "f1": 0.05278, + "f1_weighted": 0.083035 + } + ], + "main_score": 0.105783, + "hf_subset": "zh-TW", + "languages": [ + "cmo-Hant" + ] + }, + { + "accuracy": 0.479455, + "f1": 0.45265, + "f1_weighted": 0.466741, + "scores_per_experiment": [ + { + "accuracy": 0.461668, + "f1": 0.448756, + "f1_weighted": 0.436549 + }, + { + "accuracy": 0.478144, + "f1": 0.454853, + "f1_weighted": 0.470873 + }, + { + "accuracy": 0.503699, + "f1": 0.473719, + "f1_weighted": 0.494169 + }, + { + "accuracy": 0.466039, + "f1": 0.430051, + "f1_weighted": 0.450751 + }, + { + "accuracy": 0.515804, + "f1": 0.470291, + "f1_weighted": 0.506601 + }, + { + "accuracy": 0.48924, + "f1": 0.456177, + "f1_weighted": 0.474566 + }, + { + "accuracy": 0.49193, + "f1": 0.469707, + "f1_weighted": 0.485052 + }, + { + "accuracy": 0.464694, + "f1": 0.444004, + "f1_weighted": 0.453109 + }, + { + "accuracy": 0.442502, + "f1": 0.424646, + "f1_weighted": 0.420025 + }, + { + "accuracy": 0.480834, + "f1": 0.454292, + "f1_weighted": 0.475716 + } + ], + "main_score": 0.479455, + "hf_subset": "tl", + "languages": [ + "tgl-Latn" + ] + }, + { + "accuracy": 0.074849, + "f1": 0.029607, + "f1_weighted": 0.036261, + "scores_per_experiment": [ + { + "accuracy": 0.125084, + "f1": 0.043557, + "f1_weighted": 0.070778 + }, + { + "accuracy": 0.062542, + "f1": 0.021005, + "f1_weighted": 0.020452 + }, + { + "accuracy": 0.052455, + "f1": 0.01671, + "f1_weighted": 0.021084 + }, + { + "accuracy": 0.06456, + "f1": 0.030471, + "f1_weighted": 0.022123 + }, + { + "accuracy": 0.073638, + "f1": 0.031872, + "f1_weighted": 0.03621 + }, + { + "accuracy": 0.066577, + "f1": 0.023119, + "f1_weighted": 0.027072 + }, + { + "accuracy": 0.083053, + "f1": 0.038769, + "f1_weighted": 0.050997 + }, + { + "accuracy": 0.131137, + "f1": 0.054498, + "f1_weighted": 0.085356 + }, + { + "accuracy": 0.031271, + "f1": 0.014259, + "f1_weighted": 0.009602 + }, + { + "accuracy": 0.058171, + "f1": 0.021807, + "f1_weighted": 0.018937 + } + ], + "main_score": 0.074849, + "hf_subset": "ml", + "languages": [ + "mal-Mlym" + ] + }, + { + "accuracy": 0.53154, + "f1": 0.51709, + "f1_weighted": 0.516473, + "scores_per_experiment": [ + { + "accuracy": 0.520847, + "f1": 0.508672, + "f1_weighted": 0.500752 + }, + { + "accuracy": 0.564896, + "f1": 0.540393, + "f1_weighted": 0.559163 + }, + { + "accuracy": 0.557498, + "f1": 0.540323, + "f1_weighted": 0.544038 + }, + { + "accuracy": 0.530262, + "f1": 0.51849, + "f1_weighted": 0.518412 + }, + { + "accuracy": 0.52152, + "f1": 0.507189, + "f1_weighted": 0.508352 + }, + { + "accuracy": 0.527236, + "f1": 0.511235, + "f1_weighted": 0.515862 + }, + { + "accuracy": 0.506052, + "f1": 0.489246, + "f1_weighted": 0.486066 + }, + { + "accuracy": 0.532952, + "f1": 0.520975, + "f1_weighted": 0.512886 + }, + { + "accuracy": 0.539341, + "f1": 0.53426, + "f1_weighted": 0.524625 + }, + { + "accuracy": 0.514795, + "f1": 0.50012, + "f1_weighted": 0.494571 + } + ], + "main_score": 0.53154, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ] + }, + { + "accuracy": 0.743813, + "f1": 0.734987, + "f1_weighted": 0.740363, + "scores_per_experiment": [ + { + "accuracy": 0.747142, + "f1": 0.741753, + "f1_weighted": 0.744287 + }, + { + "accuracy": 0.740417, + "f1": 0.736266, + "f1_weighted": 0.735518 + }, + { + "accuracy": 0.770343, + "f1": 0.754278, + "f1_weighted": 0.765252 + }, + { + "accuracy": 0.746133, + "f1": 0.733127, + "f1_weighted": 0.743196 + }, + { + "accuracy": 0.748823, + "f1": 0.735122, + "f1_weighted": 0.743656 + }, + { + "accuracy": 0.735709, + "f1": 0.729142, + "f1_weighted": 0.73185 + }, + { + "accuracy": 0.743107, + "f1": 0.733047, + "f1_weighted": 0.740774 + }, + { + "accuracy": 0.731002, + "f1": 0.724029, + "f1_weighted": 0.733079 + }, + { + "accuracy": 0.755548, + "f1": 0.74419, + "f1_weighted": 0.750221 + }, + { + "accuracy": 0.719906, + "f1": 0.71892, + "f1_weighted": 0.7158 + } + ], + "main_score": 0.743813, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.40649, + "f1": 0.369633, + "f1_weighted": 0.398846, + "scores_per_experiment": [ + { + "accuracy": 0.397781, + "f1": 0.371405, + "f1_weighted": 0.390754 + }, + { + "accuracy": 0.422999, + "f1": 0.379459, + "f1_weighted": 0.416402 + }, + { + "accuracy": 0.441493, + "f1": 0.398073, + "f1_weighted": 0.430088 + }, + { + "accuracy": 0.391056, + "f1": 0.354147, + "f1_weighted": 0.384253 + }, + { + "accuracy": 0.427707, + "f1": 0.381604, + "f1_weighted": 0.416216 + }, + { + "accuracy": 0.386348, + "f1": 0.351023, + "f1_weighted": 0.371863 + }, + { + "accuracy": 0.422327, + "f1": 0.379725, + "f1_weighted": 0.423413 + }, + { + "accuracy": 0.404842, + "f1": 0.378427, + "f1_weighted": 0.406937 + }, + { + "accuracy": 0.382986, + "f1": 0.350192, + "f1_weighted": 0.376464 + }, + { + "accuracy": 0.387357, + "f1": 0.35228, + "f1_weighted": 0.37207 + } + ], + "main_score": 0.40649, + "hf_subset": "cy", + "languages": [ + "cym-Latn" + ] + }, + { + "accuracy": 0.586853, + "f1": 0.561079, + "f1_weighted": 0.580554, + "scores_per_experiment": [ + { + "accuracy": 0.594149, + "f1": 0.565625, + "f1_weighted": 0.587747 + }, + { + "accuracy": 0.605245, + "f1": 0.583194, + "f1_weighted": 0.601244 + }, + { + "accuracy": 0.604909, + "f1": 0.577255, + "f1_weighted": 0.594563 + }, + { + "accuracy": 0.602892, + "f1": 0.571631, + "f1_weighted": 0.603 + }, + { + "accuracy": 0.60121, + "f1": 0.56958, + "f1_weighted": 0.593242 + }, + { + "accuracy": 0.573974, + "f1": 0.54915, + "f1_weighted": 0.563897 + }, + { + "accuracy": 0.565232, + "f1": 0.5445, + "f1_weighted": 0.565828 + }, + { + "accuracy": 0.587424, + "f1": 0.57096, + "f1_weighted": 0.587946 + }, + { + "accuracy": 0.578009, + "f1": 0.550894, + "f1_weighted": 0.56356 + }, + { + "accuracy": 0.555481, + "f1": 0.528001, + "f1_weighted": 0.544516 + } + ], + "main_score": 0.586853, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ] + }, + { + "accuracy": 0.40612, + "f1": 0.380418, + "f1_weighted": 0.398923, + "scores_per_experiment": [ + { + "accuracy": 0.397108, + "f1": 0.373738, + "f1_weighted": 0.393982 + }, + { + "accuracy": 0.423336, + "f1": 0.389179, + "f1_weighted": 0.426814 + }, + { + "accuracy": 0.427371, + "f1": 0.388236, + "f1_weighted": 0.424054 + }, + { + "accuracy": 0.399798, + "f1": 0.370171, + "f1_weighted": 0.396963 + }, + { + "accuracy": 0.387021, + "f1": 0.349333, + "f1_weighted": 0.356342 + }, + { + "accuracy": 0.406859, + "f1": 0.389044, + "f1_weighted": 0.404017 + }, + { + "accuracy": 0.391728, + "f1": 0.383964, + "f1_weighted": 0.395324 + }, + { + "accuracy": 0.437794, + "f1": 0.414225, + "f1_weighted": 0.425352 + }, + { + "accuracy": 0.357767, + "f1": 0.339486, + "f1_weighted": 0.34579 + }, + { + "accuracy": 0.432414, + "f1": 0.406801, + "f1_weighted": 0.420591 + } + ], + "main_score": 0.40612, + "hf_subset": "jv", + "languages": [ + "jav-Latn" + ] + }, + { + "accuracy": 0.462239, + "f1": 0.427288, + "f1_weighted": 0.459288, + "scores_per_experiment": [ + { + "accuracy": 0.476126, + "f1": 0.440124, + "f1_weighted": 0.473373 + }, + { + "accuracy": 0.487559, + "f1": 0.444496, + "f1_weighted": 0.487842 + }, + { + "accuracy": 0.472428, + "f1": 0.43907, + "f1_weighted": 0.467197 + }, + { + "accuracy": 0.452589, + "f1": 0.420645, + "f1_weighted": 0.456952 + }, + { + "accuracy": 0.480834, + "f1": 0.421791, + "f1_weighted": 0.463993 + }, + { + "accuracy": 0.431069, + "f1": 0.401922, + "f1_weighted": 0.430117 + }, + { + "accuracy": 0.479489, + "f1": 0.438125, + "f1_weighted": 0.474077 + }, + { + "accuracy": 0.46503, + "f1": 0.44572, + "f1_weighted": 0.462844 + }, + { + "accuracy": 0.435777, + "f1": 0.404085, + "f1_weighted": 0.435143 + }, + { + "accuracy": 0.441493, + "f1": 0.416905, + "f1_weighted": 0.441346 + } + ], + "main_score": 0.462239, + "hf_subset": "af", + "languages": [ + "afr-Latn" + ] + }, + { + "accuracy": 0.083961, + "f1": 0.034587, + "f1_weighted": 0.041437, + "scores_per_experiment": [ + { + "accuracy": 0.134163, + "f1": 0.042506, + "f1_weighted": 0.06912 + }, + { + "accuracy": 0.148285, + "f1": 0.045716, + "f1_weighted": 0.073143 + }, + { + "accuracy": 0.044385, + "f1": 0.021666, + "f1_weighted": 0.021688 + }, + { + "accuracy": 0.056826, + "f1": 0.031809, + "f1_weighted": 0.037183 + }, + { + "accuracy": 0.161399, + "f1": 0.052262, + "f1_weighted": 0.082359 + }, + { + "accuracy": 0.090451, + "f1": 0.041364, + "f1_weighted": 0.044225 + }, + { + "accuracy": 0.0538, + "f1": 0.024931, + "f1_weighted": 0.018943 + }, + { + "accuracy": 0.062206, + "f1": 0.037076, + "f1_weighted": 0.030147 + }, + { + "accuracy": 0.042031, + "f1": 0.026555, + "f1_weighted": 0.015503 + }, + { + "accuracy": 0.046066, + "f1": 0.021983, + "f1_weighted": 0.022062 + } + ], + "main_score": 0.083961, + "hf_subset": "bn", + "languages": [ + "ben-Beng" + ] + }, + { + "accuracy": 0.611163, + "f1": 0.59326, + "f1_weighted": 0.603628, + "scores_per_experiment": [ + { + "accuracy": 0.622058, + "f1": 0.612081, + "f1_weighted": 0.612456 + }, + { + "accuracy": 0.630128, + "f1": 0.611416, + "f1_weighted": 0.622826 + }, + { + "accuracy": 0.620377, + "f1": 0.594052, + "f1_weighted": 0.611704 + }, + { + "accuracy": 0.625084, + "f1": 0.600346, + "f1_weighted": 0.623203 + }, + { + "accuracy": 0.613315, + "f1": 0.596192, + "f1_weighted": 0.600887 + }, + { + "accuracy": 0.596167, + "f1": 0.580421, + "f1_weighted": 0.594225 + }, + { + "accuracy": 0.615669, + "f1": 0.586931, + "f1_weighted": 0.607965 + }, + { + "accuracy": 0.605918, + "f1": 0.592578, + "f1_weighted": 0.608519 + }, + { + "accuracy": 0.581372, + "f1": 0.573882, + "f1_weighted": 0.563302 + }, + { + "accuracy": 0.601547, + "f1": 0.584702, + "f1_weighted": 0.591191 + } + ], + "main_score": 0.611163, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ] + }, + { + "accuracy": 0.094317, + "f1": 0.044457, + "f1_weighted": 0.050101, + "scores_per_experiment": [ + { + "accuracy": 0.132145, + "f1": 0.044957, + "f1_weighted": 0.058788 + }, + { + "accuracy": 0.105245, + "f1": 0.042553, + "f1_weighted": 0.041853 + }, + { + "accuracy": 0.082381, + "f1": 0.040587, + "f1_weighted": 0.04286 + }, + { + "accuracy": 0.059516, + "f1": 0.025313, + "f1_weighted": 0.030862 + }, + { + "accuracy": 0.103228, + "f1": 0.048588, + "f1_weighted": 0.049968 + }, + { + "accuracy": 0.062878, + "f1": 0.036959, + "f1_weighted": 0.026975 + }, + { + "accuracy": 0.111298, + "f1": 0.058467, + "f1_weighted": 0.088694 + }, + { + "accuracy": 0.084398, + "f1": 0.044575, + "f1_weighted": 0.036029 + }, + { + "accuracy": 0.098184, + "f1": 0.052193, + "f1_weighted": 0.073913 + }, + { + "accuracy": 0.1039, + "f1": 0.050375, + "f1_weighted": 0.051068 + } + ], + "main_score": 0.094317, + "hf_subset": "my", + "languages": [ + "mya-Mymr" + ] + }, + { + "accuracy": 0.513248, + "f1": 0.474219, + "f1_weighted": 0.51052, + "scores_per_experiment": [ + { + "accuracy": 0.523537, + "f1": 0.487705, + "f1_weighted": 0.519797 + }, + { + "accuracy": 0.530599, + "f1": 0.482111, + "f1_weighted": 0.524131 + }, + { + "accuracy": 0.513786, + "f1": 0.466937, + "f1_weighted": 0.503223 + }, + { + "accuracy": 0.5269, + "f1": 0.484523, + "f1_weighted": 0.526754 + }, + { + "accuracy": 0.512441, + "f1": 0.467776, + "f1_weighted": 0.509759 + }, + { + "accuracy": 0.495629, + "f1": 0.456119, + "f1_weighted": 0.491792 + }, + { + "accuracy": 0.519502, + "f1": 0.480967, + "f1_weighted": 0.515809 + }, + { + "accuracy": 0.500672, + "f1": 0.473583, + "f1_weighted": 0.501997 + }, + { + "accuracy": 0.515131, + "f1": 0.483899, + "f1_weighted": 0.513834 + }, + { + "accuracy": 0.494284, + "f1": 0.458566, + "f1_weighted": 0.498109 + } + ], + "main_score": 0.513248, + "hf_subset": "nl", + "languages": [ + "nld-Latn" + ] + }, + { + "accuracy": 0.403699, + "f1": 0.373693, + "f1_weighted": 0.394801, + "scores_per_experiment": [ + { + "accuracy": 0.406523, + "f1": 0.38407, + "f1_weighted": 0.390478 + }, + { + "accuracy": 0.411231, + "f1": 0.376438, + "f1_weighted": 0.407474 + }, + { + "accuracy": 0.416611, + "f1": 0.381623, + "f1_weighted": 0.414612 + }, + { + "accuracy": 0.403497, + "f1": 0.376779, + "f1_weighted": 0.398049 + }, + { + "accuracy": 0.387693, + "f1": 0.350786, + "f1_weighted": 0.379734 + }, + { + "accuracy": 0.415938, + "f1": 0.387655, + "f1_weighted": 0.40987 + }, + { + "accuracy": 0.395091, + "f1": 0.368036, + "f1_weighted": 0.386474 + }, + { + "accuracy": 0.415602, + "f1": 0.384968, + "f1_weighted": 0.408667 + }, + { + "accuracy": 0.38534, + "f1": 0.354668, + "f1_weighted": 0.370242 + }, + { + "accuracy": 0.399462, + "f1": 0.371912, + "f1_weighted": 0.382414 + } + ], + "main_score": 0.403699, + "hf_subset": "hu", + "languages": [ + "hun-Latn" + ] + }, + { + "accuracy": 0.405952, + "f1": 0.376129, + "f1_weighted": 0.399526, + "scores_per_experiment": [ + { + "accuracy": 0.408204, + "f1": 0.388179, + "f1_weighted": 0.4003 + }, + { + "accuracy": 0.395091, + "f1": 0.360068, + "f1_weighted": 0.382917 + }, + { + "accuracy": 0.438467, + "f1": 0.396074, + "f1_weighted": 0.421503 + }, + { + "accuracy": 0.391056, + "f1": 0.364479, + "f1_weighted": 0.399791 + }, + { + "accuracy": 0.425689, + "f1": 0.385384, + "f1_weighted": 0.417932 + }, + { + "accuracy": 0.371217, + "f1": 0.344183, + "f1_weighted": 0.361475 + }, + { + "accuracy": 0.44082, + "f1": 0.409787, + "f1_weighted": 0.445577 + }, + { + "accuracy": 0.421654, + "f1": 0.399783, + "f1_weighted": 0.414318 + }, + { + "accuracy": 0.35037, + "f1": 0.332591, + "f1_weighted": 0.340088 + }, + { + "accuracy": 0.416947, + "f1": 0.380766, + "f1_weighted": 0.411363 + } + ], + "main_score": 0.405952, + "hf_subset": "lv", + "languages": [ + "lav-Latn" + ] + }, + { + "accuracy": 0.509583, + "f1": 0.47378, + "f1_weighted": 0.501137, + "scores_per_experiment": [ + { + "accuracy": 0.497983, + "f1": 0.468, + "f1_weighted": 0.494143 + }, + { + "accuracy": 0.502354, + "f1": 0.462466, + "f1_weighted": 0.49076 + }, + { + "accuracy": 0.527909, + "f1": 0.509934, + "f1_weighted": 0.528704 + }, + { + "accuracy": 0.529926, + "f1": 0.4894, + "f1_weighted": 0.524992 + }, + { + "accuracy": 0.51076, + "f1": 0.463577, + "f1_weighted": 0.495653 + }, + { + "accuracy": 0.484533, + "f1": 0.449552, + "f1_weighted": 0.472412 + }, + { + "accuracy": 0.522529, + "f1": 0.481587, + "f1_weighted": 0.518155 + }, + { + "accuracy": 0.517149, + "f1": 0.480297, + "f1_weighted": 0.512021 + }, + { + "accuracy": 0.502354, + "f1": 0.471906, + "f1_weighted": 0.487296 + }, + { + "accuracy": 0.500336, + "f1": 0.461076, + "f1_weighted": 0.487236 + } + ], + "main_score": 0.509583, + "hf_subset": "da", + "languages": [ + "dan-Latn" + ] + }, + { + "accuracy": 0.40195, + "f1": 0.368876, + "f1_weighted": 0.398158, + "scores_per_experiment": [ + { + "accuracy": 0.398117, + "f1": 0.37862, + "f1_weighted": 0.396804 + }, + { + "accuracy": 0.406523, + "f1": 0.374746, + "f1_weighted": 0.409105 + }, + { + "accuracy": 0.429388, + "f1": 0.392178, + "f1_weighted": 0.420521 + }, + { + "accuracy": 0.3961, + "f1": 0.354889, + "f1_weighted": 0.39602 + }, + { + "accuracy": 0.396436, + "f1": 0.368417, + "f1_weighted": 0.390627 + }, + { + "accuracy": 0.409549, + "f1": 0.365265, + "f1_weighted": 0.403463 + }, + { + "accuracy": 0.423672, + "f1": 0.383899, + "f1_weighted": 0.414644 + }, + { + "accuracy": 0.422663, + "f1": 0.402671, + "f1_weighted": 0.432733 + }, + { + "accuracy": 0.370881, + "f1": 0.333332, + "f1_weighted": 0.357322 + }, + { + "accuracy": 0.366174, + "f1": 0.334741, + "f1_weighted": 0.36034 + } + ], + "main_score": 0.40195, + "hf_subset": "is", + "languages": [ + "isl-Latn" + ] + }, + { + "accuracy": 0.170545, + "f1": 0.120314, + "f1_weighted": 0.139491, + "scores_per_experiment": [ + { + "accuracy": 0.223605, + "f1": 0.147653, + "f1_weighted": 0.188287 + }, + { + "accuracy": 0.140888, + "f1": 0.100393, + "f1_weighted": 0.109487 + }, + { + "accuracy": 0.148621, + "f1": 0.109378, + "f1_weighted": 0.117388 + }, + { + "accuracy": 0.150303, + "f1": 0.120199, + "f1_weighted": 0.123672 + }, + { + "accuracy": 0.15501, + "f1": 0.119765, + "f1_weighted": 0.125493 + }, + { + "accuracy": 0.176194, + "f1": 0.129903, + "f1_weighted": 0.137496 + }, + { + "accuracy": 0.139206, + "f1": 0.088537, + "f1_weighted": 0.111601 + }, + { + "accuracy": 0.224277, + "f1": 0.124531, + "f1_weighted": 0.16949 + }, + { + "accuracy": 0.177875, + "f1": 0.135822, + "f1_weighted": 0.157753 + }, + { + "accuracy": 0.169469, + "f1": 0.126961, + "f1_weighted": 0.154246 + } + ], + "main_score": 0.170545, + "hf_subset": "el", + "languages": [ + "ell-Grek" + ] + }, + { + "accuracy": 0.40733, + "f1": 0.388444, + "f1_weighted": 0.394112, + "scores_per_experiment": [ + { + "accuracy": 0.410558, + "f1": 0.39988, + "f1_weighted": 0.388419 + }, + { + "accuracy": 0.400471, + "f1": 0.377122, + "f1_weighted": 0.39006 + }, + { + "accuracy": 0.447882, + "f1": 0.41811, + "f1_weighted": 0.439631 + }, + { + "accuracy": 0.407868, + "f1": 0.386288, + "f1_weighted": 0.39084 + }, + { + "accuracy": 0.419301, + "f1": 0.39638, + "f1_weighted": 0.408754 + }, + { + "accuracy": 0.360121, + "f1": 0.333705, + "f1_weighted": 0.327261 + }, + { + "accuracy": 0.415602, + "f1": 0.399998, + "f1_weighted": 0.404442 + }, + { + "accuracy": 0.441157, + "f1": 0.426249, + "f1_weighted": 0.435132 + }, + { + "accuracy": 0.383995, + "f1": 0.375707, + "f1_weighted": 0.383313 + }, + { + "accuracy": 0.386348, + "f1": 0.371005, + "f1_weighted": 0.373266 + } + ], + "main_score": 0.40733, + "hf_subset": "sl", + "languages": [ + "slv-Latn" + ] + }, + { + "accuracy": 0.390383, + "f1": 0.362853, + "f1_weighted": 0.381188, + "scores_per_experiment": [ + { + "accuracy": 0.3961, + "f1": 0.367639, + "f1_weighted": 0.381149 + }, + { + "accuracy": 0.392065, + "f1": 0.368466, + "f1_weighted": 0.388781 + }, + { + "accuracy": 0.402152, + "f1": 0.367081, + "f1_weighted": 0.39279 + }, + { + "accuracy": 0.405514, + "f1": 0.375901, + "f1_weighted": 0.401386 + }, + { + "accuracy": 0.398453, + "f1": 0.360334, + "f1_weighted": 0.396048 + }, + { + "accuracy": 0.397781, + "f1": 0.362412, + "f1_weighted": 0.379686 + }, + { + "accuracy": 0.404842, + "f1": 0.375348, + "f1_weighted": 0.395883 + }, + { + "accuracy": 0.387357, + "f1": 0.363877, + "f1_weighted": 0.371938 + }, + { + "accuracy": 0.33692, + "f1": 0.316989, + "f1_weighted": 0.326986 + }, + { + "accuracy": 0.38265, + "f1": 0.370478, + "f1_weighted": 0.37723 + } + ], + "main_score": 0.390383, + "hf_subset": "az", + "languages": [ + "aze-Latn" + ] + }, + { + "accuracy": 0.437693, + "f1": 0.400809, + "f1_weighted": 0.422233, + "scores_per_experiment": [ + { + "accuracy": 0.446537, + "f1": 0.41934, + "f1_weighted": 0.43749 + }, + { + "accuracy": 0.439475, + "f1": 0.402824, + "f1_weighted": 0.430661 + }, + { + "accuracy": 0.4462, + "f1": 0.406298, + "f1_weighted": 0.433331 + }, + { + "accuracy": 0.44351, + "f1": 0.397305, + "f1_weighted": 0.424061 + }, + { + "accuracy": 0.428379, + "f1": 0.384051, + "f1_weighted": 0.40672 + }, + { + "accuracy": 0.424681, + "f1": 0.375732, + "f1_weighted": 0.401123 + }, + { + "accuracy": 0.446873, + "f1": 0.40699, + "f1_weighted": 0.429753 + }, + { + "accuracy": 0.467384, + "f1": 0.430663, + "f1_weighted": 0.451774 + }, + { + "accuracy": 0.392737, + "f1": 0.370324, + "f1_weighted": 0.38023 + }, + { + "accuracy": 0.441157, + "f1": 0.414566, + "f1_weighted": 0.427182 + } + ], + "main_score": 0.437693, + "hf_subset": "sw", + "languages": [ + "swa-Latn" + ] + }, + { + "accuracy": 0.323235, + "f1": 0.306747, + "f1_weighted": 0.313701, + "scores_per_experiment": [ + { + "accuracy": 0.317081, + "f1": 0.297839, + "f1_weighted": 0.295381 + }, + { + "accuracy": 0.315064, + "f1": 0.302069, + "f1_weighted": 0.307067 + }, + { + "accuracy": 0.34768, + "f1": 0.325417, + "f1_weighted": 0.347062 + }, + { + "accuracy": 0.310693, + "f1": 0.284918, + "f1_weighted": 0.294911 + }, + { + "accuracy": 0.334566, + "f1": 0.325115, + "f1_weighted": 0.328877 + }, + { + "accuracy": 0.293544, + "f1": 0.284556, + "f1_weighted": 0.291518 + }, + { + "accuracy": 0.352051, + "f1": 0.33707, + "f1_weighted": 0.346485 + }, + { + "accuracy": 0.309011, + "f1": 0.301796, + "f1_weighted": 0.289945 + }, + { + "accuracy": 0.333557, + "f1": 0.311338, + "f1_weighted": 0.322321 + }, + { + "accuracy": 0.319099, + "f1": 0.297355, + "f1_weighted": 0.313441 + } + ], + "main_score": 0.323235, + "hf_subset": "vi", + "languages": [ + "vie-Latn" + ] + }, + { + "accuracy": 0.075151, + "f1": 0.02176, + "f1_weighted": 0.02805, + "scores_per_experiment": [ + { + "accuracy": 0.135508, + "f1": 0.030499, + "f1_weighted": 0.061895 + }, + { + "accuracy": 0.052455, + "f1": 0.017328, + "f1_weighted": 0.011904 + }, + { + "accuracy": 0.044721, + "f1": 0.025171, + "f1_weighted": 0.021439 + }, + { + "accuracy": 0.062542, + "f1": 0.019744, + "f1_weighted": 0.016019 + }, + { + "accuracy": 0.079018, + "f1": 0.021287, + "f1_weighted": 0.021348 + }, + { + "accuracy": 0.0538, + "f1": 0.014284, + "f1_weighted": 0.015593 + }, + { + "accuracy": 0.040013, + "f1": 0.014521, + "f1_weighted": 0.009637 + }, + { + "accuracy": 0.120377, + "f1": 0.024865, + "f1_weighted": 0.047861 + }, + { + "accuracy": 0.098521, + "f1": 0.02514, + "f1_weighted": 0.047871 + }, + { + "accuracy": 0.06456, + "f1": 0.024759, + "f1_weighted": 0.026933 + } + ], + "main_score": 0.075151, + "hf_subset": "ta", + "languages": [ + "tam-Taml" + ] + }, + { + "accuracy": 0.049496, + "f1": 0.009184, + "f1_weighted": 0.012168, + "scores_per_experiment": [ + { + "accuracy": 0.02421, + "f1": 0.008834, + "f1_weighted": 0.010769 + }, + { + "accuracy": 0.025219, + "f1": 0.007485, + "f1_weighted": 0.008369 + }, + { + "accuracy": 0.097848, + "f1": 0.012939, + "f1_weighted": 0.021279 + }, + { + "accuracy": 0.127102, + "f1": 0.016197, + "f1_weighted": 0.033696 + }, + { + "accuracy": 0.037996, + "f1": 0.008815, + "f1_weighted": 0.009414 + }, + { + "accuracy": 0.040686, + "f1": 0.01241, + "f1_weighted": 0.014196 + }, + { + "accuracy": 0.03228, + "f1": 0.003474, + "f1_weighted": 0.002019 + }, + { + "accuracy": 0.034297, + "f1": 0.007072, + "f1_weighted": 0.007225 + }, + { + "accuracy": 0.03497, + "f1": 0.005823, + "f1_weighted": 0.004007 + }, + { + "accuracy": 0.04035, + "f1": 0.00879, + "f1_weighted": 0.010709 + } + ], + "main_score": 0.049496, + "hf_subset": "zh-CN", + "languages": [ + "cmo-Hans" + ] + }, + { + "accuracy": 0.496133, + "f1": 0.451953, + "f1_weighted": 0.487377, + "scores_per_experiment": [ + { + "accuracy": 0.533289, + "f1": 0.493199, + "f1_weighted": 0.528341 + }, + { + "accuracy": 0.491594, + "f1": 0.454576, + "f1_weighted": 0.485624 + }, + { + "accuracy": 0.505716, + "f1": 0.453677, + "f1_weighted": 0.503174 + }, + { + "accuracy": 0.506052, + "f1": 0.461435, + "f1_weighted": 0.49991 + }, + { + "accuracy": 0.482851, + "f1": 0.433946, + "f1_weighted": 0.462617 + }, + { + "accuracy": 0.47848, + "f1": 0.424941, + "f1_weighted": 0.468422 + }, + { + "accuracy": 0.500336, + "f1": 0.44844, + "f1_weighted": 0.484494 + }, + { + "accuracy": 0.505044, + "f1": 0.472482, + "f1_weighted": 0.507475 + }, + { + "accuracy": 0.453934, + "f1": 0.418371, + "f1_weighted": 0.444304 + }, + { + "accuracy": 0.504035, + "f1": 0.458461, + "f1_weighted": 0.489409 + } + ], + "main_score": 0.496133, + "hf_subset": "sq", + "languages": [ + "sqi-Latn" + ] + }, + { + "accuracy": 0.081574, + "f1": 0.023271, + "f1_weighted": 0.034157, + "scores_per_experiment": [ + { + "accuracy": 0.107935, + "f1": 0.027362, + "f1_weighted": 0.046722 + }, + { + "accuracy": 0.039005, + "f1": 0.014503, + "f1_weighted": 0.011575 + }, + { + "accuracy": 0.039677, + "f1": 0.014336, + "f1_weighted": 0.01544 + }, + { + "accuracy": 0.056826, + "f1": 0.016389, + "f1_weighted": 0.01563 + }, + { + "accuracy": 0.082381, + "f1": 0.025881, + "f1_weighted": 0.03945 + }, + { + "accuracy": 0.043376, + "f1": 0.011676, + "f1_weighted": 0.012042 + }, + { + "accuracy": 0.130464, + "f1": 0.032818, + "f1_weighted": 0.062206 + }, + { + "accuracy": 0.136516, + "f1": 0.033881, + "f1_weighted": 0.0549 + }, + { + "accuracy": 0.109617, + "f1": 0.025516, + "f1_weighted": 0.044722 + }, + { + "accuracy": 0.069939, + "f1": 0.030346, + "f1_weighted": 0.038885 + } + ], + "main_score": 0.081574, + "hf_subset": "he", + "languages": [ + "heb-Hebr" + ] + }, + { + "accuracy": 0.054707, + "f1": 0.031458, + "f1_weighted": 0.024612, + "scores_per_experiment": [ + { + "accuracy": 0.046738, + "f1": 0.048265, + "f1_weighted": 0.043139 + }, + { + "accuracy": 0.055817, + "f1": 0.030027, + "f1_weighted": 0.02515 + }, + { + "accuracy": 0.057162, + "f1": 0.024212, + "f1_weighted": 0.016599 + }, + { + "accuracy": 0.081372, + "f1": 0.036376, + "f1_weighted": 0.028804 + }, + { + "accuracy": 0.040686, + "f1": 0.026412, + "f1_weighted": 0.018348 + }, + { + "accuracy": 0.043712, + "f1": 0.028267, + "f1_weighted": 0.024095 + }, + { + "accuracy": 0.03766, + "f1": 0.020553, + "f1_weighted": 0.012878 + }, + { + "accuracy": 0.082044, + "f1": 0.03514, + "f1_weighted": 0.02818 + }, + { + "accuracy": 0.04304, + "f1": 0.029934, + "f1_weighted": 0.020602 + }, + { + "accuracy": 0.058843, + "f1": 0.035392, + "f1_weighted": 0.028323 + } + ], + "main_score": 0.054707, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ] + }, + { + "accuracy": 0.06735, + "f1": 0.030117, + "f1_weighted": 0.034626, + "scores_per_experiment": [ + { + "accuracy": 0.133154, + "f1": 0.031142, + "f1_weighted": 0.047629 + }, + { + "accuracy": 0.05918, + "f1": 0.027, + "f1_weighted": 0.029577 + }, + { + "accuracy": 0.057835, + "f1": 0.031648, + "f1_weighted": 0.033494 + }, + { + "accuracy": 0.064896, + "f1": 0.034853, + "f1_weighted": 0.038952 + }, + { + "accuracy": 0.031944, + "f1": 0.015773, + "f1_weighted": 0.008351 + }, + { + "accuracy": 0.05918, + "f1": 0.034956, + "f1_weighted": 0.043019 + }, + { + "accuracy": 0.070612, + "f1": 0.044104, + "f1_weighted": 0.058731 + }, + { + "accuracy": 0.079018, + "f1": 0.030715, + "f1_weighted": 0.033606 + }, + { + "accuracy": 0.047075, + "f1": 0.026657, + "f1_weighted": 0.028422 + }, + { + "accuracy": 0.070612, + "f1": 0.024321, + "f1_weighted": 0.024483 + } + ], + "main_score": 0.06735, + "hf_subset": "te", + "languages": [ + "tel-Telu" + ] + }, + { + "accuracy": 0.079556, + "f1": 0.040826, + "f1_weighted": 0.041537, + "scores_per_experiment": [ + { + "accuracy": 0.089442, + "f1": 0.058857, + "f1_weighted": 0.070781 + }, + { + "accuracy": 0.052791, + "f1": 0.035676, + "f1_weighted": 0.02495 + }, + { + "accuracy": 0.057162, + "f1": 0.036236, + "f1_weighted": 0.032499 + }, + { + "accuracy": 0.041695, + "f1": 0.01746, + "f1_weighted": 0.008541 + }, + { + "accuracy": 0.136516, + "f1": 0.056242, + "f1_weighted": 0.06849 + }, + { + "accuracy": 0.058843, + "f1": 0.032497, + "f1_weighted": 0.030648 + }, + { + "accuracy": 0.047075, + "f1": 0.034578, + "f1_weighted": 0.020949 + }, + { + "accuracy": 0.13887, + "f1": 0.058412, + "f1_weighted": 0.070881 + }, + { + "accuracy": 0.127102, + "f1": 0.054251, + "f1_weighted": 0.073107 + }, + { + "accuracy": 0.046066, + "f1": 0.024047, + "f1_weighted": 0.014521 + } + ], + "main_score": 0.079556, + "hf_subset": "hi", + "languages": [ + "hin-Deva" + ] + }, + { + "accuracy": 0.097007, + "f1": 0.048951, + "f1_weighted": 0.056626, + "scores_per_experiment": [ + { + "accuracy": 0.14156, + "f1": 0.051506, + "f1_weighted": 0.066486 + }, + { + "accuracy": 0.056826, + "f1": 0.032925, + "f1_weighted": 0.034604 + }, + { + "accuracy": 0.070948, + "f1": 0.043374, + "f1_weighted": 0.037843 + }, + { + "accuracy": 0.127102, + "f1": 0.052936, + "f1_weighted": 0.085265 + }, + { + "accuracy": 0.076664, + "f1": 0.045014, + "f1_weighted": 0.036593 + }, + { + "accuracy": 0.065232, + "f1": 0.044742, + "f1_weighted": 0.035756 + }, + { + "accuracy": 0.05918, + "f1": 0.042835, + "f1_weighted": 0.039231 + }, + { + "accuracy": 0.140551, + "f1": 0.077704, + "f1_weighted": 0.103651 + }, + { + "accuracy": 0.119368, + "f1": 0.046426, + "f1_weighted": 0.058271 + }, + { + "accuracy": 0.112643, + "f1": 0.052047, + "f1_weighted": 0.068555 + } + ], + "main_score": 0.097007, + "hf_subset": "ur", + "languages": [ + "urd-Arab" + ] + }, + { + "accuracy": 0.546806, + "f1": 0.528859, + "f1_weighted": 0.535907, + "scores_per_experiment": [ + { + "accuracy": 0.56456, + "f1": 0.552732, + "f1_weighted": 0.555407 + }, + { + "accuracy": 0.543712, + "f1": 0.53069, + "f1_weighted": 0.533491 + }, + { + "accuracy": 0.543712, + "f1": 0.512311, + "f1_weighted": 0.533045 + }, + { + "accuracy": 0.512441, + "f1": 0.501919, + "f1_weighted": 0.497353 + }, + { + "accuracy": 0.56456, + "f1": 0.54352, + "f1_weighted": 0.552378 + }, + { + "accuracy": 0.531944, + "f1": 0.51315, + "f1_weighted": 0.519572 + }, + { + "accuracy": 0.557498, + "f1": 0.531441, + "f1_weighted": 0.543403 + }, + { + "accuracy": 0.579018, + "f1": 0.565853, + "f1_weighted": 0.576091 + }, + { + "accuracy": 0.542367, + "f1": 0.527942, + "f1_weighted": 0.528842 + }, + { + "accuracy": 0.528245, + "f1": 0.509032, + "f1_weighted": 0.519484 + } + ], + "main_score": 0.546806, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ] + }, + { + "accuracy": 0.457196, + "f1": 0.434586, + "f1_weighted": 0.449552, + "scores_per_experiment": [ + { + "accuracy": 0.471755, + "f1": 0.444514, + "f1_weighted": 0.46774 + }, + { + "accuracy": 0.4462, + "f1": 0.424107, + "f1_weighted": 0.440571 + }, + { + "accuracy": 0.48655, + "f1": 0.454339, + "f1_weighted": 0.480135 + }, + { + "accuracy": 0.464358, + "f1": 0.445412, + "f1_weighted": 0.458712 + }, + { + "accuracy": 0.460659, + "f1": 0.440129, + "f1_weighted": 0.459736 + }, + { + "accuracy": 0.463013, + "f1": 0.43714, + "f1_weighted": 0.454829 + }, + { + "accuracy": 0.44351, + "f1": 0.427282, + "f1_weighted": 0.435037 + }, + { + "accuracy": 0.471755, + "f1": 0.449687, + "f1_weighted": 0.463222 + }, + { + "accuracy": 0.418964, + "f1": 0.409083, + "f1_weighted": 0.401893 + }, + { + "accuracy": 0.445192, + "f1": 0.414162, + "f1_weighted": 0.43364 + } + ], + "main_score": 0.457196, + "hf_subset": "id", + "languages": [ + "ind-Latn" + ] + }, + { + "accuracy": 0.081944, + "f1": 0.045125, + "f1_weighted": 0.044733, + "scores_per_experiment": [ + { + "accuracy": 0.04842, + "f1": 0.034675, + "f1_weighted": 0.024195 + }, + { + "accuracy": 0.118023, + "f1": 0.064052, + "f1_weighted": 0.074863 + }, + { + "accuracy": 0.0538, + "f1": 0.032074, + "f1_weighted": 0.025304 + }, + { + "accuracy": 0.113988, + "f1": 0.049169, + "f1_weighted": 0.064986 + }, + { + "accuracy": 0.116678, + "f1": 0.057015, + "f1_weighted": 0.059658 + }, + { + "accuracy": 0.071957, + "f1": 0.051417, + "f1_weighted": 0.048672 + }, + { + "accuracy": 0.080363, + "f1": 0.048051, + "f1_weighted": 0.043962 + }, + { + "accuracy": 0.075992, + "f1": 0.045433, + "f1_weighted": 0.039804 + }, + { + "accuracy": 0.055817, + "f1": 0.03419, + "f1_weighted": 0.029475 + }, + { + "accuracy": 0.084398, + "f1": 0.035177, + "f1_weighted": 0.036409 + } + ], + "main_score": 0.081944, + "hf_subset": "th", + "languages": [ + "tha-Thai" + ] + }, + { + "accuracy": 0.414223, + "f1": 0.370067, + "f1_weighted": 0.411556, + "scores_per_experiment": [ + { + "accuracy": 0.381305, + "f1": 0.368961, + "f1_weighted": 0.375465 + }, + { + "accuracy": 0.416947, + "f1": 0.370005, + "f1_weighted": 0.410613 + }, + { + "accuracy": 0.432414, + "f1": 0.382538, + "f1_weighted": 0.433891 + }, + { + "accuracy": 0.419301, + "f1": 0.3618, + "f1_weighted": 0.407194 + }, + { + "accuracy": 0.403833, + "f1": 0.356083, + "f1_weighted": 0.404297 + }, + { + "accuracy": 0.401479, + "f1": 0.364831, + "f1_weighted": 0.396383 + }, + { + "accuracy": 0.44889, + "f1": 0.400563, + "f1_weighted": 0.45592 + }, + { + "accuracy": 0.437794, + "f1": 0.396694, + "f1_weighted": 0.440733 + }, + { + "accuracy": 0.391392, + "f1": 0.336782, + "f1_weighted": 0.393083 + }, + { + "accuracy": 0.408877, + "f1": 0.362417, + "f1_weighted": 0.397983 + } + ], + "main_score": 0.414223, + "hf_subset": "fi", + "languages": [ + "fin-Latn" + ] + }, + { + "accuracy": 0.451345, + "f1": 0.403109, + "f1_weighted": 0.432027, + "scores_per_experiment": [ + { + "accuracy": 0.457633, + "f1": 0.41174, + "f1_weighted": 0.441665 + }, + { + "accuracy": 0.470746, + "f1": 0.411993, + "f1_weighted": 0.447591 + }, + { + "accuracy": 0.472764, + "f1": 0.421885, + "f1_weighted": 0.45274 + }, + { + "accuracy": 0.438467, + "f1": 0.388942, + "f1_weighted": 0.416522 + }, + { + "accuracy": 0.473436, + "f1": 0.412152, + "f1_weighted": 0.449107 + }, + { + "accuracy": 0.434095, + "f1": 0.389272, + "f1_weighted": 0.415708 + }, + { + "accuracy": 0.465367, + "f1": 0.408201, + "f1_weighted": 0.448583 + }, + { + "accuracy": 0.46503, + "f1": 0.420216, + "f1_weighted": 0.447431 + }, + { + "accuracy": 0.401143, + "f1": 0.375364, + "f1_weighted": 0.385486 + }, + { + "accuracy": 0.434768, + "f1": 0.391324, + "f1_weighted": 0.415435 + } + ], + "main_score": 0.451345, + "hf_subset": "ms", + "languages": [ + "msa-Latn" + ] + }, + { + "accuracy": 0.099765, + "f1": 0.036403, + "f1_weighted": 0.049606, + "scores_per_experiment": [ + { + "accuracy": 0.142569, + "f1": 0.050274, + "f1_weighted": 0.076475 + }, + { + "accuracy": 0.104573, + "f1": 0.032886, + "f1_weighted": 0.043934 + }, + { + "accuracy": 0.051446, + "f1": 0.026002, + "f1_weighted": 0.023994 + }, + { + "accuracy": 0.105245, + "f1": 0.037165, + "f1_weighted": 0.04409 + }, + { + "accuracy": 0.068931, + "f1": 0.032009, + "f1_weighted": 0.043751 + }, + { + "accuracy": 0.052791, + "f1": 0.030683, + "f1_weighted": 0.031294 + }, + { + "accuracy": 0.159045, + "f1": 0.041652, + "f1_weighted": 0.075101 + }, + { + "accuracy": 0.149966, + "f1": 0.048484, + "f1_weighted": 0.068921 + }, + { + "accuracy": 0.047747, + "f1": 0.021453, + "f1_weighted": 0.023717 + }, + { + "accuracy": 0.115333, + "f1": 0.043422, + "f1_weighted": 0.064785 + } + ], + "main_score": 0.099765, + "hf_subset": "km", + "languages": [ + "khm-Khmr" + ] + }, + { + "accuracy": 0.081876, + "f1": 0.033104, + "f1_weighted": 0.037729, + "scores_per_experiment": [ + { + "accuracy": 0.132145, + "f1": 0.03657, + "f1_weighted": 0.054353 + }, + { + "accuracy": 0.056826, + "f1": 0.020915, + "f1_weighted": 0.016873 + }, + { + "accuracy": 0.056826, + "f1": 0.032901, + "f1_weighted": 0.025333 + }, + { + "accuracy": 0.057162, + "f1": 0.028054, + "f1_weighted": 0.020674 + }, + { + "accuracy": 0.083726, + "f1": 0.045288, + "f1_weighted": 0.038731 + }, + { + "accuracy": 0.091796, + "f1": 0.040633, + "f1_weighted": 0.043923 + }, + { + "accuracy": 0.053463, + "f1": 0.026005, + "f1_weighted": 0.032033 + }, + { + "accuracy": 0.126429, + "f1": 0.042936, + "f1_weighted": 0.067382 + }, + { + "accuracy": 0.113988, + "f1": 0.037944, + "f1_weighted": 0.065509 + }, + { + "accuracy": 0.046402, + "f1": 0.01979, + "f1_weighted": 0.012478 + } + ], + "main_score": 0.081876, + "hf_subset": "am", + "languages": [ + "amh-Ethi" + ] + } + ] + }, + "evaluation_time": 366.8217399120331, + "kg_co2_emissions": null } \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/MedrxivClusteringP2P.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/MedrxivClusteringP2P.json deleted file mode 100644 index c6bf53df87..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/MedrxivClusteringP2P.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "test": { - "evaluation_time": 436.35, - "v_measure": 0.2756514819237031, - "v_measure_std": 0.013239017881101744 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "MedrxivClusteringP2P", - "dataset_revision": "dcefc037ef84348e49b0d29109e891c01067226b" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/MedrxivClusteringS2S.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/MedrxivClusteringS2S.json deleted file mode 100644 index c175761474..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/MedrxivClusteringS2S.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "test": { - "evaluation_time": 39.25, - "v_measure": 0.2513156119229876, - "v_measure_std": 0.012367155188568199 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "MedrxivClusteringS2S", - "dataset_revision": "3cd0e71dfbe09d4de0f9e5ecba43e7ce280959dc" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/MindSmallReranking.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/MindSmallReranking.json deleted file mode 100644 index 66889bd3f4..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/MindSmallReranking.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "mteb_version": "0.0.2", - "test": { - "evaluation_time": 2107.84, - "map": 0.313303339508014, - "mrr": 0.32430976502172476 - }, - "mteb_dataset_name": "MindSmallReranking", - "dataset_revision": "3bdac13927fdc888b903db93b2ffdbd90b295a69" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/Moroco.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/Moroco.json new file mode 100644 index 0000000000..5540ced5e8 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/Moroco.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "d64d9b8cd876056a5c24552afe3caf7e6fd26c8e", + "task_name": "Moroco", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.645752, + "f1": 0.623382, + "f1_weighted": 0.641272, + "scores_per_experiment": [ + { + "accuracy": 0.693848, + "f1": 0.671961, + "f1_weighted": 0.698265 + }, + { + "accuracy": 0.674805, + "f1": 0.650969, + "f1_weighted": 0.671206 + }, + { + "accuracy": 0.534668, + "f1": 0.528277, + "f1_weighted": 0.525934 + }, + { + "accuracy": 0.644531, + "f1": 0.621909, + "f1_weighted": 0.639608 + }, + { + "accuracy": 0.690918, + "f1": 0.666756, + "f1_weighted": 0.688124 + }, + { + "accuracy": 0.649414, + "f1": 0.626425, + "f1_weighted": 0.638734 + }, + { + "accuracy": 0.675293, + "f1": 0.655897, + "f1_weighted": 0.673072 + }, + { + "accuracy": 0.629395, + "f1": 0.590676, + "f1_weighted": 0.623945 + }, + { + "accuracy": 0.633301, + "f1": 0.614273, + "f1_weighted": 0.635526 + }, + { + "accuracy": 0.631348, + "f1": 0.606676, + "f1_weighted": 0.618302 + } + ], + "main_score": 0.645752, + "hf_subset": "default", + "languages": [ + "ron-Latn" + ] + } + ] + }, + "evaluation_time": 23.392173528671265, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/MovieReviewSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/MovieReviewSentimentClassification.json new file mode 100644 index 0000000000..5c8d28dbc6 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/MovieReviewSentimentClassification.json @@ -0,0 +1,181 @@ +{ + "dataset_revision": "a4654f4896408912913a62ace89614879a549287", + "task_name": "MovieReviewSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "validation": [ + { + "accuracy": 0.574268, + "f1": 0.569661, + "f1_weighted": 0.569982, + "ap": 0.532957, + "ap_weighted": 0.532957, + "scores_per_experiment": [ + { + "accuracy": 0.585449, + "f1": 0.583586, + "f1_weighted": 0.584157, + "ap": 0.54007, + "ap_weighted": 0.54007 + }, + { + "accuracy": 0.54834, + "f1": 0.543494, + "f1_weighted": 0.54253, + "ap": 0.517214, + "ap_weighted": 0.517214 + }, + { + "accuracy": 0.568359, + "f1": 0.568339, + "f1_weighted": 0.5684, + "ap": 0.528709, + "ap_weighted": 0.528709 + }, + { + "accuracy": 0.573242, + "f1": 0.564387, + "f1_weighted": 0.565661, + "ap": 0.531916, + "ap_weighted": 0.531916 + }, + { + "accuracy": 0.599121, + "f1": 0.595377, + "f1_weighted": 0.594579, + "ap": 0.549036, + "ap_weighted": 0.549036 + }, + { + "accuracy": 0.575195, + "f1": 0.57426, + "f1_weighted": 0.574669, + "ap": 0.533073, + "ap_weighted": 0.533073 + }, + { + "accuracy": 0.552246, + "f1": 0.550678, + "f1_weighted": 0.550133, + "ap": 0.519163, + "ap_weighted": 0.519163 + }, + { + "accuracy": 0.580566, + "f1": 0.579662, + "f1_weighted": 0.580062, + "ap": 0.536671, + "ap_weighted": 0.536671 + }, + { + "accuracy": 0.571289, + "f1": 0.553207, + "f1_weighted": 0.555051, + "ap": 0.53093, + "ap_weighted": 0.53093 + }, + { + "accuracy": 0.588867, + "f1": 0.583616, + "f1_weighted": 0.584575, + "ap": 0.542792, + "ap_weighted": 0.542792 + } + ], + "main_score": 0.574268, + "hf_subset": "default", + "languages": [ + "fra-Latn" + ] + } + ], + "test": [ + { + "accuracy": 0.57627, + "f1": 0.571775, + "f1_weighted": 0.572573, + "ap": 0.523904, + "ap_weighted": 0.523904, + "scores_per_experiment": [ + { + "accuracy": 0.604004, + "f1": 0.602009, + "f1_weighted": 0.603165, + "ap": 0.542511, + "ap_weighted": 0.542511 + }, + { + "accuracy": 0.557617, + "f1": 0.553443, + "f1_weighted": 0.551672, + "ap": 0.513996, + "ap_weighted": 0.513996 + }, + { + "accuracy": 0.577637, + "f1": 0.577184, + "f1_weighted": 0.577752, + "ap": 0.52442, + "ap_weighted": 0.52442 + }, + { + "accuracy": 0.575684, + "f1": 0.567304, + "f1_weighted": 0.569774, + "ap": 0.521757, + "ap_weighted": 0.521757 + }, + { + "accuracy": 0.588867, + "f1": 0.586279, + "f1_weighted": 0.584936, + "ap": 0.533482, + "ap_weighted": 0.533482 + }, + { + "accuracy": 0.553711, + "f1": 0.552557, + "f1_weighted": 0.553489, + "ap": 0.508783, + "ap_weighted": 0.508783 + }, + { + "accuracy": 0.556641, + "f1": 0.555783, + "f1_weighted": 0.554982, + "ap": 0.512381, + "ap_weighted": 0.512381 + }, + { + "accuracy": 0.568359, + "f1": 0.566274, + "f1_weighted": 0.567508, + "ap": 0.517759, + "ap_weighted": 0.517759 + }, + { + "accuracy": 0.583496, + "f1": 0.565661, + "f1_weighted": 0.569271, + "ap": 0.526986, + "ap_weighted": 0.526986 + }, + { + "accuracy": 0.59668, + "f1": 0.591252, + "f1_weighted": 0.593184, + "ap": 0.536967, + "ap_weighted": 0.536967 + } + ], + "main_score": 0.57627, + "hf_subset": "default", + "languages": [ + "fra-Latn" + ] + } + ] + }, + "evaluation_time": 26.955711364746094, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/MultiHateClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/MultiHateClassification.json new file mode 100644 index 0000000000..bba9148d0b --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/MultiHateClassification.json @@ -0,0 +1,935 @@ +{ + "dataset_revision": "8f95949846bb9e33c6aaf730ccfdb8fe6bcfb7a9", + "task_name": "MultiHateClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.6083, + "f1": 0.573701, + "f1_weighted": 0.616123, + "ap": 0.36672, + "ap_weighted": 0.36672, + "scores_per_experiment": [ + { + "accuracy": 0.624, + "f1": 0.596288, + "f1_weighted": 0.635635, + "ap": 0.379072, + "ap_weighted": 0.379072 + }, + { + "accuracy": 0.677, + "f1": 0.620792, + "f1_weighted": 0.675102, + "ap": 0.393533, + "ap_weighted": 0.393533 + }, + { + "accuracy": 0.689, + "f1": 0.630813, + "f1_weighted": 0.685336, + "ap": 0.402897, + "ap_weighted": 0.402897 + }, + { + "accuracy": 0.605, + "f1": 0.581493, + "f1_weighted": 0.61839, + "ap": 0.370137, + "ap_weighted": 0.370137 + }, + { + "accuracy": 0.586, + "f1": 0.573976, + "f1_weighted": 0.600601, + "ap": 0.373555, + "ap_weighted": 0.373555 + }, + { + "accuracy": 0.551, + "f1": 0.522328, + "f1_weighted": 0.565863, + "ap": 0.33005, + "ap_weighted": 0.33005 + }, + { + "accuracy": 0.545, + "f1": 0.538011, + "f1_weighted": 0.559149, + "ap": 0.354841, + "ap_weighted": 0.354841 + }, + { + "accuracy": 0.697, + "f1": 0.617895, + "f1_weighted": 0.68257, + "ap": 0.39485, + "ap_weighted": 0.39485 + }, + { + "accuracy": 0.535, + "f1": 0.513333, + "f1_weighted": 0.551533, + "ap": 0.327957, + "ap_weighted": 0.327957 + }, + { + "accuracy": 0.574, + "f1": 0.542085, + "f1_weighted": 0.587056, + "ap": 0.34031, + "ap_weighted": 0.34031 + } + ], + "main_score": 0.6083, + "hf_subset": "eng", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.5445, + "f1": 0.528746, + "f1_weighted": 0.561098, + "ap": 0.333268, + "ap_weighted": 0.333268, + "scores_per_experiment": [ + { + "accuracy": 0.534, + "f1": 0.523564, + "f1_weighted": 0.551628, + "ap": 0.332593, + "ap_weighted": 0.332593 + }, + { + "accuracy": 0.623, + "f1": 0.587595, + "f1_weighted": 0.635688, + "ap": 0.35878, + "ap_weighted": 0.35878 + }, + { + "accuracy": 0.494, + "f1": 0.487098, + "f1_weighted": 0.510778, + "ap": 0.315558, + "ap_weighted": 0.315558 + }, + { + "accuracy": 0.601, + "f1": 0.569374, + "f1_weighted": 0.615821, + "ap": 0.347328, + "ap_weighted": 0.347328 + }, + { + "accuracy": 0.518, + "f1": 0.512523, + "f1_weighted": 0.533088, + "ap": 0.332746, + "ap_weighted": 0.332746 + }, + { + "accuracy": 0.508, + "f1": 0.498728, + "f1_weighted": 0.525861, + "ap": 0.319151, + "ap_weighted": 0.319151 + }, + { + "accuracy": 0.502, + "f1": 0.494724, + "f1_weighted": 0.518856, + "ap": 0.319325, + "ap_weighted": 0.319325 + }, + { + "accuracy": 0.565, + "f1": 0.545526, + "f1_weighted": 0.582969, + "ap": 0.338359, + "ap_weighted": 0.338359 + }, + { + "accuracy": 0.547, + "f1": 0.52753, + "f1_weighted": 0.565703, + "ap": 0.327157, + "ap_weighted": 0.327157 + }, + { + "accuracy": 0.553, + "f1": 0.5408, + "f1_weighted": 0.57059, + "ap": 0.341688, + "ap_weighted": 0.341688 + } + ], + "main_score": 0.5445, + "hf_subset": "fra", + "languages": [ + "fra-Latn" + ] + }, + { + "accuracy": 0.5983, + "f1": 0.482358, + "f1_weighted": 0.548662, + "ap": 0.344239, + "ap_weighted": 0.344239, + "scores_per_experiment": [ + { + "accuracy": 0.39, + "f1": 0.37004, + "f1_weighted": 0.331242, + "ap": 0.332841, + "ap_weighted": 0.332841 + }, + { + "accuracy": 0.672, + "f1": 0.514146, + "f1_weighted": 0.609966, + "ap": 0.353225, + "ap_weighted": 0.353225 + }, + { + "accuracy": 0.694, + "f1": 0.54364, + "f1_weighted": 0.634275, + "ap": 0.378204, + "ap_weighted": 0.378204 + }, + { + "accuracy": 0.675, + "f1": 0.516132, + "f1_weighted": 0.612063, + "ap": 0.355541, + "ap_weighted": 0.355541 + }, + { + "accuracy": 0.607, + "f1": 0.515279, + "f1_weighted": 0.588234, + "ap": 0.336033, + "ap_weighted": 0.336033 + }, + { + "accuracy": 0.673, + "f1": 0.402271, + "f1_weighted": 0.541457, + "ap": 0.327, + "ap_weighted": 0.327 + }, + { + "accuracy": 0.579, + "f1": 0.50484, + "f1_weighted": 0.571143, + "ap": 0.329544, + "ap_weighted": 0.329544 + }, + { + "accuracy": 0.45, + "f1": 0.443509, + "f1_weighted": 0.422714, + "ap": 0.352796, + "ap_weighted": 0.352796 + }, + { + "accuracy": 0.579, + "f1": 0.50484, + "f1_weighted": 0.571143, + "ap": 0.329544, + "ap_weighted": 0.329544 + }, + { + "accuracy": 0.664, + "f1": 0.508884, + "f1_weighted": 0.604382, + "ap": 0.347659, + "ap_weighted": 0.347659 + } + ], + "main_score": 0.5983, + "hf_subset": "cmn", + "languages": [ + "cmn-Hans" + ] + }, + { + "accuracy": 0.411, + "f1": 0.386514, + "f1_weighted": 0.360726, + "ap": 0.315491, + "ap_weighted": 0.315491, + "scores_per_experiment": [ + { + "accuracy": 0.533, + "f1": 0.474756, + "f1_weighted": 0.544369, + "ap": 0.291772, + "ap_weighted": 0.291772 + }, + { + "accuracy": 0.512, + "f1": 0.496401, + "f1_weighted": 0.531677, + "ap": 0.312288, + "ap_weighted": 0.312288 + }, + { + "accuracy": 0.374, + "f1": 0.351567, + "f1_weighted": 0.303565, + "ap": 0.315311, + "ap_weighted": 0.315311 + }, + { + "accuracy": 0.38, + "f1": 0.34786, + "f1_weighted": 0.290239, + "ap": 0.326819, + "ap_weighted": 0.326819 + }, + { + "accuracy": 0.373, + "f1": 0.35028, + "f1_weighted": 0.301924, + "ap": 0.314979, + "ap_weighted": 0.314979 + }, + { + "accuracy": 0.365, + "f1": 0.350582, + "f1_weighted": 0.312071, + "ap": 0.302764, + "ap_weighted": 0.302764 + }, + { + "accuracy": 0.403, + "f1": 0.398447, + "f1_weighted": 0.377617, + "ap": 0.309713, + "ap_weighted": 0.309713 + }, + { + "accuracy": 0.417, + "f1": 0.397253, + "f1_weighted": 0.353833, + "ap": 0.339515, + "ap_weighted": 0.339515 + }, + { + "accuracy": 0.378, + "f1": 0.345142, + "f1_weighted": 0.28676, + "ap": 0.326111, + "ap_weighted": 0.326111 + }, + { + "accuracy": 0.375, + "f1": 0.352851, + "f1_weighted": 0.305202, + "ap": 0.315644, + "ap_weighted": 0.315644 + } + ], + "main_score": 0.411, + "hf_subset": "ara", + "languages": [ + "ara-Arab" + ] + }, + { + "accuracy": 0.5561, + "f1": 0.522963, + "f1_weighted": 0.566261, + "ap": 0.32724, + "ap_weighted": 0.32724, + "scores_per_experiment": [ + { + "accuracy": 0.519, + "f1": 0.50716, + "f1_weighted": 0.53741, + "ap": 0.321993, + "ap_weighted": 0.321993 + }, + { + "accuracy": 0.522, + "f1": 0.515016, + "f1_weighted": 0.538063, + "ap": 0.33245, + "ap_weighted": 0.33245 + }, + { + "accuracy": 0.557, + "f1": 0.519606, + "f1_weighted": 0.572682, + "ap": 0.315689, + "ap_weighted": 0.315689 + }, + { + "accuracy": 0.507, + "f1": 0.502883, + "f1_weighted": 0.520797, + "ap": 0.33001, + "ap_weighted": 0.33001 + }, + { + "accuracy": 0.524, + "f1": 0.512437, + "f1_weighted": 0.54217, + "ap": 0.325323, + "ap_weighted": 0.325323 + }, + { + "accuracy": 0.58, + "f1": 0.561007, + "f1_weighted": 0.597166, + "ap": 0.349788, + "ap_weighted": 0.349788 + }, + { + "accuracy": 0.664, + "f1": 0.554367, + "f1_weighted": 0.641897, + "ap": 0.333665, + "ap_weighted": 0.333665 + }, + { + "accuracy": 0.514, + "f1": 0.500902, + "f1_weighted": 0.53292, + "ap": 0.317361, + "ap_weighted": 0.317361 + }, + { + "accuracy": 0.664, + "f1": 0.569769, + "f1_weighted": 0.649503, + "ap": 0.342267, + "ap_weighted": 0.342267 + }, + { + "accuracy": 0.51, + "f1": 0.486483, + "f1_weighted": 0.53, + "ap": 0.303855, + "ap_weighted": 0.303855 + } + ], + "main_score": 0.5561, + "hf_subset": "hin", + "languages": [ + "hin-Deva" + ] + }, + { + "accuracy": 0.5423, + "f1": 0.513248, + "f1_weighted": 0.557698, + "ap": 0.31702, + "ap_weighted": 0.31702, + "scores_per_experiment": [ + { + "accuracy": 0.555, + "f1": 0.528517, + "f1_weighted": 0.573214, + "ap": 0.323009, + "ap_weighted": 0.323009 + }, + { + "accuracy": 0.575, + "f1": 0.53968, + "f1_weighted": 0.590684, + "ap": 0.325949, + "ap_weighted": 0.325949 + }, + { + "accuracy": 0.531, + "f1": 0.498927, + "f1_weighted": 0.549635, + "ap": 0.305079, + "ap_weighted": 0.305079 + }, + { + "accuracy": 0.491, + "f1": 0.489341, + "f1_weighted": 0.500984, + "ap": 0.327111, + "ap_weighted": 0.327111 + }, + { + "accuracy": 0.609, + "f1": 0.550409, + "f1_weighted": 0.61533, + "ap": 0.32719, + "ap_weighted": 0.32719 + }, + { + "accuracy": 0.509, + "f1": 0.49904, + "f1_weighted": 0.527295, + "ap": 0.317884, + "ap_weighted": 0.317884 + }, + { + "accuracy": 0.563, + "f1": 0.518179, + "f1_weighted": 0.576961, + "ap": 0.31143, + "ap_weighted": 0.31143 + }, + { + "accuracy": 0.533, + "f1": 0.495985, + "f1_weighted": 0.55062, + "ap": 0.302341, + "ap_weighted": 0.302341 + }, + { + "accuracy": 0.513, + "f1": 0.503685, + "f1_weighted": 0.530883, + "ap": 0.32126, + "ap_weighted": 0.32126 + }, + { + "accuracy": 0.544, + "f1": 0.508714, + "f1_weighted": 0.56138, + "ap": 0.308944, + "ap_weighted": 0.308944 + } + ], + "main_score": 0.5423, + "hf_subset": "ita", + "languages": [ + "ita-Latn" + ] + }, + { + "accuracy": 0.5309, + "f1": 0.508486, + "f1_weighted": 0.546821, + "ap": 0.317412, + "ap_weighted": 0.317412, + "scores_per_experiment": [ + { + "accuracy": 0.582, + "f1": 0.534314, + "f1_weighted": 0.594518, + "ap": 0.317752, + "ap_weighted": 0.317752 + }, + { + "accuracy": 0.579, + "f1": 0.533275, + "f1_weighted": 0.592294, + "ap": 0.317594, + "ap_weighted": 0.317594 + }, + { + "accuracy": 0.479, + "f1": 0.475386, + "f1_weighted": 0.492977, + "ap": 0.312532, + "ap_weighted": 0.312532 + }, + { + "accuracy": 0.541, + "f1": 0.530578, + "f1_weighted": 0.558836, + "ap": 0.335115, + "ap_weighted": 0.335115 + }, + { + "accuracy": 0.501, + "f1": 0.469685, + "f1_weighted": 0.521747, + "ap": 0.290661, + "ap_weighted": 0.290661 + }, + { + "accuracy": 0.569, + "f1": 0.53852, + "f1_weighted": 0.586434, + "ap": 0.325516, + "ap_weighted": 0.325516 + }, + { + "accuracy": 0.495, + "f1": 0.490783, + "f1_weighted": 0.509504, + "ap": 0.320102, + "ap_weighted": 0.320102 + }, + { + "accuracy": 0.488, + "f1": 0.486389, + "f1_weighted": 0.498009, + "ap": 0.324359, + "ap_weighted": 0.324359 + }, + { + "accuracy": 0.556, + "f1": 0.537112, + "f1_weighted": 0.574888, + "ap": 0.331262, + "ap_weighted": 0.331262 + }, + { + "accuracy": 0.519, + "f1": 0.488815, + "f1_weighted": 0.538999, + "ap": 0.299229, + "ap_weighted": 0.299229 + } + ], + "main_score": 0.5309, + "hf_subset": "pol", + "languages": [ + "pol-Latn" + ] + }, + { + "accuracy": 0.5642, + "f1": 0.527637, + "f1_weighted": 0.575787, + "ap": 0.323105, + "ap_weighted": 0.323105, + "scores_per_experiment": [ + { + "accuracy": 0.564, + "f1": 0.530801, + "f1_weighted": 0.581723, + "ap": 0.318241, + "ap_weighted": 0.318241 + }, + { + "accuracy": 0.637, + "f1": 0.583978, + "f1_weighted": 0.644575, + "ap": 0.346825, + "ap_weighted": 0.346825 + }, + { + "accuracy": 0.496, + "f1": 0.495417, + "f1_weighted": 0.502416, + "ap": 0.334065, + "ap_weighted": 0.334065 + }, + { + "accuracy": 0.578, + "f1": 0.538665, + "f1_weighted": 0.593626, + "ap": 0.320681, + "ap_weighted": 0.320681 + }, + { + "accuracy": 0.543, + "f1": 0.51014, + "f1_weighted": 0.561904, + "ap": 0.307082, + "ap_weighted": 0.307082 + }, + { + "accuracy": 0.616, + "f1": 0.563088, + "f1_weighted": 0.625123, + "ap": 0.332399, + "ap_weighted": 0.332399 + }, + { + "accuracy": 0.503, + "f1": 0.498474, + "f1_weighted": 0.517913, + "ap": 0.322861, + "ap_weighted": 0.322861 + }, + { + "accuracy": 0.552, + "f1": 0.538988, + "f1_weighted": 0.570588, + "ap": 0.336352, + "ap_weighted": 0.336352 + }, + { + "accuracy": 0.591, + "f1": 0.483032, + "f1_weighted": 0.579424, + "ap": 0.290399, + "ap_weighted": 0.290399 + }, + { + "accuracy": 0.562, + "f1": 0.533787, + "f1_weighted": 0.58058, + "ap": 0.322144, + "ap_weighted": 0.322144 + } + ], + "main_score": 0.5642, + "hf_subset": "spa", + "languages": [ + "spa-Latn" + ] + }, + { + "accuracy": 0.5473, + "f1": 0.515197, + "f1_weighted": 0.562174, + "ap": 0.319718, + "ap_weighted": 0.319718, + "scores_per_experiment": [ + { + "accuracy": 0.536, + "f1": 0.500669, + "f1_weighted": 0.553267, + "ap": 0.306694, + "ap_weighted": 0.306694 + }, + { + "accuracy": 0.585, + "f1": 0.549422, + "f1_weighted": 0.59956, + "ap": 0.333461, + "ap_weighted": 0.333461 + }, + { + "accuracy": 0.506, + "f1": 0.502656, + "f1_weighted": 0.518806, + "ap": 0.331593, + "ap_weighted": 0.331593 + }, + { + "accuracy": 0.551, + "f1": 0.541357, + "f1_weighted": 0.567692, + "ap": 0.345752, + "ap_weighted": 0.345752 + }, + { + "accuracy": 0.542, + "f1": 0.472049, + "f1_weighted": 0.54815, + "ap": 0.291528, + "ap_weighted": 0.291528 + }, + { + "accuracy": 0.589, + "f1": 0.549239, + "f1_weighted": 0.602254, + "ap": 0.332017, + "ap_weighted": 0.332017 + }, + { + "accuracy": 0.524, + "f1": 0.474825, + "f1_weighted": 0.538463, + "ap": 0.29336, + "ap_weighted": 0.29336 + }, + { + "accuracy": 0.567, + "f1": 0.535347, + "f1_weighted": 0.583372, + "ap": 0.326376, + "ap_weighted": 0.326376 + }, + { + "accuracy": 0.564, + "f1": 0.536399, + "f1_weighted": 0.581194, + "ap": 0.328714, + "ap_weighted": 0.328714 + }, + { + "accuracy": 0.509, + "f1": 0.490003, + "f1_weighted": 0.528981, + "ap": 0.307688, + "ap_weighted": 0.307688 + } + ], + "main_score": 0.5473, + "hf_subset": "por", + "languages": [ + "por-Latn" + ] + }, + { + "accuracy": 0.5385, + "f1": 0.504849, + "f1_weighted": 0.550878, + "ap": 0.311763, + "ap_weighted": 0.311763, + "scores_per_experiment": [ + { + "accuracy": 0.591, + "f1": 0.529712, + "f1_weighted": 0.5983, + "ap": 0.313144, + "ap_weighted": 0.313144 + }, + { + "accuracy": 0.438, + "f1": 0.437099, + "f1_weighted": 0.428003, + "ap": 0.31733, + "ap_weighted": 0.31733 + }, + { + "accuracy": 0.621, + "f1": 0.555848, + "f1_weighted": 0.624572, + "ap": 0.327966, + "ap_weighted": 0.327966 + }, + { + "accuracy": 0.52, + "f1": 0.494164, + "f1_weighted": 0.540349, + "ap": 0.303339, + "ap_weighted": 0.303339 + }, + { + "accuracy": 0.517, + "f1": 0.494494, + "f1_weighted": 0.537586, + "ap": 0.304987, + "ap_weighted": 0.304987 + }, + { + "accuracy": 0.49, + "f1": 0.478261, + "f1_weighted": 0.509878, + "ap": 0.303688, + "ap_weighted": 0.303688 + }, + { + "accuracy": 0.562, + "f1": 0.512032, + "f1_weighted": 0.575117, + "ap": 0.305615, + "ap_weighted": 0.305615 + }, + { + "accuracy": 0.532, + "f1": 0.5156, + "f1_weighted": 0.551609, + "ap": 0.320134, + "ap_weighted": 0.320134 + }, + { + "accuracy": 0.544, + "f1": 0.518995, + "f1_weighted": 0.563302, + "ap": 0.316734, + "ap_weighted": 0.316734 + }, + { + "accuracy": 0.57, + "f1": 0.512286, + "f1_weighted": 0.580066, + "ap": 0.30469, + "ap_weighted": 0.30469 + } + ], + "main_score": 0.5385, + "hf_subset": "nld", + "languages": [ + "nld-Latn" + ] + }, + { + "accuracy": 0.5454, + "f1": 0.524812, + "f1_weighted": 0.558675, + "ap": 0.33246, + "ap_weighted": 0.33246, + "scores_per_experiment": [ + { + "accuracy": 0.542, + "f1": 0.536182, + "f1_weighted": 0.556961, + "ap": 0.347046, + "ap_weighted": 0.347046 + }, + { + "accuracy": 0.563, + "f1": 0.540131, + "f1_weighted": 0.581151, + "ap": 0.331957, + "ap_weighted": 0.331957 + }, + { + "accuracy": 0.593, + "f1": 0.575507, + "f1_weighted": 0.609976, + "ap": 0.36003, + "ap_weighted": 0.36003 + }, + { + "accuracy": 0.566, + "f1": 0.499714, + "f1_weighted": 0.572556, + "ap": 0.3002, + "ap_weighted": 0.3002 + }, + { + "accuracy": 0.457, + "f1": 0.456878, + "f1_weighted": 0.460137, + "ap": 0.316669, + "ap_weighted": 0.316669 + }, + { + "accuracy": 0.492, + "f1": 0.482138, + "f1_weighted": 0.510724, + "ap": 0.308819, + "ap_weighted": 0.308819 + }, + { + "accuracy": 0.537, + "f1": 0.531433, + "f1_weighted": 0.551862, + "ap": 0.344297, + "ap_weighted": 0.344297 + }, + { + "accuracy": 0.595, + "f1": 0.535081, + "f1_weighted": 0.601843, + "ap": 0.318109, + "ap_weighted": 0.318109 + }, + { + "accuracy": 0.538, + "f1": 0.531476, + "f1_weighted": 0.553591, + "ap": 0.342577, + "ap_weighted": 0.342577 + }, + { + "accuracy": 0.571, + "f1": 0.559584, + "f1_weighted": 0.587947, + "ap": 0.354895, + "ap_weighted": 0.354895 + } + ], + "main_score": 0.5454, + "hf_subset": "deu", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 16.24806022644043, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/MultilingualSentiment.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/MultilingualSentiment.json new file mode 100644 index 0000000000..ca18e83d69 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/MultilingualSentiment.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "46958b007a63fdbf239b7672c25d0bea67b5ea1a", + "task_name": "MultilingualSentiment", + "mteb_version": "1.34.14", + "scores": { + "validation": [ + { + "accuracy": 0.3902, + "f1": 0.375704, + "f1_weighted": 0.375704, + "scores_per_experiment": [ + { + "accuracy": 0.400667, + "f1": 0.389679, + "f1_weighted": 0.389679 + }, + { + "accuracy": 0.378667, + "f1": 0.364689, + "f1_weighted": 0.364689 + }, + { + "accuracy": 0.392, + "f1": 0.372259, + "f1_weighted": 0.372259 + }, + { + "accuracy": 0.366667, + "f1": 0.341907, + "f1_weighted": 0.341907 + }, + { + "accuracy": 0.392, + "f1": 0.378671, + "f1_weighted": 0.378671 + }, + { + "accuracy": 0.372333, + "f1": 0.372372, + "f1_weighted": 0.372372 + }, + { + "accuracy": 0.405667, + "f1": 0.397212, + "f1_weighted": 0.397212 + }, + { + "accuracy": 0.409333, + "f1": 0.3695, + "f1_weighted": 0.3695 + }, + { + "accuracy": 0.385, + "f1": 0.372866, + "f1_weighted": 0.372866 + }, + { + "accuracy": 0.399667, + "f1": 0.397885, + "f1_weighted": 0.397885 + } + ], + "main_score": 0.3902, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ], + "test": [ + { + "accuracy": 0.384233, + "f1": 0.370079, + "f1_weighted": 0.370079, + "scores_per_experiment": [ + { + "accuracy": 0.381333, + "f1": 0.371615, + "f1_weighted": 0.371615 + }, + { + "accuracy": 0.380333, + "f1": 0.365701, + "f1_weighted": 0.365701 + }, + { + "accuracy": 0.388333, + "f1": 0.367418, + "f1_weighted": 0.367418 + }, + { + "accuracy": 0.368, + "f1": 0.345152, + "f1_weighted": 0.345152 + }, + { + "accuracy": 0.371333, + "f1": 0.361374, + "f1_weighted": 0.361374 + }, + { + "accuracy": 0.386667, + "f1": 0.38682, + "f1_weighted": 0.38682 + }, + { + "accuracy": 0.375667, + "f1": 0.367618, + "f1_weighted": 0.367618 + }, + { + "accuracy": 0.397667, + "f1": 0.362688, + "f1_weighted": 0.362688 + }, + { + "accuracy": 0.379333, + "f1": 0.36083, + "f1_weighted": 0.36083 + }, + { + "accuracy": 0.413667, + "f1": 0.411579, + "f1_weighted": 0.411579 + } + ], + "main_score": 0.384233, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 11.349958181381226, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/MultilingualSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/MultilingualSentimentClassification.json new file mode 100644 index 0000000000..1b34dae65e --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/MultilingualSentimentClassification.json @@ -0,0 +1,2447 @@ +{ + "dataset_revision": "2b9b4d10fc589af67794141fe8cbd3739de1eb33", + "task_name": "MultilingualSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.484986, + "f1": 0.456307, + "f1_weighted": 0.456162, + "ap": 0.492177, + "ap_weighted": 0.492177, + "scores_per_experiment": [ + { + "accuracy": 0.463173, + "f1": 0.447475, + "f1_weighted": 0.446947, + "ap": 0.480202, + "ap_weighted": 0.480202 + }, + { + "accuracy": 0.518414, + "f1": 0.514168, + "f1_weighted": 0.51391, + "ap": 0.50695, + "ap_weighted": 0.50695 + }, + { + "accuracy": 0.483003, + "f1": 0.460234, + "f1_weighted": 0.459606, + "ap": 0.489436, + "ap_weighted": 0.489436 + }, + { + "accuracy": 0.453258, + "f1": 0.453253, + "f1_weighted": 0.453245, + "ap": 0.475982, + "ap_weighted": 0.475982 + }, + { + "accuracy": 0.423513, + "f1": 0.412421, + "f1_weighted": 0.412878, + "ap": 0.466754, + "ap_weighted": 0.466754 + }, + { + "accuracy": 0.505666, + "f1": 0.397621, + "f1_weighted": 0.396176, + "ap": 0.501243, + "ap_weighted": 0.501243 + }, + { + "accuracy": 0.446176, + "f1": 0.405344, + "f1_weighted": 0.406227, + "ap": 0.475905, + "ap_weighted": 0.475905 + }, + { + "accuracy": 0.536827, + "f1": 0.503466, + "f1_weighted": 0.504195, + "ap": 0.517437, + "ap_weighted": 0.517437 + }, + { + "accuracy": 0.546742, + "f1": 0.541979, + "f1_weighted": 0.542243, + "ap": 0.522933, + "ap_weighted": 0.522933 + }, + { + "accuracy": 0.473088, + "f1": 0.427113, + "f1_weighted": 0.426194, + "ap": 0.484931, + "ap_weighted": 0.484931 + } + ], + "main_score": 0.484986, + "hf_subset": "ara", + "languages": [ + "ara-Arab" + ] + }, + { + "accuracy": 0.515217, + "f1": 0.498877, + "f1_weighted": 0.520729, + "ap": 0.648092, + "ap_weighted": 0.648092, + "scores_per_experiment": [ + { + "accuracy": 0.48913, + "f1": 0.484194, + "f1_weighted": 0.498454, + "ap": 0.642015, + "ap_weighted": 0.642015 + }, + { + "accuracy": 0.630435, + "f1": 0.615725, + "f1_weighted": 0.636973, + "ap": 0.708155, + "ap_weighted": 0.708155 + }, + { + "accuracy": 0.597826, + "f1": 0.565761, + "f1_weighted": 0.599109, + "ap": 0.674085, + "ap_weighted": 0.674085 + }, + { + "accuracy": 0.467391, + "f1": 0.46429, + "f1_weighted": 0.475809, + "ap": 0.634396, + "ap_weighted": 0.634396 + }, + { + "accuracy": 0.543478, + "f1": 0.52924, + "f1_weighted": 0.552377, + "ap": 0.659295, + "ap_weighted": 0.659295 + }, + { + "accuracy": 0.456522, + "f1": 0.452381, + "f1_weighted": 0.465839, + "ap": 0.627936, + "ap_weighted": 0.627936 + }, + { + "accuracy": 0.521739, + "f1": 0.455914, + "f1_weighted": 0.509397, + "ap": 0.623732, + "ap_weighted": 0.623732 + }, + { + "accuracy": 0.543478, + "f1": 0.525307, + "f1_weighted": 0.551554, + "ap": 0.65592, + "ap_weighted": 0.65592 + }, + { + "accuracy": 0.467391, + "f1": 0.462245, + "f1_weighted": 0.477112, + "ap": 0.631496, + "ap_weighted": 0.631496 + }, + { + "accuracy": 0.434783, + "f1": 0.433712, + "f1_weighted": 0.44067, + "ap": 0.623887, + "ap_weighted": 0.623887 + } + ], + "main_score": 0.515217, + "hf_subset": "dza", + "languages": [ + "dza-Arab" + ] + }, + { + "accuracy": 0.607094, + "f1": 0.561379, + "f1_weighted": 0.639665, + "ap": 0.827262, + "ap_weighted": 0.827262, + "scores_per_experiment": [ + { + "accuracy": 0.615561, + "f1": 0.556902, + "f1_weighted": 0.647288, + "ap": 0.817281, + "ap_weighted": 0.817281 + }, + { + "accuracy": 0.610984, + "f1": 0.575097, + "f1_weighted": 0.644328, + "ap": 0.839875, + "ap_weighted": 0.839875 + }, + { + "accuracy": 0.627002, + "f1": 0.589553, + "f1_weighted": 0.659061, + "ap": 0.845822, + "ap_weighted": 0.845822 + }, + { + "accuracy": 0.567506, + "f1": 0.532234, + "f1_weighted": 0.604247, + "ap": 0.818918, + "ap_weighted": 0.818918 + }, + { + "accuracy": 0.569794, + "f1": 0.531458, + "f1_weighted": 0.606597, + "ap": 0.816423, + "ap_weighted": 0.816423 + }, + { + "accuracy": 0.668192, + "f1": 0.594783, + "f1_weighted": 0.691478, + "ap": 0.827418, + "ap_weighted": 0.827418 + }, + { + "accuracy": 0.610984, + "f1": 0.548114, + "f1_weighted": 0.642612, + "ap": 0.811716, + "ap_weighted": 0.811716 + }, + { + "accuracy": 0.643021, + "f1": 0.590115, + "f1_weighted": 0.672674, + "ap": 0.835098, + "ap_weighted": 0.835098 + }, + { + "accuracy": 0.546911, + "f1": 0.518015, + "f1_weighted": 0.584179, + "ap": 0.816796, + "ap_weighted": 0.816796 + }, + { + "accuracy": 0.610984, + "f1": 0.577514, + "f1_weighted": 0.644182, + "ap": 0.843275, + "ap_weighted": 0.843275 + } + ], + "main_score": 0.607094, + "hf_subset": "hrv", + "languages": [ + "hrv-Latn" + ] + }, + { + "accuracy": 0.495261, + "f1": 0.481555, + "f1_weighted": 0.485832, + "ap": 0.594853, + "ap_weighted": 0.594853, + "scores_per_experiment": [ + { + "accuracy": 0.516588, + "f1": 0.510775, + "f1_weighted": 0.520631, + "ap": 0.599327, + "ap_weighted": 0.599327 + }, + { + "accuracy": 0.421801, + "f1": 0.418023, + "f1_weighted": 0.409356, + "ap": 0.572062, + "ap_weighted": 0.572062 + }, + { + "accuracy": 0.488152, + "f1": 0.480864, + "f1_weighted": 0.492233, + "ap": 0.584307, + "ap_weighted": 0.584307 + }, + { + "accuracy": 0.49763, + "f1": 0.491589, + "f1_weighted": 0.501833, + "ap": 0.589723, + "ap_weighted": 0.589723 + }, + { + "accuracy": 0.49763, + "f1": 0.481693, + "f1_weighted": 0.464893, + "ap": 0.620014, + "ap_weighted": 0.620014 + }, + { + "accuracy": 0.459716, + "f1": 0.442575, + "f1_weighted": 0.460642, + "ap": 0.566849, + "ap_weighted": 0.566849 + }, + { + "accuracy": 0.530806, + "f1": 0.522395, + "f1_weighted": 0.53411, + "ap": 0.604523, + "ap_weighted": 0.604523 + }, + { + "accuracy": 0.450237, + "f1": 0.416126, + "f1_weighted": 0.390041, + "ap": 0.59878, + "ap_weighted": 0.59878 + }, + { + "accuracy": 0.563981, + "f1": 0.532196, + "f1_weighted": 0.554734, + "ap": 0.609542, + "ap_weighted": 0.609542 + }, + { + "accuracy": 0.526066, + "f1": 0.519318, + "f1_weighted": 0.529845, + "ap": 0.603403, + "ap_weighted": 0.603403 + } + ], + "main_score": 0.495261, + "hf_subset": "tur", + "languages": [ + "tur-Latn" + ] + }, + { + "accuracy": 0.536934, + "f1": 0.530299, + "f1_weighted": 0.530232, + "ap": 0.535341, + "ap_weighted": 0.535341, + "scores_per_experiment": [ + { + "accuracy": 0.537226, + "f1": 0.53289, + "f1_weighted": 0.531642, + "ap": 0.536036, + "ap_weighted": 0.536036 + }, + { + "accuracy": 0.49635, + "f1": 0.493201, + "f1_weighted": 0.494309, + "ap": 0.511168, + "ap_weighted": 0.511168 + }, + { + "accuracy": 0.588321, + "f1": 0.580925, + "f1_weighted": 0.579381, + "ap": 0.571819, + "ap_weighted": 0.571819 + }, + { + "accuracy": 0.547445, + "f1": 0.520983, + "f1_weighted": 0.517861, + "ap": 0.546766, + "ap_weighted": 0.546766 + }, + { + "accuracy": 0.545985, + "f1": 0.543553, + "f1_weighted": 0.542629, + "ap": 0.540824, + "ap_weighted": 0.540824 + }, + { + "accuracy": 0.508029, + "f1": 0.504719, + "f1_weighted": 0.505842, + "ap": 0.516974, + "ap_weighted": 0.516974 + }, + { + "accuracy": 0.518248, + "f1": 0.516987, + "f1_weighted": 0.517672, + "ap": 0.522745, + "ap_weighted": 0.522745 + }, + { + "accuracy": 0.563504, + "f1": 0.548716, + "f1_weighted": 0.550981, + "ap": 0.545829, + "ap_weighted": 0.545829 + }, + { + "accuracy": 0.525547, + "f1": 0.522355, + "f1_weighted": 0.523438, + "ap": 0.526181, + "ap_weighted": 0.526181 + }, + { + "accuracy": 0.538686, + "f1": 0.538662, + "f1_weighted": 0.538568, + "ap": 0.535072, + "ap_weighted": 0.535072 + } + ], + "main_score": 0.536934, + "hf_subset": "vie", + "languages": [ + "vie-Latn" + ] + }, + { + "accuracy": 0.569396, + "f1": 0.502361, + "f1_weighted": 0.605658, + "ap": 0.815714, + "ap_weighted": 0.815714, + "scores_per_experiment": [ + { + "accuracy": 0.591751, + "f1": 0.505712, + "f1_weighted": 0.628361, + "ap": 0.80782, + "ap_weighted": 0.80782 + }, + { + "accuracy": 0.473401, + "f1": 0.434651, + "f1_weighted": 0.522679, + "ap": 0.794666, + "ap_weighted": 0.794666 + }, + { + "accuracy": 0.682606, + "f1": 0.571465, + "f1_weighted": 0.70126, + "ap": 0.827072, + "ap_weighted": 0.827072 + }, + { + "accuracy": 0.453676, + "f1": 0.441703, + "f1_weighted": 0.490328, + "ap": 0.820784, + "ap_weighted": 0.820784 + }, + { + "accuracy": 0.546922, + "f1": 0.453731, + "f1_weighted": 0.58792, + "ap": 0.78668, + "ap_weighted": 0.78668 + }, + { + "accuracy": 0.531381, + "f1": 0.497402, + "f1_weighted": 0.575124, + "ap": 0.826116, + "ap_weighted": 0.826116 + }, + { + "accuracy": 0.655708, + "f1": 0.556586, + "f1_weighted": 0.681272, + "ap": 0.824281, + "ap_weighted": 0.824281 + }, + { + "accuracy": 0.529588, + "f1": 0.499217, + "f1_weighted": 0.572563, + "ap": 0.829874, + "ap_weighted": 0.829874 + }, + { + "accuracy": 0.603706, + "f1": 0.506011, + "f1_weighted": 0.636665, + "ap": 0.805534, + "ap_weighted": 0.805534 + }, + { + "accuracy": 0.625224, + "f1": 0.557133, + "f1_weighted": 0.660411, + "ap": 0.83431, + "ap_weighted": 0.83431 + } + ], + "main_score": 0.569396, + "hf_subset": "bul", + "languages": [ + "bul-Cyrl" + ] + }, + { + "accuracy": 0.581667, + "f1": 0.570013, + "f1_weighted": 0.576485, + "ap": 0.483165, + "ap_weighted": 0.483165, + "scores_per_experiment": [ + { + "accuracy": 0.591667, + "f1": 0.591553, + "f1_weighted": 0.590674, + "ap": 0.496807, + "ap_weighted": 0.496807 + }, + { + "accuracy": 0.5, + "f1": 0.499965, + "f1_weighted": 0.500504, + "ap": 0.439104, + "ap_weighted": 0.439104 + }, + { + "accuracy": 0.510417, + "f1": 0.509085, + "f1_weighted": 0.512388, + "ap": 0.441594, + "ap_weighted": 0.441594 + }, + { + "accuracy": 0.566667, + "f1": 0.560687, + "f1_weighted": 0.567307, + "ap": 0.469992, + "ap_weighted": 0.469992 + }, + { + "accuracy": 0.60625, + "f1": 0.581378, + "f1_weighted": 0.594558, + "ap": 0.489331, + "ap_weighted": 0.489331 + }, + { + "accuracy": 0.595833, + "f1": 0.566932, + "f1_weighted": 0.581383, + "ap": 0.48022, + "ap_weighted": 0.48022 + }, + { + "accuracy": 0.59375, + "f1": 0.593734, + "f1_weighted": 0.593406, + "ap": 0.497516, + "ap_weighted": 0.497516 + }, + { + "accuracy": 0.61875, + "f1": 0.6058, + "f1_weighted": 0.615029, + "ap": 0.503358, + "ap_weighted": 0.503358 + }, + { + "accuracy": 0.58125, + "f1": 0.580593, + "f1_weighted": 0.582737, + "ap": 0.485461, + "ap_weighted": 0.485461 + }, + { + "accuracy": 0.652083, + "f1": 0.610403, + "f1_weighted": 0.626863, + "ap": 0.528268, + "ap_weighted": 0.528268 + } + ], + "main_score": 0.581667, + "hf_subset": "pol", + "languages": [ + "pol-Latn" + ] + }, + { + "accuracy": 0.697176, + "f1": 0.679779, + "f1_weighted": 0.699426, + "ap": 0.759713, + "ap_weighted": 0.759713, + "scores_per_experiment": [ + { + "accuracy": 0.755516, + "f1": 0.738878, + "f1_weighted": 0.758309, + "ap": 0.791857, + "ap_weighted": 0.791857 + }, + { + "accuracy": 0.654898, + "f1": 0.640598, + "f1_weighted": 0.661732, + "ap": 0.730791, + "ap_weighted": 0.730791 + }, + { + "accuracy": 0.733892, + "f1": 0.722815, + "f1_weighted": 0.73915, + "ap": 0.789943, + "ap_weighted": 0.789943 + }, + { + "accuracy": 0.714475, + "f1": 0.708478, + "f1_weighted": 0.720804, + "ap": 0.791422, + "ap_weighted": 0.791422 + }, + { + "accuracy": 0.681818, + "f1": 0.673343, + "f1_weighted": 0.688854, + "ap": 0.76005, + "ap_weighted": 0.76005 + }, + { + "accuracy": 0.68579, + "f1": 0.674166, + "f1_weighted": 0.692308, + "ap": 0.75566, + "ap_weighted": 0.75566 + }, + { + "accuracy": 0.59797, + "f1": 0.597797, + "f1_weighted": 0.600256, + "ap": 0.733986, + "ap_weighted": 0.733986 + }, + { + "accuracy": 0.763019, + "f1": 0.745596, + "f1_weighted": 0.765222, + "ap": 0.795117, + "ap_weighted": 0.795117 + }, + { + "accuracy": 0.666372, + "f1": 0.641218, + "f1_weighted": 0.669223, + "ap": 0.72414, + "ap_weighted": 0.72414 + }, + { + "accuracy": 0.718005, + "f1": 0.654899, + "f1_weighted": 0.698403, + "ap": 0.724167, + "ap_weighted": 0.724167 + } + ], + "main_score": 0.697176, + "hf_subset": "ind", + "languages": [ + "ind-Latn" + ] + }, + { + "accuracy": 0.59651, + "f1": 0.49274, + "f1_weighted": 0.652091, + "ap": 0.164126, + "ap_weighted": 0.164126, + "scores_per_experiment": [ + { + "accuracy": 0.626174, + "f1": 0.497035, + "f1_weighted": 0.679028, + "ap": 0.154462, + "ap_weighted": 0.154462 + }, + { + "accuracy": 0.500671, + "f1": 0.444449, + "f1_weighted": 0.570652, + "ap": 0.159817, + "ap_weighted": 0.159817 + }, + { + "accuracy": 0.499329, + "f1": 0.4317, + "f1_weighted": 0.571694, + "ap": 0.147147, + "ap_weighted": 0.147147 + }, + { + "accuracy": 0.551678, + "f1": 0.480358, + "f1_weighted": 0.617829, + "ap": 0.170367, + "ap_weighted": 0.170367 + }, + { + "accuracy": 0.716107, + "f1": 0.554734, + "f1_weighted": 0.746151, + "ap": 0.17517, + "ap_weighted": 0.17517 + }, + { + "accuracy": 0.531544, + "f1": 0.463686, + "f1_weighted": 0.599913, + "ap": 0.162889, + "ap_weighted": 0.162889 + }, + { + "accuracy": 0.541611, + "f1": 0.474347, + "f1_weighted": 0.608622, + "ap": 0.169444, + "ap_weighted": 0.169444 + }, + { + "accuracy": 0.72349, + "f1": 0.558994, + "f1_weighted": 0.751328, + "ap": 0.176841, + "ap_weighted": 0.176841 + }, + { + "accuracy": 0.663758, + "f1": 0.519203, + "f1_weighted": 0.707461, + "ap": 0.16072, + "ap_weighted": 0.16072 + }, + { + "accuracy": 0.610738, + "f1": 0.502895, + "f1_weighted": 0.668234, + "ap": 0.164399, + "ap_weighted": 0.164399 + } + ], + "main_score": 0.59651, + "hf_subset": "deu", + "languages": [ + "deu-Latn" + ] + }, + { + "accuracy": 0.657701, + "f1": 0.62992, + "f1_weighted": 0.665621, + "ap": 0.759642, + "ap_weighted": 0.759642, + "scores_per_experiment": [ + { + "accuracy": 0.63167, + "f1": 0.619269, + "f1_weighted": 0.644638, + "ap": 0.764259, + "ap_weighted": 0.764259 + }, + { + "accuracy": 0.638612, + "f1": 0.621065, + "f1_weighted": 0.65117, + "ap": 0.759415, + "ap_weighted": 0.759415 + }, + { + "accuracy": 0.675054, + "f1": 0.645066, + "f1_weighted": 0.683156, + "ap": 0.763671, + "ap_weighted": 0.763671 + }, + { + "accuracy": 0.582213, + "f1": 0.551847, + "f1_weighted": 0.594916, + "ap": 0.713598, + "ap_weighted": 0.713598 + }, + { + "accuracy": 0.701518, + "f1": 0.667116, + "f1_weighted": 0.706625, + "ap": 0.773162, + "ap_weighted": 0.773162 + }, + { + "accuracy": 0.685466, + "f1": 0.649323, + "f1_weighted": 0.690888, + "ap": 0.762616, + "ap_weighted": 0.762616 + }, + { + "accuracy": 0.681562, + "f1": 0.655052, + "f1_weighted": 0.690357, + "ap": 0.771663, + "ap_weighted": 0.771663 + }, + { + "accuracy": 0.613449, + "f1": 0.605852, + "f1_weighted": 0.626055, + "ap": 0.763432, + "ap_weighted": 0.763432 + }, + { + "accuracy": 0.646855, + "f1": 0.632536, + "f1_weighted": 0.659316, + "ap": 0.770049, + "ap_weighted": 0.770049 + }, + { + "accuracy": 0.720607, + "f1": 0.652075, + "f1_weighted": 0.709085, + "ap": 0.754555, + "ap_weighted": 0.754555 + } + ], + "main_score": 0.657701, + "hf_subset": "heb", + "languages": [ + "heb-Latn" + ] + }, + { + "accuracy": 0.636504, + "f1": 0.495187, + "f1_weighted": 0.680796, + "ap": 0.898383, + "ap_weighted": 0.898383, + "scores_per_experiment": [ + { + "accuracy": 0.781213, + "f1": 0.567754, + "f1_weighted": 0.799272, + "ap": 0.899574, + "ap_weighted": 0.899574 + }, + { + "accuracy": 0.790725, + "f1": 0.563158, + "f1_weighted": 0.803472, + "ap": 0.896957, + "ap_weighted": 0.896957 + }, + { + "accuracy": 0.624257, + "f1": 0.49539, + "f1_weighted": 0.689752, + "ap": 0.896266, + "ap_weighted": 0.896266 + }, + { + "accuracy": 0.769322, + "f1": 0.56946, + "f1_weighted": 0.793041, + "ap": 0.901917, + "ap_weighted": 0.901917 + }, + { + "accuracy": 0.774078, + "f1": 0.559458, + "f1_weighted": 0.793822, + "ap": 0.897738, + "ap_weighted": 0.897738 + }, + { + "accuracy": 0.542212, + "f1": 0.451507, + "f1_weighted": 0.621512, + "ap": 0.892736, + "ap_weighted": 0.892736 + }, + { + "accuracy": 0.738407, + "f1": 0.544151, + "f1_weighted": 0.77096, + "ap": 0.897115, + "ap_weighted": 0.897115 + }, + { + "accuracy": 0.510107, + "f1": 0.454275, + "f1_weighted": 0.587318, + "ap": 0.911569, + "ap_weighted": 0.911569 + }, + { + "accuracy": 0.275862, + "f1": 0.271644, + "f1_weighted": 0.31389, + "ap": 0.885357, + "ap_weighted": 0.885357 + }, + { + "accuracy": 0.558859, + "f1": 0.475069, + "f1_weighted": 0.634917, + "ap": 0.904598, + "ap_weighted": 0.904598 + } + ], + "main_score": 0.636504, + "hf_subset": "uig", + "languages": [ + "uig-Hans" + ] + }, + { + "accuracy": 0.53846, + "f1": 0.517175, + "f1_weighted": 0.526873, + "ap": 0.631965, + "ap_weighted": 0.631965, + "scores_per_experiment": [ + { + "accuracy": 0.538807, + "f1": 0.533748, + "f1_weighted": 0.544204, + "ap": 0.628713, + "ap_weighted": 0.628713 + }, + { + "accuracy": 0.558007, + "f1": 0.557964, + "f1_weighted": 0.557031, + "ap": 0.656744, + "ap_weighted": 0.656744 + }, + { + "accuracy": 0.525123, + "f1": 0.503621, + "f1_weighted": 0.525861, + "ap": 0.609392, + "ap_weighted": 0.609392 + }, + { + "accuracy": 0.572917, + "f1": 0.571231, + "f1_weighted": 0.577019, + "ap": 0.655101, + "ap_weighted": 0.655101 + }, + { + "accuracy": 0.56107, + "f1": 0.55277, + "f1_weighted": 0.565886, + "ap": 0.637443, + "ap_weighted": 0.637443 + }, + { + "accuracy": 0.52982, + "f1": 0.472175, + "f1_weighted": 0.509726, + "ap": 0.598751, + "ap_weighted": 0.598751 + }, + { + "accuracy": 0.492647, + "f1": 0.488129, + "f1_weighted": 0.477775, + "ap": 0.627453, + "ap_weighted": 0.627453 + }, + { + "accuracy": 0.48366, + "f1": 0.445906, + "f1_weighted": 0.41477, + "ap": 0.651596, + "ap_weighted": 0.651596 + }, + { + "accuracy": 0.569649, + "f1": 0.566193, + "f1_weighted": 0.574528, + "ap": 0.64907, + "ap_weighted": 0.64907 + }, + { + "accuracy": 0.5529, + "f1": 0.480016, + "f1_weighted": 0.521926, + "ap": 0.605388, + "ap_weighted": 0.605388 + } + ], + "main_score": 0.53846, + "hf_subset": "zho", + "languages": [ + "zho-Hans" + ] + }, + { + "accuracy": 0.497619, + "f1": 0.458757, + "f1_weighted": 0.460102, + "ap": 0.484567, + "ap_weighted": 0.484567, + "scores_per_experiment": [ + { + "accuracy": 0.493197, + "f1": 0.459899, + "f1_weighted": 0.456249, + "ap": 0.486557, + "ap_weighted": 0.486557 + }, + { + "accuracy": 0.513605, + "f1": 0.432746, + "f1_weighted": 0.438574, + "ap": 0.488297, + "ap_weighted": 0.488297 + }, + { + "accuracy": 0.44898, + "f1": 0.448878, + "f1_weighted": 0.449082, + "ap": 0.46356, + "ap_weighted": 0.46356 + }, + { + "accuracy": 0.510204, + "f1": 0.446125, + "f1_weighted": 0.451251, + "ap": 0.487059, + "ap_weighted": 0.487059 + }, + { + "accuracy": 0.496599, + "f1": 0.454135, + "f1_weighted": 0.458278, + "ap": 0.481331, + "ap_weighted": 0.481331 + }, + { + "accuracy": 0.503401, + "f1": 0.411839, + "f1_weighted": 0.418154, + "ap": 0.483122, + "ap_weighted": 0.483122 + }, + { + "accuracy": 0.510204, + "f1": 0.502748, + "f1_weighted": 0.504405, + "ap": 0.490085, + "ap_weighted": 0.490085 + }, + { + "accuracy": 0.479592, + "f1": 0.41892, + "f1_weighted": 0.413811, + "ap": 0.48085, + "ap_weighted": 0.48085 + }, + { + "accuracy": 0.5, + "f1": 0.492812, + "f1_weighted": 0.491169, + "ap": 0.48821, + "ap_weighted": 0.48821 + }, + { + "accuracy": 0.520408, + "f1": 0.519469, + "f1_weighted": 0.520047, + "ap": 0.496602, + "ap_weighted": 0.496602 + } + ], + "main_score": 0.497619, + "hf_subset": "urd", + "languages": [ + "urd-Arab" + ] + }, + { + "accuracy": 0.523501, + "f1": 0.492168, + "f1_weighted": 0.540346, + "ap": 0.727119, + "ap_weighted": 0.727119, + "scores_per_experiment": [ + { + "accuracy": 0.410072, + "f1": 0.405887, + "f1_weighted": 0.427531, + "ap": 0.70193, + "ap_weighted": 0.70193 + }, + { + "accuracy": 0.556355, + "f1": 0.523175, + "f1_weighted": 0.577771, + "ap": 0.735884, + "ap_weighted": 0.735884 + }, + { + "accuracy": 0.498801, + "f1": 0.493158, + "f1_weighted": 0.516371, + "ap": 0.742627, + "ap_weighted": 0.742627 + }, + { + "accuracy": 0.52518, + "f1": 0.491714, + "f1_weighted": 0.548325, + "ap": 0.72107, + "ap_weighted": 0.72107 + }, + { + "accuracy": 0.498801, + "f1": 0.4834, + "f1_weighted": 0.522116, + "ap": 0.726305, + "ap_weighted": 0.726305 + }, + { + "accuracy": 0.577938, + "f1": 0.499591, + "f1_weighted": 0.585535, + "ap": 0.717222, + "ap_weighted": 0.717222 + }, + { + "accuracy": 0.645084, + "f1": 0.581075, + "f1_weighted": 0.652152, + "ap": 0.754344, + "ap_weighted": 0.754344 + }, + { + "accuracy": 0.422062, + "f1": 0.417723, + "f1_weighted": 0.439542, + "ap": 0.706847, + "ap_weighted": 0.706847 + }, + { + "accuracy": 0.616307, + "f1": 0.554796, + "f1_weighted": 0.626625, + "ap": 0.742899, + "ap_weighted": 0.742899 + }, + { + "accuracy": 0.484412, + "f1": 0.471165, + "f1_weighted": 0.507495, + "ap": 0.722061, + "ap_weighted": 0.722061 + } + ], + "main_score": 0.523501, + "hf_subset": "nor", + "languages": [ + "nor-Latn" + ] + }, + { + "accuracy": 0.648273, + "f1": 0.567524, + "f1_weighted": 0.703439, + "ap": 0.934869, + "ap_weighted": 0.934869, + "scores_per_experiment": [ + { + "accuracy": 0.728407, + "f1": 0.630188, + "f1_weighted": 0.77395, + "ap": 0.946126, + "ap_weighted": 0.946126 + }, + { + "accuracy": 0.6881, + "f1": 0.589337, + "f1_weighted": 0.74125, + "ap": 0.933398, + "ap_weighted": 0.933398 + }, + { + "accuracy": 0.478887, + "f1": 0.44569, + "f1_weighted": 0.548014, + "ap": 0.921366, + "ap_weighted": 0.921366 + }, + { + "accuracy": 0.706334, + "f1": 0.607943, + "f1_weighted": 0.756095, + "ap": 0.939463, + "ap_weighted": 0.939463 + }, + { + "accuracy": 0.677543, + "f1": 0.586947, + "f1_weighted": 0.732866, + "ap": 0.937249, + "ap_weighted": 0.937249 + }, + { + "accuracy": 0.508637, + "f1": 0.467382, + "f1_weighted": 0.579197, + "ap": 0.923678, + "ap_weighted": 0.923678 + }, + { + "accuracy": 0.684261, + "f1": 0.575739, + "f1_weighted": 0.737596, + "ap": 0.92424, + "ap_weighted": 0.92424 + }, + { + "accuracy": 0.81094, + "f1": 0.700363, + "f1_weighted": 0.837668, + "ap": 0.953106, + "ap_weighted": 0.953106 + }, + { + "accuracy": 0.663148, + "f1": 0.583788, + "f1_weighted": 0.72088, + "ap": 0.943417, + "ap_weighted": 0.943417 + }, + { + "accuracy": 0.536468, + "f1": 0.487866, + "f1_weighted": 0.606874, + "ap": 0.926645, + "ap_weighted": 0.926645 + } + ], + "main_score": 0.648273, + "hf_subset": "slk", + "languages": [ + "slk-Latn" + ] + }, + { + "accuracy": 0.499271, + "f1": 0.426028, + "f1_weighted": 0.564951, + "ap": 0.861031, + "ap_weighted": 0.861031, + "scores_per_experiment": [ + { + "accuracy": 0.510039, + "f1": 0.444944, + "f1_weighted": 0.580742, + "ap": 0.86775, + "ap_weighted": 0.86775 + }, + { + "accuracy": 0.483343, + "f1": 0.420345, + "f1_weighted": 0.556865, + "ap": 0.858721, + "ap_weighted": 0.858721 + }, + { + "accuracy": 0.421873, + "f1": 0.389697, + "f1_weighted": 0.48981, + "ap": 0.863854, + "ap_weighted": 0.863854 + }, + { + "accuracy": 0.517667, + "f1": 0.448609, + "f1_weighted": 0.588017, + "ap": 0.867458, + "ap_weighted": 0.867458 + }, + { + "accuracy": 0.508469, + "f1": 0.435197, + "f1_weighted": 0.580531, + "ap": 0.860403, + "ap_weighted": 0.860403 + }, + { + "accuracy": 0.402692, + "f1": 0.372549, + "f1_weighted": 0.470799, + "ap": 0.857858, + "ap_weighted": 0.857858 + }, + { + "accuracy": 0.670107, + "f1": 0.481149, + "f1_weighted": 0.704842, + "ap": 0.854304, + "ap_weighted": 0.854304 + }, + { + "accuracy": 0.480426, + "f1": 0.421973, + "f1_weighted": 0.553292, + "ap": 0.861469, + "ap_weighted": 0.861469 + }, + { + "accuracy": 0.400224, + "f1": 0.370604, + "f1_weighted": 0.46815, + "ap": 0.857424, + "ap_weighted": 0.857424 + }, + { + "accuracy": 0.597869, + "f1": 0.47521, + "f1_weighted": 0.656466, + "ap": 0.861072, + "ap_weighted": 0.861072 + } + ], + "main_score": 0.499271, + "hf_subset": "cmn", + "languages": [ + "cmn-Hans" + ] + }, + { + "accuracy": 0.564078, + "f1": 0.550635, + "f1_weighted": 0.561632, + "ap": 0.445195, + "ap_weighted": 0.445195, + "scores_per_experiment": [ + { + "accuracy": 0.538823, + "f1": 0.53771, + "f1_weighted": 0.53353, + "ap": 0.444294, + "ap_weighted": 0.444294 + }, + { + "accuracy": 0.569966, + "f1": 0.568982, + "f1_weighted": 0.572777, + "ap": 0.453797, + "ap_weighted": 0.453797 + }, + { + "accuracy": 0.493601, + "f1": 0.46664, + "f1_weighted": 0.488741, + "ap": 0.394012, + "ap_weighted": 0.394012 + }, + { + "accuracy": 0.602816, + "f1": 0.592946, + "f1_weighted": 0.604627, + "ap": 0.465056, + "ap_weighted": 0.465056 + }, + { + "accuracy": 0.598123, + "f1": 0.593865, + "f1_weighted": 0.601529, + "ap": 0.467559, + "ap_weighted": 0.467559 + }, + { + "accuracy": 0.550768, + "f1": 0.549646, + "f1_weighted": 0.553789, + "ap": 0.441001, + "ap_weighted": 0.441001 + }, + { + "accuracy": 0.633532, + "f1": 0.592353, + "f1_weighted": 0.616231, + "ap": 0.472707, + "ap_weighted": 0.472707 + }, + { + "accuracy": 0.426621, + "f1": 0.422748, + "f1_weighted": 0.431462, + "ap": 0.378244, + "ap_weighted": 0.378244 + }, + { + "accuracy": 0.631399, + "f1": 0.591809, + "f1_weighted": 0.615238, + "ap": 0.471419, + "ap_weighted": 0.471419 + }, + { + "accuracy": 0.595137, + "f1": 0.589652, + "f1_weighted": 0.598395, + "ap": 0.463864, + "ap_weighted": 0.463864 + } + ], + "main_score": 0.564078, + "hf_subset": "tha", + "languages": [ + "tha-Thai" + ] + }, + { + "accuracy": 0.676689, + "f1": 0.592954, + "f1_weighted": 0.708916, + "ap": 0.876196, + "ap_weighted": 0.876196, + "scores_per_experiment": [ + { + "accuracy": 0.668919, + "f1": 0.583577, + "f1_weighted": 0.705858, + "ap": 0.870688, + "ap_weighted": 0.870688 + }, + { + "accuracy": 0.496622, + "f1": 0.467697, + "f1_weighted": 0.548184, + "ap": 0.854528, + "ap_weighted": 0.854528 + }, + { + "accuracy": 0.77027, + "f1": 0.675439, + "f1_weighted": 0.789237, + "ap": 0.896494, + "ap_weighted": 0.896494 + }, + { + "accuracy": 0.550676, + "f1": 0.499078, + "f1_weighted": 0.60336, + "ap": 0.852465, + "ap_weighted": 0.852465 + }, + { + "accuracy": 0.75, + "f1": 0.653856, + "f1_weighted": 0.772187, + "ap": 0.889645, + "ap_weighted": 0.889645 + }, + { + "accuracy": 0.709459, + "f1": 0.640025, + "f1_weighted": 0.742574, + "ap": 0.900784, + "ap_weighted": 0.900784 + }, + { + "accuracy": 0.689189, + "f1": 0.635058, + "f1_weighted": 0.726227, + "ap": 0.911323, + "ap_weighted": 0.911323 + }, + { + "accuracy": 0.743243, + "f1": 0.612352, + "f1_weighted": 0.758463, + "ap": 0.865031, + "ap_weighted": 0.865031 + }, + { + "accuracy": 0.658784, + "f1": 0.565762, + "f1_weighted": 0.696128, + "ap": 0.86105, + "ap_weighted": 0.86105 + }, + { + "accuracy": 0.72973, + "f1": 0.596703, + "f1_weighted": 0.746945, + "ap": 0.859954, + "ap_weighted": 0.859954 + } + ], + "main_score": 0.676689, + "hf_subset": "spa", + "languages": [ + "spa-Latn" + ] + }, + { + "accuracy": 0.533679, + "f1": 0.51491, + "f1_weighted": 0.536012, + "ap": 0.700692, + "ap_weighted": 0.700692, + "scores_per_experiment": [ + { + "accuracy": 0.568627, + "f1": 0.562452, + "f1_weighted": 0.580259, + "ap": 0.720825, + "ap_weighted": 0.720825 + }, + { + "accuracy": 0.544406, + "f1": 0.543005, + "f1_weighted": 0.551672, + "ap": 0.720198, + "ap_weighted": 0.720198 + }, + { + "accuracy": 0.554787, + "f1": 0.496494, + "f1_weighted": 0.555182, + "ap": 0.669737, + "ap_weighted": 0.669737 + }, + { + "accuracy": 0.536332, + "f1": 0.532044, + "f1_weighted": 0.54739, + "ap": 0.705782, + "ap_weighted": 0.705782 + }, + { + "accuracy": 0.462514, + "f1": 0.458241, + "f1_weighted": 0.441759, + "ap": 0.701221, + "ap_weighted": 0.701221 + }, + { + "accuracy": 0.54902, + "f1": 0.54571, + "f1_weighted": 0.558993, + "ap": 0.716044, + "ap_weighted": 0.716044 + }, + { + "accuracy": 0.551326, + "f1": 0.548382, + "f1_weighted": 0.560873, + "ap": 0.718619, + "ap_weighted": 0.718619 + }, + { + "accuracy": 0.529412, + "f1": 0.425313, + "f1_weighted": 0.5091, + "ap": 0.64407, + "ap_weighted": 0.64407 + }, + { + "accuracy": 0.513264, + "f1": 0.512558, + "f1_weighted": 0.518914, + "ap": 0.704252, + "ap_weighted": 0.704252 + }, + { + "accuracy": 0.527105, + "f1": 0.524905, + "f1_weighted": 0.53598, + "ap": 0.706177, + "ap_weighted": 0.706177 + } + ], + "main_score": 0.533679, + "hf_subset": "rus", + "languages": [ + "rus-Cyrl" + ] + }, + { + "accuracy": 0.525984, + "f1": 0.518749, + "f1_weighted": 0.517996, + "ap": 0.5031, + "ap_weighted": 0.5031, + "scores_per_experiment": [ + { + "accuracy": 0.530559, + "f1": 0.527602, + "f1_weighted": 0.526719, + "ap": 0.505469, + "ap_weighted": 0.505469 + }, + { + "accuracy": 0.537683, + "f1": 0.526935, + "f1_weighted": 0.52525, + "ap": 0.510283, + "ap_weighted": 0.510283 + }, + { + "accuracy": 0.547807, + "f1": 0.537347, + "f1_weighted": 0.535704, + "ap": 0.51607, + "ap_weighted": 0.51607 + }, + { + "accuracy": 0.490064, + "f1": 0.489046, + "f1_weighted": 0.489585, + "ap": 0.482959, + "ap_weighted": 0.482959 + }, + { + "accuracy": 0.534308, + "f1": 0.518381, + "f1_weighted": 0.516312, + "ap": 0.508747, + "ap_weighted": 0.508747 + }, + { + "accuracy": 0.506187, + "f1": 0.505673, + "f1_weighted": 0.506049, + "ap": 0.491076, + "ap_weighted": 0.491076 + }, + { + "accuracy": 0.517435, + "f1": 0.507001, + "f1_weighted": 0.505307, + "ap": 0.499113, + "ap_weighted": 0.499113 + }, + { + "accuracy": 0.520435, + "f1": 0.510497, + "f1_weighted": 0.512145, + "ap": 0.49728, + "ap_weighted": 0.49728 + }, + { + "accuracy": 0.518935, + "f1": 0.509701, + "f1_weighted": 0.508112, + "ap": 0.499815, + "ap_weighted": 0.499815 + }, + { + "accuracy": 0.55643, + "f1": 0.555308, + "f1_weighted": 0.55478, + "ap": 0.520191, + "ap_weighted": 0.520191 + } + ], + "main_score": 0.525984, + "hf_subset": "kor", + "languages": [ + "kor-Hang" + ] + }, + { + "accuracy": 0.53558, + "f1": 0.498807, + "f1_weighted": 0.52487, + "ap": 0.643114, + "ap_weighted": 0.643114, + "scores_per_experiment": [ + { + "accuracy": 0.600705, + "f1": 0.549685, + "f1_weighted": 0.590311, + "ap": 0.658425, + "ap_weighted": 0.658425 + }, + { + "accuracy": 0.431034, + "f1": 0.429644, + "f1_weighted": 0.422096, + "ap": 0.624595, + "ap_weighted": 0.624595 + }, + { + "accuracy": 0.61442, + "f1": 0.550245, + "f1_weighted": 0.59578, + "ap": 0.659838, + "ap_weighted": 0.659838 + }, + { + "accuracy": 0.396944, + "f1": 0.389754, + "f1_weighted": 0.372001, + "ap": 0.617207, + "ap_weighted": 0.617207 + }, + { + "accuracy": 0.573276, + "f1": 0.543964, + "f1_weighted": 0.574952, + "ap": 0.655717, + "ap_weighted": 0.655717 + }, + { + "accuracy": 0.576411, + "f1": 0.450124, + "f1_weighted": 0.520754, + "ap": 0.626075, + "ap_weighted": 0.626075 + }, + { + "accuracy": 0.419279, + "f1": 0.418118, + "f1_weighted": 0.425085, + "ap": 0.60872, + "ap_weighted": 0.60872 + }, + { + "accuracy": 0.568574, + "f1": 0.547798, + "f1_weighted": 0.573777, + "ap": 0.659038, + "ap_weighted": 0.659038 + }, + { + "accuracy": 0.584639, + "f1": 0.550216, + "f1_weighted": 0.583567, + "ap": 0.658539, + "ap_weighted": 0.658539 + }, + { + "accuracy": 0.590517, + "f1": 0.558524, + "f1_weighted": 0.590378, + "ap": 0.662984, + "ap_weighted": 0.662984 + } + ], + "main_score": 0.53558, + "hf_subset": "jpn", + "languages": [ + "jpn-Jpan" + ] + }, + { + "accuracy": 0.657888, + "f1": 0.648717, + "f1_weighted": 0.650616, + "ap": 0.601639, + "ap_weighted": 0.601639, + "scores_per_experiment": [ + { + "accuracy": 0.722295, + "f1": 0.715794, + "f1_weighted": 0.717643, + "ap": 0.653114, + "ap_weighted": 0.653114 + }, + { + "accuracy": 0.662321, + "f1": 0.660761, + "f1_weighted": 0.661751, + "ap": 0.588254, + "ap_weighted": 0.588254 + }, + { + "accuracy": 0.580183, + "f1": 0.572473, + "f1_weighted": 0.574943, + "ap": 0.52381, + "ap_weighted": 0.52381 + }, + { + "accuracy": 0.739244, + "f1": 0.720424, + "f1_weighted": 0.723545, + "ap": 0.697551, + "ap_weighted": 0.697551 + }, + { + "accuracy": 0.611473, + "f1": 0.610917, + "f1_weighted": 0.610284, + "ap": 0.547646, + "ap_weighted": 0.547646 + }, + { + "accuracy": 0.73794, + "f1": 0.721189, + "f1_weighted": 0.72413, + "ap": 0.691188, + "ap_weighted": 0.691188 + }, + { + "accuracy": 0.645372, + "f1": 0.642806, + "f1_weighted": 0.644109, + "ap": 0.574156, + "ap_weighted": 0.574156 + }, + { + "accuracy": 0.594524, + "f1": 0.571987, + "f1_weighted": 0.576213, + "ap": 0.534378, + "ap_weighted": 0.534378 + }, + { + "accuracy": 0.753585, + "f1": 0.738883, + "f1_weighted": 0.741549, + "ap": 0.710352, + "ap_weighted": 0.710352 + }, + { + "accuracy": 0.531943, + "f1": 0.531939, + "f1_weighted": 0.531992, + "ap": 0.495946, + "ap_weighted": 0.495946 + } + ], + "main_score": 0.657888, + "hf_subset": "ell", + "languages": [ + "ell-Latn" + ] + }, + { + "accuracy": 0.607024, + "f1": 0.589006, + "f1_weighted": 0.595891, + "ap": 0.509226, + "ap_weighted": 0.509226, + "scores_per_experiment": [ + { + "accuracy": 0.631413, + "f1": 0.609829, + "f1_weighted": 0.619896, + "ap": 0.522852, + "ap_weighted": 0.522852 + }, + { + "accuracy": 0.632202, + "f1": 0.60491, + "f1_weighted": 0.616302, + "ap": 0.52285, + "ap_weighted": 0.52285 + }, + { + "accuracy": 0.606946, + "f1": 0.598757, + "f1_weighted": 0.605045, + "ap": 0.506694, + "ap_weighted": 0.506694 + }, + { + "accuracy": 0.64562, + "f1": 0.631737, + "f1_weighted": 0.639581, + "ap": 0.536765, + "ap_weighted": 0.536765 + }, + { + "accuracy": 0.625099, + "f1": 0.591774, + "f1_weighted": 0.60457, + "ap": 0.515803, + "ap_weighted": 0.515803 + }, + { + "accuracy": 0.587214, + "f1": 0.584079, + "f1_weighted": 0.58804, + "ap": 0.495407, + "ap_weighted": 0.495407 + }, + { + "accuracy": 0.571429, + "f1": 0.571402, + "f1_weighted": 0.571773, + "ap": 0.489458, + "ap_weighted": 0.489458 + }, + { + "accuracy": 0.528019, + "f1": 0.525985, + "f1_weighted": 0.522578, + "ap": 0.467296, + "ap_weighted": 0.467296 + }, + { + "accuracy": 0.627466, + "f1": 0.627461, + "f1_weighted": 0.627299, + "ap": 0.530129, + "ap_weighted": 0.530129 + }, + { + "accuracy": 0.614838, + "f1": 0.544126, + "f1_weighted": 0.563824, + "ap": 0.505001, + "ap_weighted": 0.505001 + } + ], + "main_score": 0.607024, + "hf_subset": "fin", + "languages": [ + "fin-Latn" + ] + }, + { + "accuracy": 0.5112, + "f1": 0.484853, + "f1_weighted": 0.484853, + "ap": 0.509701, + "ap_weighted": 0.509701, + "scores_per_experiment": [ + { + "accuracy": 0.538143, + "f1": 0.538093, + "f1_weighted": 0.538093, + "ap": 0.520497, + "ap_weighted": 0.520497 + }, + { + "accuracy": 0.467714, + "f1": 0.371735, + "f1_weighted": 0.371735, + "ap": 0.488632, + "ap_weighted": 0.488632 + }, + { + "accuracy": 0.457429, + "f1": 0.407608, + "f1_weighted": 0.407608, + "ap": 0.483029, + "ap_weighted": 0.483029 + }, + { + "accuracy": 0.591857, + "f1": 0.591306, + "f1_weighted": 0.591306, + "ap": 0.555035, + "ap_weighted": 0.555035 + }, + { + "accuracy": 0.512286, + "f1": 0.450811, + "f1_weighted": 0.450811, + "ap": 0.506599, + "ap_weighted": 0.506599 + }, + { + "accuracy": 0.482, + "f1": 0.440896, + "f1_weighted": 0.440896, + "ap": 0.491708, + "ap_weighted": 0.491708 + }, + { + "accuracy": 0.585286, + "f1": 0.582039, + "f1_weighted": 0.582039, + "ap": 0.551473, + "ap_weighted": 0.551473 + }, + { + "accuracy": 0.453571, + "f1": 0.452664, + "f1_weighted": 0.452664, + "ap": 0.478779, + "ap_weighted": 0.478779 + }, + { + "accuracy": 0.586571, + "f1": 0.580206, + "f1_weighted": 0.580206, + "ap": 0.549299, + "ap_weighted": 0.549299 + }, + { + "accuracy": 0.437143, + "f1": 0.43317, + "f1_weighted": 0.43317, + "ap": 0.471956, + "ap_weighted": 0.471956 + } + ], + "main_score": 0.5112, + "hf_subset": "fas", + "languages": [ + "fas-Arab" + ] + }, + { + "accuracy": 0.648325, + "f1": 0.646975, + "f1_weighted": 0.646968, + "ap": 0.595826, + "ap_weighted": 0.595826, + "scores_per_experiment": [ + { + "accuracy": 0.680945, + "f1": 0.677049, + "f1_weighted": 0.67699, + "ap": 0.616646, + "ap_weighted": 0.616646 + }, + { + "accuracy": 0.693575, + "f1": 0.690849, + "f1_weighted": 0.690801, + "ap": 0.627648, + "ap_weighted": 0.627648 + }, + { + "accuracy": 0.669962, + "f1": 0.66585, + "f1_weighted": 0.665911, + "ap": 0.621093, + "ap_weighted": 0.621093 + }, + { + "accuracy": 0.617243, + "f1": 0.617187, + "f1_weighted": 0.617195, + "ap": 0.57187, + "ap_weighted": 0.57187 + }, + { + "accuracy": 0.624931, + "f1": 0.624408, + "f1_weighted": 0.624385, + "ap": 0.576213, + "ap_weighted": 0.576213 + }, + { + "accuracy": 0.632619, + "f1": 0.632584, + "f1_weighted": 0.63259, + "ap": 0.583417, + "ap_weighted": 0.583417 + }, + { + "accuracy": 0.604613, + "f1": 0.603867, + "f1_weighted": 0.603839, + "ap": 0.561605, + "ap_weighted": 0.561605 + }, + { + "accuracy": 0.632619, + "f1": 0.632424, + "f1_weighted": 0.63241, + "ap": 0.582329, + "ap_weighted": 0.582329 + }, + { + "accuracy": 0.649094, + "f1": 0.647946, + "f1_weighted": 0.647979, + "ap": 0.598736, + "ap_weighted": 0.598736 + }, + { + "accuracy": 0.67765, + "f1": 0.677584, + "f1_weighted": 0.677576, + "ap": 0.618707, + "ap_weighted": 0.618707 + } + ], + "main_score": 0.648325, + "hf_subset": "eng", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.518713, + "f1": 0.506405, + "f1_weighted": 0.526183, + "ap": 0.36355, + "ap_weighted": 0.36355, + "scores_per_experiment": [ + { + "accuracy": 0.48538, + "f1": 0.485222, + "f1_weighted": 0.487915, + "ap": 0.363873, + "ap_weighted": 0.363873 + }, + { + "accuracy": 0.526316, + "f1": 0.515106, + "f1_weighted": 0.537095, + "ap": 0.364527, + "ap_weighted": 0.364527 + }, + { + "accuracy": 0.526316, + "f1": 0.521008, + "f1_weighted": 0.536046, + "ap": 0.372609, + "ap_weighted": 0.372609 + }, + { + "accuracy": 0.508772, + "f1": 0.484644, + "f1_weighted": 0.517901, + "ap": 0.345444, + "ap_weighted": 0.345444 + }, + { + "accuracy": 0.51462, + "f1": 0.501247, + "f1_weighted": 0.525604, + "ap": 0.356161, + "ap_weighted": 0.356161 + }, + { + "accuracy": 0.578947, + "f1": 0.558266, + "f1_weighted": 0.586773, + "ap": 0.386032, + "ap_weighted": 0.386032 + }, + { + "accuracy": 0.532164, + "f1": 0.521946, + "f1_weighted": 0.54279, + "ap": 0.368941, + "ap_weighted": 0.368941 + }, + { + "accuracy": 0.526316, + "f1": 0.523119, + "f1_weighted": 0.534764, + "ap": 0.376765, + "ap_weighted": 0.376765 + }, + { + "accuracy": 0.532164, + "f1": 0.497354, + "f1_weighted": 0.536805, + "ap": 0.34996, + "ap_weighted": 0.34996 + }, + { + "accuracy": 0.45614, + "f1": 0.45614, + "f1_weighted": 0.45614, + "ap": 0.351185, + "ap_weighted": 0.351185 + } + ], + "main_score": 0.518713, + "hf_subset": "mlt", + "languages": [ + "mlt-Latn" + ] + }, + { + "accuracy": 0.636107, + "f1": 0.604299, + "f1_weighted": 0.651134, + "ap": 0.801924, + "ap_weighted": 0.801924, + "scores_per_experiment": [ + { + "accuracy": 0.68945, + "f1": 0.660023, + "f1_weighted": 0.706542, + "ap": 0.828632, + "ap_weighted": 0.828632 + }, + { + "accuracy": 0.548291, + "f1": 0.540251, + "f1_weighted": 0.568527, + "ap": 0.787705, + "ap_weighted": 0.787705 + }, + { + "accuracy": 0.68945, + "f1": 0.636914, + "f1_weighted": 0.701148, + "ap": 0.800559, + "ap_weighted": 0.800559 + }, + { + "accuracy": 0.592868, + "f1": 0.574177, + "f1_weighted": 0.615668, + "ap": 0.792007, + "ap_weighted": 0.792007 + }, + { + "accuracy": 0.683507, + "f1": 0.660273, + "f1_weighted": 0.701592, + "ap": 0.836209, + "ap_weighted": 0.836209 + }, + { + "accuracy": 0.676077, + "f1": 0.627516, + "f1_weighted": 0.690066, + "ap": 0.798036, + "ap_weighted": 0.798036 + }, + { + "accuracy": 0.473997, + "f1": 0.473987, + "f1_weighted": 0.475077, + "ap": 0.780626, + "ap_weighted": 0.780626 + }, + { + "accuracy": 0.637444, + "f1": 0.579943, + "f1_weighted": 0.652224, + "ap": 0.772196, + "ap_weighted": 0.772196 + }, + { + "accuracy": 0.702823, + "f1": 0.658272, + "f1_weighted": 0.715657, + "ap": 0.815175, + "ap_weighted": 0.815175 + }, + { + "accuracy": 0.667162, + "f1": 0.631637, + "f1_weighted": 0.68484, + "ap": 0.808096, + "ap_weighted": 0.808096 + } + ], + "main_score": 0.636107, + "hf_subset": "bam", + "languages": [ + "bam-Latn" + ] + }, + { + "accuracy": 0.618062, + "f1": 0.526559, + "f1_weighted": 0.664631, + "ap": 0.87631, + "ap_weighted": 0.87631, + "scores_per_experiment": [ + { + "accuracy": 0.546256, + "f1": 0.498574, + "f1_weighted": 0.605517, + "ap": 0.885218, + "ap_weighted": 0.885218 + }, + { + "accuracy": 0.814978, + "f1": 0.627694, + "f1_weighted": 0.810325, + "ap": 0.879008, + "ap_weighted": 0.879008 + }, + { + "accuracy": 0.638767, + "f1": 0.553921, + "f1_weighted": 0.688474, + "ap": 0.886884, + "ap_weighted": 0.886884 + }, + { + "accuracy": 0.577093, + "f1": 0.487295, + "f1_weighted": 0.635697, + "ap": 0.859596, + "ap_weighted": 0.859596 + }, + { + "accuracy": 0.625551, + "f1": 0.49926, + "f1_weighted": 0.673186, + "ap": 0.854645, + "ap_weighted": 0.854645 + }, + { + "accuracy": 0.678414, + "f1": 0.579726, + "f1_weighted": 0.720581, + "ap": 0.890179, + "ap_weighted": 0.890179 + }, + { + "accuracy": 0.603524, + "f1": 0.535259, + "f1_weighted": 0.65845, + "ap": 0.887942, + "ap_weighted": 0.887942 + }, + { + "accuracy": 0.524229, + "f1": 0.472638, + "f1_weighted": 0.58672, + "ap": 0.870758, + "ap_weighted": 0.870758 + }, + { + "accuracy": 0.480176, + "f1": 0.447757, + "f1_weighted": 0.540299, + "ap": 0.873702, + "ap_weighted": 0.873702 + }, + { + "accuracy": 0.69163, + "f1": 0.563462, + "f1_weighted": 0.727059, + "ap": 0.875172, + "ap_weighted": 0.875172 + } + ], + "main_score": 0.618062, + "hf_subset": "eus", + "languages": [ + "eus-Latn" + ] + }, + { + "accuracy": 0.525195, + "f1": 0.517302, + "f1_weighted": 0.517302, + "ap": 0.51477, + "ap_weighted": 0.51477, + "scores_per_experiment": [ + { + "accuracy": 0.512207, + "f1": 0.508106, + "f1_weighted": 0.508106, + "ap": 0.50623, + "ap_weighted": 0.50623 + }, + { + "accuracy": 0.54834, + "f1": 0.548223, + "f1_weighted": 0.548223, + "ap": 0.526584, + "ap_weighted": 0.526584 + }, + { + "accuracy": 0.503906, + "f1": 0.498935, + "f1_weighted": 0.498935, + "ap": 0.501966, + "ap_weighted": 0.501966 + }, + { + "accuracy": 0.552246, + "f1": 0.539169, + "f1_weighted": 0.539169, + "ap": 0.53024, + "ap_weighted": 0.53024 + }, + { + "accuracy": 0.546387, + "f1": 0.540643, + "f1_weighted": 0.540643, + "ap": 0.525965, + "ap_weighted": 0.525965 + }, + { + "accuracy": 0.458008, + "f1": 0.447073, + "f1_weighted": 0.447073, + "ap": 0.48038, + "ap_weighted": 0.48038 + }, + { + "accuracy": 0.554199, + "f1": 0.554168, + "f1_weighted": 0.554168, + "ap": 0.529989, + "ap_weighted": 0.529989 + }, + { + "accuracy": 0.541016, + "f1": 0.506173, + "f1_weighted": 0.506173, + "ap": 0.524097, + "ap_weighted": 0.524097 + }, + { + "accuracy": 0.556641, + "f1": 0.552792, + "f1_weighted": 0.552792, + "ap": 0.532259, + "ap_weighted": 0.532259 + }, + { + "accuracy": 0.479004, + "f1": 0.477734, + "f1_weighted": 0.477734, + "ap": 0.489991, + "ap_weighted": 0.489991 + } + ], + "main_score": 0.525195, + "hf_subset": "cym", + "languages": [ + "cym-Latn" + ] + } + ] + }, + "evaluation_time": 127.30599641799927, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/MyanmarNews.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/MyanmarNews.json new file mode 100644 index 0000000000..c8151b9746 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/MyanmarNews.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "b899ec06227db3679b0fe3c4188a6b48cc0b65eb", + "task_name": "MyanmarNews", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.31084, + "f1": 0.268059, + "f1_weighted": 0.267761, + "scores_per_experiment": [ + { + "accuracy": 0.310547, + "f1": 0.293714, + "f1_weighted": 0.292861 + }, + { + "accuracy": 0.275879, + "f1": 0.1963, + "f1_weighted": 0.198012 + }, + { + "accuracy": 0.302246, + "f1": 0.257126, + "f1_weighted": 0.255644 + }, + { + "accuracy": 0.302734, + "f1": 0.266526, + "f1_weighted": 0.265572 + }, + { + "accuracy": 0.337891, + "f1": 0.310963, + "f1_weighted": 0.311518 + }, + { + "accuracy": 0.299316, + "f1": 0.234109, + "f1_weighted": 0.233801 + }, + { + "accuracy": 0.297363, + "f1": 0.242255, + "f1_weighted": 0.240787 + }, + { + "accuracy": 0.333496, + "f1": 0.287577, + "f1_weighted": 0.288468 + }, + { + "accuracy": 0.288574, + "f1": 0.245359, + "f1_weighted": 0.243716 + }, + { + "accuracy": 0.360352, + "f1": 0.346665, + "f1_weighted": 0.347234 + } + ], + "main_score": 0.31084, + "hf_subset": "default", + "languages": [ + "mya-Mymr" + ] + } + ] + }, + "evaluation_time": 4.669776201248169, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/NFCorpus.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/NFCorpus.json deleted file mode 100644 index f5f32271c5..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/NFCorpus.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "test": { - "evaluation_time": 37.2, - "map_at_1": 0.04977, - "map_at_10": 0.10938, - "map_at_100": 0.13433, - "map_at_1000": 0.14672, - "map_at_3": 0.0822, - "map_at_5": 0.09539, - "ndcg_at_1": 0.39009, - "ndcg_at_10": 0.30223, - "ndcg_at_100": 0.27687, - "ndcg_at_1000": 0.36538, - "ndcg_at_3": 0.35294, - "ndcg_at_5": 0.33581, - "precision_at_1": 0.40248, - "precision_at_10": 0.21858, - "precision_at_100": 0.07015, - "precision_at_1000": 0.01972, - "precision_at_3": 0.33127, - "precision_at_5": 0.28978, - "recall_at_1": 0.04977, - "recall_at_10": 0.14696, - "recall_at_100": 0.28256, - "recall_at_1000": 0.61065, - "recall_at_3": 0.09239, - "recall_at_5": 0.11778 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "NFCorpus", - "dataset_revision": "7eb63cc0c1eb59324d709ebed25fcab851fa7610" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/NLPTwitterAnalysisClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/NLPTwitterAnalysisClassification.json new file mode 100644 index 0000000000..354f1a5765 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/NLPTwitterAnalysisClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "4ceb1312583fd2c7c73ad2d550b726124dcd39a0", + "task_name": "NLPTwitterAnalysisClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.104926, + "f1": 0.038971, + "f1_weighted": 0.049477, + "scores_per_experiment": [ + { + "accuracy": 0.094853, + "f1": 0.03778, + "f1_weighted": 0.051249 + }, + { + "accuracy": 0.101471, + "f1": 0.041977, + "f1_weighted": 0.053592 + }, + { + "accuracy": 0.106618, + "f1": 0.033638, + "f1_weighted": 0.043094 + }, + { + "accuracy": 0.113235, + "f1": 0.04074, + "f1_weighted": 0.052188 + }, + { + "accuracy": 0.079412, + "f1": 0.033179, + "f1_weighted": 0.038237 + }, + { + "accuracy": 0.113235, + "f1": 0.043466, + "f1_weighted": 0.05437 + }, + { + "accuracy": 0.114706, + "f1": 0.040412, + "f1_weighted": 0.051848 + }, + { + "accuracy": 0.109559, + "f1": 0.047438, + "f1_weighted": 0.05934 + }, + { + "accuracy": 0.105147, + "f1": 0.030834, + "f1_weighted": 0.040414 + }, + { + "accuracy": 0.111029, + "f1": 0.040247, + "f1_weighted": 0.050433 + } + ], + "main_score": 0.104926, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 22.17754626274109, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/NQ.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/NQ.json deleted file mode 100644 index 1763e3f6ee..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/NQ.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "test": { - "evaluation_time": 8457.39, - "map_at_1": 0.2878, - "map_at_10": 0.43055, - "map_at_100": 0.44043, - "map_at_1000": 0.44085, - "map_at_3": 0.38898, - "map_at_5": 0.41353, - "ndcg_at_1": 0.32184, - "ndcg_at_10": 0.50469, - "ndcg_at_100": 0.5478, - "ndcg_at_1000": 0.55794, - "ndcg_at_3": 0.42677, - "ndcg_at_5": 0.46721, - "precision_at_1": 0.32184, - "precision_at_10": 0.0836, - "precision_at_100": 0.01084, - "precision_at_1000": 0.00118, - "precision_at_3": 0.19341, - "precision_at_5": 0.1398, - "recall_at_1": 0.2878, - "recall_at_10": 0.70551, - "recall_at_100": 0.89499, - "recall_at_1000": 0.97106, - "recall_at_3": 0.50451, - "recall_at_5": 0.59671 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "NQ", - "dataset_revision": "6062aefc120bfe8ece5897809fb2e53bfe0d128c" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/NYSJudicialEthicsLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/NYSJudicialEthicsLegalBenchClassification.json new file mode 100644 index 0000000000..171d9b9c70 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/NYSJudicialEthicsLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "NYSJudicialEthicsLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.496575, + "f1": 0.473603, + "f1_weighted": 0.469084, + "ap": 0.523614, + "ap_weighted": 0.523614, + "scores_per_experiment": [ + { + "accuracy": 0.496575, + "f1": 0.473603, + "f1_weighted": 0.469084, + "ap": 0.523614, + "ap_weighted": 0.523614 + }, + { + "accuracy": 0.496575, + "f1": 0.473603, + "f1_weighted": 0.469084, + "ap": 0.523614, + "ap_weighted": 0.523614 + }, + { + "accuracy": 0.496575, + "f1": 0.473603, + "f1_weighted": 0.469084, + "ap": 0.523614, + "ap_weighted": 0.523614 + }, + { + "accuracy": 0.496575, + "f1": 0.473603, + "f1_weighted": 0.469084, + "ap": 0.523614, + "ap_weighted": 0.523614 + }, + { + "accuracy": 0.496575, + "f1": 0.473603, + "f1_weighted": 0.469084, + "ap": 0.523614, + "ap_weighted": 0.523614 + }, + { + "accuracy": 0.496575, + "f1": 0.473603, + "f1_weighted": 0.469084, + "ap": 0.523614, + "ap_weighted": 0.523614 + }, + { + "accuracy": 0.496575, + "f1": 0.473603, + "f1_weighted": 0.469084, + "ap": 0.523614, + "ap_weighted": 0.523614 + }, + { + "accuracy": 0.496575, + "f1": 0.473603, + "f1_weighted": 0.469084, + "ap": 0.523614, + "ap_weighted": 0.523614 + }, + { + "accuracy": 0.496575, + "f1": 0.473603, + "f1_weighted": 0.469084, + "ap": 0.523614, + "ap_weighted": 0.523614 + }, + { + "accuracy": 0.496575, + "f1": 0.473603, + "f1_weighted": 0.469084, + "ap": 0.523614, + "ap_weighted": 0.523614 + } + ], + "main_score": 0.496575, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.7468671798706055, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/NaijaSenti.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/NaijaSenti.json new file mode 100644 index 0000000000..e9f3cd2460 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/NaijaSenti.json @@ -0,0 +1,259 @@ +{ + "dataset_revision": "a3d0415a828178edf3466246f49cfcd83b946ab3", + "task_name": "NaijaSenti", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.493381, + "f1": 0.475893, + "f1_weighted": 0.475182, + "scores_per_experiment": [ + { + "accuracy": 0.518197, + "f1": 0.50364, + "f1_weighted": 0.502991 + }, + { + "accuracy": 0.503489, + "f1": 0.484037, + "f1_weighted": 0.482909 + }, + { + "accuracy": 0.460494, + "f1": 0.429749, + "f1_weighted": 0.428457 + }, + { + "accuracy": 0.535357, + "f1": 0.537906, + "f1_weighted": 0.537527 + }, + { + "accuracy": 0.452763, + "f1": 0.417055, + "f1_weighted": 0.41642 + }, + { + "accuracy": 0.476146, + "f1": 0.480406, + "f1_weighted": 0.480005 + }, + { + "accuracy": 0.512729, + "f1": 0.509905, + "f1_weighted": 0.509783 + }, + { + "accuracy": 0.45974, + "f1": 0.414146, + "f1_weighted": 0.412599 + }, + { + "accuracy": 0.530643, + "f1": 0.524681, + "f1_weighted": 0.524562 + }, + { + "accuracy": 0.484254, + "f1": 0.457407, + "f1_weighted": 0.456561 + } + ], + "main_score": 0.493381, + "hf_subset": "hau", + "languages": [ + "hau-Latn" + ] + }, + { + "accuracy": 0.429902, + "f1": 0.351258, + "f1_weighted": 0.380321, + "scores_per_experiment": [ + { + "accuracy": 0.438077, + "f1": 0.34135, + "f1_weighted": 0.383746 + }, + { + "accuracy": 0.46931, + "f1": 0.397364, + "f1_weighted": 0.430674 + }, + { + "accuracy": 0.348995, + "f1": 0.335867, + "f1_weighted": 0.32348 + }, + { + "accuracy": 0.458446, + "f1": 0.442411, + "f1_weighted": 0.453063 + }, + { + "accuracy": 0.461434, + "f1": 0.329255, + "f1_weighted": 0.376477 + }, + { + "accuracy": 0.439707, + "f1": 0.346948, + "f1_weighted": 0.388197 + }, + { + "accuracy": 0.448941, + "f1": 0.380792, + "f1_weighted": 0.413285 + }, + { + "accuracy": 0.422325, + "f1": 0.393525, + "f1_weighted": 0.417755 + }, + { + "accuracy": 0.364204, + "f1": 0.279893, + "f1_weighted": 0.297174 + }, + { + "accuracy": 0.447583, + "f1": 0.265179, + "f1_weighted": 0.319355 + } + ], + "main_score": 0.429902, + "hf_subset": "ibo", + "languages": [ + "ibo-Latn" + ] + }, + { + "accuracy": 0.372508, + "f1": 0.330505, + "f1_weighted": 0.403595, + "scores_per_experiment": [ + { + "accuracy": 0.386134, + "f1": 0.349505, + "f1_weighted": 0.422781 + }, + { + "accuracy": 0.37819, + "f1": 0.349479, + "f1_weighted": 0.41437 + }, + { + "accuracy": 0.42754, + "f1": 0.353046, + "f1_weighted": 0.450951 + }, + { + "accuracy": 0.3091, + "f1": 0.296437, + "f1_weighted": 0.33348 + }, + { + "accuracy": 0.397208, + "f1": 0.338235, + "f1_weighted": 0.427616 + }, + { + "accuracy": 0.336302, + "f1": 0.296698, + "f1_weighted": 0.377548 + }, + { + "accuracy": 0.39143, + "f1": 0.315797, + "f1_weighted": 0.4132 + }, + { + "accuracy": 0.330766, + "f1": 0.308123, + "f1_weighted": 0.366936 + }, + { + "accuracy": 0.403226, + "f1": 0.363101, + "f1_weighted": 0.430918 + }, + { + "accuracy": 0.36519, + "f1": 0.334632, + "f1_weighted": 0.398152 + } + ], + "main_score": 0.372508, + "hf_subset": "pcm", + "languages": [ + "pcm-Latn" + ] + }, + { + "accuracy": 0.356257, + "f1": 0.315789, + "f1_weighted": 0.319531, + "scores_per_experiment": [ + { + "accuracy": 0.323588, + "f1": 0.305738, + "f1_weighted": 0.308996 + }, + { + "accuracy": 0.375637, + "f1": 0.336235, + "f1_weighted": 0.326214 + }, + { + "accuracy": 0.393134, + "f1": 0.385064, + "f1_weighted": 0.39886 + }, + { + "accuracy": 0.332004, + "f1": 0.28901, + "f1_weighted": 0.294737 + }, + { + "accuracy": 0.361462, + "f1": 0.336648, + "f1_weighted": 0.341154 + }, + { + "accuracy": 0.419934, + "f1": 0.345563, + "f1_weighted": 0.361916 + }, + { + "accuracy": 0.263787, + "f1": 0.233602, + "f1_weighted": 0.208008 + }, + { + "accuracy": 0.345958, + "f1": 0.331166, + "f1_weighted": 0.330016 + }, + { + "accuracy": 0.407973, + "f1": 0.274543, + "f1_weighted": 0.317315 + }, + { + "accuracy": 0.339092, + "f1": 0.320321, + "f1_weighted": 0.308095 + } + ], + "main_score": 0.356257, + "hf_subset": "yor", + "languages": [ + "yor-Latn" + ] + } + ] + }, + "evaluation_time": 21.00438380241394, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/NepaliNewsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/NepaliNewsClassification.json new file mode 100644 index 0000000000..3fa8bb609c --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/NepaliNewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "79125f20d858a08f71ec4923169a6545221725c4", + "task_name": "NepaliNewsClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.518506, + "f1": 0.501644, + "f1_weighted": 0.497413, + "scores_per_experiment": [ + { + "accuracy": 0.558105, + "f1": 0.550597, + "f1_weighted": 0.548221 + }, + { + "accuracy": 0.435059, + "f1": 0.42808, + "f1_weighted": 0.422278 + }, + { + "accuracy": 0.552734, + "f1": 0.54329, + "f1_weighted": 0.542058 + }, + { + "accuracy": 0.522461, + "f1": 0.512351, + "f1_weighted": 0.512418 + }, + { + "accuracy": 0.491699, + "f1": 0.473939, + "f1_weighted": 0.465676 + }, + { + "accuracy": 0.486816, + "f1": 0.409994, + "f1_weighted": 0.396894 + }, + { + "accuracy": 0.461426, + "f1": 0.470295, + "f1_weighted": 0.464433 + }, + { + "accuracy": 0.579102, + "f1": 0.568844, + "f1_weighted": 0.567408 + }, + { + "accuracy": 0.533691, + "f1": 0.500566, + "f1_weighted": 0.499229 + }, + { + "accuracy": 0.563965, + "f1": 0.558488, + "f1_weighted": 0.555518 + } + ], + "main_score": 0.518506, + "hf_subset": "default", + "languages": [ + "nep-Deva" + ] + } + ] + }, + "evaluation_time": 4.143486022949219, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/NewsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/NewsClassification.json new file mode 100644 index 0000000000..ca03b7849a --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/NewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "eb185aade064a813bc0b7f42de02595523103ca4", + "task_name": "NewsClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.768605, + "f1": 0.767189, + "f1_weighted": 0.767189, + "scores_per_experiment": [ + { + "accuracy": 0.769342, + "f1": 0.76745, + "f1_weighted": 0.76745 + }, + { + "accuracy": 0.781316, + "f1": 0.778558, + "f1_weighted": 0.778558 + }, + { + "accuracy": 0.783553, + "f1": 0.784546, + "f1_weighted": 0.784546 + }, + { + "accuracy": 0.802105, + "f1": 0.801624, + "f1_weighted": 0.801624 + }, + { + "accuracy": 0.766711, + "f1": 0.764449, + "f1_weighted": 0.764449 + }, + { + "accuracy": 0.750658, + "f1": 0.750001, + "f1_weighted": 0.750001 + }, + { + "accuracy": 0.788026, + "f1": 0.785295, + "f1_weighted": 0.785295 + }, + { + "accuracy": 0.722237, + "f1": 0.72009, + "f1_weighted": 0.72009 + }, + { + "accuracy": 0.776447, + "f1": 0.775463, + "f1_weighted": 0.775463 + }, + { + "accuracy": 0.745658, + "f1": 0.744412, + "f1_weighted": 0.744412 + } + ], + "main_score": 0.768605, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 14.223294496536255, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/NoRecClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/NoRecClassification.json new file mode 100644 index 0000000000..078f224a63 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/NoRecClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "5b740b7c42c73d586420812a35745fc37118862f", + "task_name": "NoRecClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.386816, + "f1": 0.368869, + "f1_weighted": 0.403671, + "scores_per_experiment": [ + { + "accuracy": 0.393066, + "f1": 0.385841, + "f1_weighted": 0.415712 + }, + { + "accuracy": 0.353516, + "f1": 0.317002, + "f1_weighted": 0.361345 + }, + { + "accuracy": 0.421875, + "f1": 0.405419, + "f1_weighted": 0.440535 + }, + { + "accuracy": 0.386719, + "f1": 0.373238, + "f1_weighted": 0.405735 + }, + { + "accuracy": 0.34082, + "f1": 0.329854, + "f1_weighted": 0.355266 + }, + { + "accuracy": 0.395996, + "f1": 0.375327, + "f1_weighted": 0.413655 + }, + { + "accuracy": 0.408203, + "f1": 0.389243, + "f1_weighted": 0.431137 + }, + { + "accuracy": 0.391113, + "f1": 0.373687, + "f1_weighted": 0.405697 + }, + { + "accuracy": 0.400879, + "f1": 0.381577, + "f1_weighted": 0.418666 + }, + { + "accuracy": 0.375977, + "f1": 0.357498, + "f1_weighted": 0.388967 + } + ], + "main_score": 0.386816, + "hf_subset": "default", + "languages": [ + "nob-Latn" + ] + } + ] + }, + "evaluation_time": 3.1552419662475586, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/NordicLangClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/NordicLangClassification.json new file mode 100644 index 0000000000..acfcc57da2 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/NordicLangClassification.json @@ -0,0 +1,78 @@ +{ + "dataset_revision": "e254179d18ab0165fdb6dbef91178266222bee2a", + "task_name": "NordicLangClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.562133, + "f1": 0.549176, + "f1_weighted": 0.552148, + "scores_per_experiment": [ + { + "accuracy": 0.551667, + "f1": 0.538034, + "f1_weighted": 0.542692 + }, + { + "accuracy": 0.591333, + "f1": 0.576066, + "f1_weighted": 0.579022 + }, + { + "accuracy": 0.551667, + "f1": 0.543358, + "f1_weighted": 0.545339 + }, + { + "accuracy": 0.577333, + "f1": 0.562904, + "f1_weighted": 0.565887 + }, + { + "accuracy": 0.581, + "f1": 0.568163, + "f1_weighted": 0.571609 + }, + { + "accuracy": 0.553, + "f1": 0.537893, + "f1_weighted": 0.541941 + }, + { + "accuracy": 0.566, + "f1": 0.551889, + "f1_weighted": 0.555969 + }, + { + "accuracy": 0.540333, + "f1": 0.528016, + "f1_weighted": 0.529615 + }, + { + "accuracy": 0.544333, + "f1": 0.534094, + "f1_weighted": 0.535523 + }, + { + "accuracy": 0.564667, + "f1": 0.551342, + "f1_weighted": 0.553882 + } + ], + "main_score": 0.562133, + "hf_subset": "default", + "languages": [ + "nob-Latn", + "nno-Latn", + "dan-Latn", + "swe-Latn", + "isl-Latn", + "fao-Latn" + ] + } + ] + }, + "evaluation_time": 16.85726571083069, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/NorwegianParliamentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/NorwegianParliamentClassification.json new file mode 100644 index 0000000000..efd8bb2d50 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/NorwegianParliamentClassification.json @@ -0,0 +1,181 @@ +{ + "dataset_revision": "f7393532774c66312378d30b197610b43d751972", + "task_name": "NorwegianParliamentClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.537333, + "f1": 0.533908, + "f1_weighted": 0.533908, + "ap": 0.520438, + "ap_weighted": 0.520438, + "scores_per_experiment": [ + { + "accuracy": 0.538333, + "f1": 0.538251, + "f1_weighted": 0.538251, + "ap": 0.520676, + "ap_weighted": 0.520676 + }, + { + "accuracy": 0.528333, + "f1": 0.526121, + "f1_weighted": 0.526121, + "ap": 0.515097, + "ap_weighted": 0.515097 + }, + { + "accuracy": 0.525, + "f1": 0.515733, + "f1_weighted": 0.515733, + "ap": 0.51299, + "ap_weighted": 0.51299 + }, + { + "accuracy": 0.5325, + "f1": 0.532492, + "f1_weighted": 0.532492, + "ap": 0.517298, + "ap_weighted": 0.517298 + }, + { + "accuracy": 0.553333, + "f1": 0.551808, + "f1_weighted": 0.551808, + "ap": 0.529887, + "ap_weighted": 0.529887 + }, + { + "accuracy": 0.514167, + "f1": 0.509974, + "f1_weighted": 0.509974, + "ap": 0.507253, + "ap_weighted": 0.507253 + }, + { + "accuracy": 0.518333, + "f1": 0.518171, + "f1_weighted": 0.518171, + "ap": 0.509491, + "ap_weighted": 0.509491 + }, + { + "accuracy": 0.555833, + "f1": 0.549793, + "f1_weighted": 0.549793, + "ap": 0.530448, + "ap_weighted": 0.530448 + }, + { + "accuracy": 0.580833, + "f1": 0.580476, + "f1_weighted": 0.580476, + "ap": 0.547355, + "ap_weighted": 0.547355 + }, + { + "accuracy": 0.526667, + "f1": 0.516261, + "f1_weighted": 0.516261, + "ap": 0.513883, + "ap_weighted": 0.513883 + } + ], + "main_score": 0.537333, + "hf_subset": "default", + "languages": [ + "nob-Latn" + ] + } + ], + "validation": [ + { + "accuracy": 0.53175, + "f1": 0.528887, + "f1_weighted": 0.528887, + "ap": 0.517568, + "ap_weighted": 0.517568, + "scores_per_experiment": [ + { + "accuracy": 0.5, + "f1": 0.499999, + "f1_weighted": 0.499999, + "ap": 0.5, + "ap_weighted": 0.5 + }, + { + "accuracy": 0.510833, + "f1": 0.510082, + "f1_weighted": 0.510082, + "ap": 0.505544, + "ap_weighted": 0.505544 + }, + { + "accuracy": 0.530833, + "f1": 0.524998, + "f1_weighted": 0.524998, + "ap": 0.516195, + "ap_weighted": 0.516195 + }, + { + "accuracy": 0.555833, + "f1": 0.555808, + "f1_weighted": 0.555808, + "ap": 0.531082, + "ap_weighted": 0.531082 + }, + { + "accuracy": 0.558333, + "f1": 0.556159, + "f1_weighted": 0.556159, + "ap": 0.533123, + "ap_weighted": 0.533123 + }, + { + "accuracy": 0.525, + "f1": 0.522985, + "f1_weighted": 0.522985, + "ap": 0.513053, + "ap_weighted": 0.513053 + }, + { + "accuracy": 0.535, + "f1": 0.534937, + "f1_weighted": 0.534937, + "ap": 0.518697, + "ap_weighted": 0.518697 + }, + { + "accuracy": 0.530833, + "f1": 0.525687, + "f1_weighted": 0.525687, + "ap": 0.516203, + "ap_weighted": 0.516203 + }, + { + "accuracy": 0.576667, + "f1": 0.576097, + "f1_weighted": 0.576097, + "ap": 0.544676, + "ap_weighted": 0.544676 + }, + { + "accuracy": 0.494167, + "f1": 0.482123, + "f1_weighted": 0.482123, + "ap": 0.497109, + "ap_weighted": 0.497109 + } + ], + "main_score": 0.53175, + "hf_subset": "default", + "languages": [ + "nob-Latn" + ] + } + ] + }, + "evaluation_time": 27.0786030292511, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/NusaParagraphEmotionClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/NusaParagraphEmotionClassification.json new file mode 100644 index 0000000000..489db31914 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/NusaParagraphEmotionClassification.json @@ -0,0 +1,631 @@ +{ + "dataset_revision": "c61e8c3ee47d2dce296e9601195916b54c21d575", + "task_name": "NusaParagraphEmotionClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.2514, + "f1": 0.242879, + "f1_weighted": 0.249162, + "scores_per_experiment": [ + { + "accuracy": 0.252, + "f1": 0.228606, + "f1_weighted": 0.246233 + }, + { + "accuracy": 0.238, + "f1": 0.235394, + "f1_weighted": 0.238036 + }, + { + "accuracy": 0.282, + "f1": 0.264015, + "f1_weighted": 0.277526 + }, + { + "accuracy": 0.292, + "f1": 0.27479, + "f1_weighted": 0.28691 + }, + { + "accuracy": 0.234, + "f1": 0.233117, + "f1_weighted": 0.22488 + }, + { + "accuracy": 0.212, + "f1": 0.209964, + "f1_weighted": 0.207691 + }, + { + "accuracy": 0.264, + "f1": 0.263407, + "f1_weighted": 0.269682 + }, + { + "accuracy": 0.25, + "f1": 0.243539, + "f1_weighted": 0.253865 + }, + { + "accuracy": 0.25, + "f1": 0.24567, + "f1_weighted": 0.242697 + }, + { + "accuracy": 0.24, + "f1": 0.230283, + "f1_weighted": 0.244099 + } + ], + "main_score": 0.242879, + "hf_subset": "btk", + "languages": [ + "bbc-Latn" + ] + }, + { + "accuracy": 0.250625, + "f1": 0.234031, + "f1_weighted": 0.253865, + "scores_per_experiment": [ + { + "accuracy": 0.25875, + "f1": 0.256534, + "f1_weighted": 0.279068 + }, + { + "accuracy": 0.225, + "f1": 0.222971, + "f1_weighted": 0.216552 + }, + { + "accuracy": 0.24, + "f1": 0.225735, + "f1_weighted": 0.242469 + }, + { + "accuracy": 0.19125, + "f1": 0.182542, + "f1_weighted": 0.191633 + }, + { + "accuracy": 0.2525, + "f1": 0.233232, + "f1_weighted": 0.257587 + }, + { + "accuracy": 0.30875, + "f1": 0.285601, + "f1_weighted": 0.307585 + }, + { + "accuracy": 0.26875, + "f1": 0.238753, + "f1_weighted": 0.277722 + }, + { + "accuracy": 0.2275, + "f1": 0.219588, + "f1_weighted": 0.224355 + }, + { + "accuracy": 0.26625, + "f1": 0.23836, + "f1_weighted": 0.270985 + }, + { + "accuracy": 0.2675, + "f1": 0.236994, + "f1_weighted": 0.270694 + } + ], + "main_score": 0.234031, + "hf_subset": "bew", + "languages": [ + "bew-Latn" + ] + }, + { + "accuracy": 0.331333, + "f1": 0.255585, + "f1_weighted": 0.310441, + "scores_per_experiment": [ + { + "accuracy": 0.333333, + "f1": 0.251718, + "f1_weighted": 0.314835 + }, + { + "accuracy": 0.303333, + "f1": 0.23365, + "f1_weighted": 0.280721 + }, + { + "accuracy": 0.326667, + "f1": 0.254378, + "f1_weighted": 0.302014 + }, + { + "accuracy": 0.35, + "f1": 0.273003, + "f1_weighted": 0.327212 + }, + { + "accuracy": 0.373333, + "f1": 0.277689, + "f1_weighted": 0.349644 + }, + { + "accuracy": 0.353333, + "f1": 0.271934, + "f1_weighted": 0.331516 + }, + { + "accuracy": 0.333333, + "f1": 0.259154, + "f1_weighted": 0.313176 + }, + { + "accuracy": 0.32, + "f1": 0.250689, + "f1_weighted": 0.296641 + }, + { + "accuracy": 0.303333, + "f1": 0.236167, + "f1_weighted": 0.285843 + }, + { + "accuracy": 0.316667, + "f1": 0.247471, + "f1_weighted": 0.302807 + } + ], + "main_score": 0.255585, + "hf_subset": "bug", + "languages": [ + "bug-Latn" + ] + }, + { + "accuracy": 0.2425, + "f1": 0.226528, + "f1_weighted": 0.238058, + "scores_per_experiment": [ + { + "accuracy": 0.27375, + "f1": 0.268484, + "f1_weighted": 0.268797 + }, + { + "accuracy": 0.215, + "f1": 0.205803, + "f1_weighted": 0.212624 + }, + { + "accuracy": 0.24375, + "f1": 0.213862, + "f1_weighted": 0.234195 + }, + { + "accuracy": 0.2375, + "f1": 0.22411, + "f1_weighted": 0.233672 + }, + { + "accuracy": 0.2725, + "f1": 0.253828, + "f1_weighted": 0.271359 + }, + { + "accuracy": 0.26125, + "f1": 0.231987, + "f1_weighted": 0.254339 + }, + { + "accuracy": 0.2475, + "f1": 0.229605, + "f1_weighted": 0.246253 + }, + { + "accuracy": 0.225, + "f1": 0.225681, + "f1_weighted": 0.219049 + }, + { + "accuracy": 0.23, + "f1": 0.216225, + "f1_weighted": 0.225688 + }, + { + "accuracy": 0.21875, + "f1": 0.195692, + "f1_weighted": 0.214605 + } + ], + "main_score": 0.226528, + "hf_subset": "jav", + "languages": [ + "jav-Latn" + ] + }, + { + "accuracy": 0.2478, + "f1": 0.230932, + "f1_weighted": 0.251997, + "scores_per_experiment": [ + { + "accuracy": 0.268, + "f1": 0.231891, + "f1_weighted": 0.282247 + }, + { + "accuracy": 0.298, + "f1": 0.277889, + "f1_weighted": 0.30657 + }, + { + "accuracy": 0.168, + "f1": 0.157422, + "f1_weighted": 0.172905 + }, + { + "accuracy": 0.2, + "f1": 0.189384, + "f1_weighted": 0.206953 + }, + { + "accuracy": 0.264, + "f1": 0.253976, + "f1_weighted": 0.254325 + }, + { + "accuracy": 0.238, + "f1": 0.214905, + "f1_weighted": 0.243395 + }, + { + "accuracy": 0.228, + "f1": 0.227395, + "f1_weighted": 0.22156 + }, + { + "accuracy": 0.246, + "f1": 0.237807, + "f1_weighted": 0.231362 + }, + { + "accuracy": 0.284, + "f1": 0.259692, + "f1_weighted": 0.291888 + }, + { + "accuracy": 0.284, + "f1": 0.258958, + "f1_weighted": 0.308766 + } + ], + "main_score": 0.230932, + "hf_subset": "mad", + "languages": [ + "mad-Latn" + ] + }, + { + "accuracy": 0.2556, + "f1": 0.24811, + "f1_weighted": 0.253442, + "scores_per_experiment": [ + { + "accuracy": 0.232, + "f1": 0.220642, + "f1_weighted": 0.214139 + }, + { + "accuracy": 0.244, + "f1": 0.239493, + "f1_weighted": 0.242065 + }, + { + "accuracy": 0.25, + "f1": 0.251831, + "f1_weighted": 0.251499 + }, + { + "accuracy": 0.286, + "f1": 0.280378, + "f1_weighted": 0.289841 + }, + { + "accuracy": 0.252, + "f1": 0.247048, + "f1_weighted": 0.251954 + }, + { + "accuracy": 0.228, + "f1": 0.214838, + "f1_weighted": 0.213129 + }, + { + "accuracy": 0.236, + "f1": 0.227336, + "f1_weighted": 0.234939 + }, + { + "accuracy": 0.266, + "f1": 0.242788, + "f1_weighted": 0.274377 + }, + { + "accuracy": 0.27, + "f1": 0.263292, + "f1_weighted": 0.271434 + }, + { + "accuracy": 0.292, + "f1": 0.293455, + "f1_weighted": 0.291042 + } + ], + "main_score": 0.24811, + "hf_subset": "mak", + "languages": [ + "mak-Latn" + ] + }, + { + "accuracy": 0.241, + "f1": 0.231001, + "f1_weighted": 0.245461, + "scores_per_experiment": [ + { + "accuracy": 0.25875, + "f1": 0.241861, + "f1_weighted": 0.262492 + }, + { + "accuracy": 0.23375, + "f1": 0.228844, + "f1_weighted": 0.236208 + }, + { + "accuracy": 0.2025, + "f1": 0.197568, + "f1_weighted": 0.212354 + }, + { + "accuracy": 0.22375, + "f1": 0.211285, + "f1_weighted": 0.231843 + }, + { + "accuracy": 0.27875, + "f1": 0.272723, + "f1_weighted": 0.274891 + }, + { + "accuracy": 0.2525, + "f1": 0.240369, + "f1_weighted": 0.264803 + }, + { + "accuracy": 0.22, + "f1": 0.222843, + "f1_weighted": 0.221641 + }, + { + "accuracy": 0.25, + "f1": 0.230898, + "f1_weighted": 0.26031 + }, + { + "accuracy": 0.22, + "f1": 0.215375, + "f1_weighted": 0.220736 + }, + { + "accuracy": 0.27, + "f1": 0.248243, + "f1_weighted": 0.269336 + } + ], + "main_score": 0.231001, + "hf_subset": "min", + "languages": [ + "min-Latn" + ] + }, + { + "accuracy": 0.35775, + "f1": 0.314881, + "f1_weighted": 0.349314, + "scores_per_experiment": [ + { + "accuracy": 0.4075, + "f1": 0.365914, + "f1_weighted": 0.396214 + }, + { + "accuracy": 0.3175, + "f1": 0.279319, + "f1_weighted": 0.31099 + }, + { + "accuracy": 0.355, + "f1": 0.321015, + "f1_weighted": 0.346551 + }, + { + "accuracy": 0.4075, + "f1": 0.347197, + "f1_weighted": 0.400775 + }, + { + "accuracy": 0.295, + "f1": 0.256576, + "f1_weighted": 0.284188 + }, + { + "accuracy": 0.3825, + "f1": 0.341707, + "f1_weighted": 0.371458 + }, + { + "accuracy": 0.3625, + "f1": 0.31835, + "f1_weighted": 0.350906 + }, + { + "accuracy": 0.365, + "f1": 0.328792, + "f1_weighted": 0.36946 + }, + { + "accuracy": 0.3525, + "f1": 0.296392, + "f1_weighted": 0.3424 + }, + { + "accuracy": 0.3325, + "f1": 0.293552, + "f1_weighted": 0.320198 + } + ], + "main_score": 0.314881, + "hf_subset": "mui", + "languages": [ + "mui-Latn" + ] + }, + { + "accuracy": 0.403, + "f1": 0.386195, + "f1_weighted": 0.393211, + "scores_per_experiment": [ + { + "accuracy": 0.44, + "f1": 0.421245, + "f1_weighted": 0.43446 + }, + { + "accuracy": 0.353333, + "f1": 0.338628, + "f1_weighted": 0.344885 + }, + { + "accuracy": 0.443333, + "f1": 0.42562, + "f1_weighted": 0.436137 + }, + { + "accuracy": 0.42, + "f1": 0.395837, + "f1_weighted": 0.409685 + }, + { + "accuracy": 0.366667, + "f1": 0.357133, + "f1_weighted": 0.36373 + }, + { + "accuracy": 0.37, + "f1": 0.346558, + "f1_weighted": 0.346812 + }, + { + "accuracy": 0.45, + "f1": 0.436667, + "f1_weighted": 0.446482 + }, + { + "accuracy": 0.343333, + "f1": 0.325653, + "f1_weighted": 0.318599 + }, + { + "accuracy": 0.393333, + "f1": 0.380641, + "f1_weighted": 0.389499 + }, + { + "accuracy": 0.45, + "f1": 0.433969, + "f1_weighted": 0.441824 + } + ], + "main_score": 0.386195, + "hf_subset": "rej", + "languages": [ + "rej-Latn" + ] + }, + { + "accuracy": 0.240625, + "f1": 0.233454, + "f1_weighted": 0.239968, + "scores_per_experiment": [ + { + "accuracy": 0.23625, + "f1": 0.226224, + "f1_weighted": 0.238298 + }, + { + "accuracy": 0.26375, + "f1": 0.260949, + "f1_weighted": 0.253581 + }, + { + "accuracy": 0.28, + "f1": 0.272797, + "f1_weighted": 0.282457 + }, + { + "accuracy": 0.23375, + "f1": 0.219833, + "f1_weighted": 0.236441 + }, + { + "accuracy": 0.24875, + "f1": 0.244668, + "f1_weighted": 0.249022 + }, + { + "accuracy": 0.20125, + "f1": 0.195161, + "f1_weighted": 0.201866 + }, + { + "accuracy": 0.23875, + "f1": 0.227204, + "f1_weighted": 0.241529 + }, + { + "accuracy": 0.24625, + "f1": 0.240628, + "f1_weighted": 0.244108 + }, + { + "accuracy": 0.205, + "f1": 0.202345, + "f1_weighted": 0.20823 + }, + { + "accuracy": 0.2525, + "f1": 0.244733, + "f1_weighted": 0.244149 + } + ], + "main_score": 0.233454, + "hf_subset": "sun", + "languages": [ + "sun-Latn" + ] + } + ] + }, + "evaluation_time": 85.18696355819702, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/NusaParagraphTopicClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/NusaParagraphTopicClassification.json new file mode 100644 index 0000000000..37df01ee2c --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/NusaParagraphTopicClassification.json @@ -0,0 +1,631 @@ +{ + "dataset_revision": "abb43f8d5b9510b8724b48283aca26c4733eac5d", + "task_name": "NusaParagraphTopicClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.3726, + "f1": 0.3641, + "f1_weighted": 0.370971, + "scores_per_experiment": [ + { + "accuracy": 0.394, + "f1": 0.394469, + "f1_weighted": 0.398271 + }, + { + "accuracy": 0.368, + "f1": 0.365467, + "f1_weighted": 0.364332 + }, + { + "accuracy": 0.384, + "f1": 0.362673, + "f1_weighted": 0.381927 + }, + { + "accuracy": 0.38, + "f1": 0.37717, + "f1_weighted": 0.387563 + }, + { + "accuracy": 0.318, + "f1": 0.30711, + "f1_weighted": 0.312574 + }, + { + "accuracy": 0.354, + "f1": 0.358364, + "f1_weighted": 0.356419 + }, + { + "accuracy": 0.308, + "f1": 0.292381, + "f1_weighted": 0.303655 + }, + { + "accuracy": 0.406, + "f1": 0.398102, + "f1_weighted": 0.407241 + }, + { + "accuracy": 0.424, + "f1": 0.404875, + "f1_weighted": 0.423797 + }, + { + "accuracy": 0.39, + "f1": 0.380386, + "f1_weighted": 0.373932 + } + ], + "main_score": 0.3641, + "hf_subset": "btk", + "languages": [ + "bbc-Latn" + ] + }, + { + "accuracy": 0.482875, + "f1": 0.461843, + "f1_weighted": 0.483582, + "scores_per_experiment": [ + { + "accuracy": 0.45375, + "f1": 0.441061, + "f1_weighted": 0.461703 + }, + { + "accuracy": 0.50625, + "f1": 0.484439, + "f1_weighted": 0.505881 + }, + { + "accuracy": 0.48125, + "f1": 0.475929, + "f1_weighted": 0.496984 + }, + { + "accuracy": 0.48125, + "f1": 0.456644, + "f1_weighted": 0.476658 + }, + { + "accuracy": 0.47125, + "f1": 0.443444, + "f1_weighted": 0.468795 + }, + { + "accuracy": 0.53, + "f1": 0.509936, + "f1_weighted": 0.533705 + }, + { + "accuracy": 0.49625, + "f1": 0.466572, + "f1_weighted": 0.495601 + }, + { + "accuracy": 0.4825, + "f1": 0.459063, + "f1_weighted": 0.480794 + }, + { + "accuracy": 0.46875, + "f1": 0.436277, + "f1_weighted": 0.455972 + }, + { + "accuracy": 0.4575, + "f1": 0.445067, + "f1_weighted": 0.459731 + } + ], + "main_score": 0.461843, + "hf_subset": "bew", + "languages": [ + "bew-Latn" + ] + }, + { + "accuracy": 0.472, + "f1": 0.326461, + "f1_weighted": 0.426568, + "scores_per_experiment": [ + { + "accuracy": 0.466667, + "f1": 0.32538, + "f1_weighted": 0.42736 + }, + { + "accuracy": 0.426667, + "f1": 0.297828, + "f1_weighted": 0.385056 + }, + { + "accuracy": 0.516667, + "f1": 0.353951, + "f1_weighted": 0.469662 + }, + { + "accuracy": 0.47, + "f1": 0.32703, + "f1_weighted": 0.430753 + }, + { + "accuracy": 0.45, + "f1": 0.318563, + "f1_weighted": 0.404197 + }, + { + "accuracy": 0.47, + "f1": 0.321876, + "f1_weighted": 0.421691 + }, + { + "accuracy": 0.453333, + "f1": 0.311462, + "f1_weighted": 0.415598 + }, + { + "accuracy": 0.533333, + "f1": 0.370486, + "f1_weighted": 0.480948 + }, + { + "accuracy": 0.443333, + "f1": 0.304346, + "f1_weighted": 0.395525 + }, + { + "accuracy": 0.49, + "f1": 0.333687, + "f1_weighted": 0.434894 + } + ], + "main_score": 0.326461, + "hf_subset": "bug", + "languages": [ + "bug-Latn" + ] + }, + { + "accuracy": 0.392625, + "f1": 0.376775, + "f1_weighted": 0.393685, + "scores_per_experiment": [ + { + "accuracy": 0.3775, + "f1": 0.373717, + "f1_weighted": 0.391369 + }, + { + "accuracy": 0.39125, + "f1": 0.376828, + "f1_weighted": 0.391429 + }, + { + "accuracy": 0.3875, + "f1": 0.373651, + "f1_weighted": 0.389853 + }, + { + "accuracy": 0.37125, + "f1": 0.345248, + "f1_weighted": 0.358506 + }, + { + "accuracy": 0.39875, + "f1": 0.364754, + "f1_weighted": 0.379235 + }, + { + "accuracy": 0.385, + "f1": 0.378555, + "f1_weighted": 0.390314 + }, + { + "accuracy": 0.39875, + "f1": 0.377817, + "f1_weighted": 0.396161 + }, + { + "accuracy": 0.4075, + "f1": 0.391529, + "f1_weighted": 0.413459 + }, + { + "accuracy": 0.41625, + "f1": 0.403469, + "f1_weighted": 0.42224 + }, + { + "accuracy": 0.3925, + "f1": 0.382176, + "f1_weighted": 0.404288 + } + ], + "main_score": 0.376775, + "hf_subset": "jav", + "languages": [ + "jav-Latn" + ] + }, + { + "accuracy": 0.405143, + "f1": 0.378918, + "f1_weighted": 0.403973, + "scores_per_experiment": [ + { + "accuracy": 0.365714, + "f1": 0.320681, + "f1_weighted": 0.365482 + }, + { + "accuracy": 0.44, + "f1": 0.390941, + "f1_weighted": 0.423756 + }, + { + "accuracy": 0.445714, + "f1": 0.399739, + "f1_weighted": 0.439669 + }, + { + "accuracy": 0.347143, + "f1": 0.351328, + "f1_weighted": 0.351255 + }, + { + "accuracy": 0.394286, + "f1": 0.374459, + "f1_weighted": 0.399104 + }, + { + "accuracy": 0.451429, + "f1": 0.422314, + "f1_weighted": 0.450507 + }, + { + "accuracy": 0.368571, + "f1": 0.365071, + "f1_weighted": 0.364834 + }, + { + "accuracy": 0.39, + "f1": 0.375775, + "f1_weighted": 0.39223 + }, + { + "accuracy": 0.431429, + "f1": 0.392308, + "f1_weighted": 0.431042 + }, + { + "accuracy": 0.417143, + "f1": 0.396563, + "f1_weighted": 0.421856 + } + ], + "main_score": 0.378918, + "hf_subset": "mad", + "languages": [ + "mad-Latn" + ] + }, + { + "accuracy": 0.473857, + "f1": 0.449642, + "f1_weighted": 0.472659, + "scores_per_experiment": [ + { + "accuracy": 0.478571, + "f1": 0.461322, + "f1_weighted": 0.474133 + }, + { + "accuracy": 0.442857, + "f1": 0.424357, + "f1_weighted": 0.43563 + }, + { + "accuracy": 0.471429, + "f1": 0.436144, + "f1_weighted": 0.474878 + }, + { + "accuracy": 0.497143, + "f1": 0.464298, + "f1_weighted": 0.494328 + }, + { + "accuracy": 0.457143, + "f1": 0.45266, + "f1_weighted": 0.465856 + }, + { + "accuracy": 0.405714, + "f1": 0.395488, + "f1_weighted": 0.395433 + }, + { + "accuracy": 0.477143, + "f1": 0.448201, + "f1_weighted": 0.471855 + }, + { + "accuracy": 0.5, + "f1": 0.474122, + "f1_weighted": 0.506874 + }, + { + "accuracy": 0.472857, + "f1": 0.438431, + "f1_weighted": 0.471163 + }, + { + "accuracy": 0.535714, + "f1": 0.501398, + "f1_weighted": 0.536443 + } + ], + "main_score": 0.449642, + "hf_subset": "mak", + "languages": [ + "mak-Latn" + ] + }, + { + "accuracy": 0.45025, + "f1": 0.443037, + "f1_weighted": 0.446026, + "scores_per_experiment": [ + { + "accuracy": 0.45125, + "f1": 0.446104, + "f1_weighted": 0.446283 + }, + { + "accuracy": 0.5, + "f1": 0.482878, + "f1_weighted": 0.493576 + }, + { + "accuracy": 0.43875, + "f1": 0.438535, + "f1_weighted": 0.436229 + }, + { + "accuracy": 0.435, + "f1": 0.425172, + "f1_weighted": 0.434022 + }, + { + "accuracy": 0.44625, + "f1": 0.450469, + "f1_weighted": 0.456117 + }, + { + "accuracy": 0.41125, + "f1": 0.398406, + "f1_weighted": 0.403269 + }, + { + "accuracy": 0.47375, + "f1": 0.471172, + "f1_weighted": 0.464262 + }, + { + "accuracy": 0.48625, + "f1": 0.471918, + "f1_weighted": 0.477012 + }, + { + "accuracy": 0.44875, + "f1": 0.447389, + "f1_weighted": 0.450203 + }, + { + "accuracy": 0.41125, + "f1": 0.398323, + "f1_weighted": 0.399285 + } + ], + "main_score": 0.443037, + "hf_subset": "min", + "languages": [ + "min-Latn" + ] + }, + { + "accuracy": 0.57775, + "f1": 0.5779, + "f1_weighted": 0.584514, + "scores_per_experiment": [ + { + "accuracy": 0.5725, + "f1": 0.570165, + "f1_weighted": 0.577569 + }, + { + "accuracy": 0.58, + "f1": 0.578555, + "f1_weighted": 0.581965 + }, + { + "accuracy": 0.575, + "f1": 0.578281, + "f1_weighted": 0.581946 + }, + { + "accuracy": 0.5675, + "f1": 0.568697, + "f1_weighted": 0.57654 + }, + { + "accuracy": 0.585, + "f1": 0.587811, + "f1_weighted": 0.59398 + }, + { + "accuracy": 0.57, + "f1": 0.567286, + "f1_weighted": 0.575206 + }, + { + "accuracy": 0.565, + "f1": 0.563298, + "f1_weighted": 0.57099 + }, + { + "accuracy": 0.5575, + "f1": 0.55874, + "f1_weighted": 0.565078 + }, + { + "accuracy": 0.62, + "f1": 0.62233, + "f1_weighted": 0.630527 + }, + { + "accuracy": 0.585, + "f1": 0.583834, + "f1_weighted": 0.59134 + } + ], + "main_score": 0.5779, + "hf_subset": "mui", + "languages": [ + "mui-Latn" + ] + }, + { + "accuracy": 0.625714, + "f1": 0.55552, + "f1_weighted": 0.611839, + "scores_per_experiment": [ + { + "accuracy": 0.605714, + "f1": 0.535136, + "f1_weighted": 0.591825 + }, + { + "accuracy": 0.622857, + "f1": 0.564111, + "f1_weighted": 0.616233 + }, + { + "accuracy": 0.625714, + "f1": 0.548754, + "f1_weighted": 0.610115 + }, + { + "accuracy": 0.637143, + "f1": 0.564735, + "f1_weighted": 0.625064 + }, + { + "accuracy": 0.645714, + "f1": 0.571095, + "f1_weighted": 0.631596 + }, + { + "accuracy": 0.631429, + "f1": 0.558414, + "f1_weighted": 0.61619 + }, + { + "accuracy": 0.625714, + "f1": 0.558923, + "f1_weighted": 0.613192 + }, + { + "accuracy": 0.625714, + "f1": 0.553249, + "f1_weighted": 0.610233 + }, + { + "accuracy": 0.594286, + "f1": 0.52668, + "f1_weighted": 0.571118 + }, + { + "accuracy": 0.642857, + "f1": 0.574104, + "f1_weighted": 0.632829 + } + ], + "main_score": 0.55552, + "hf_subset": "rej", + "languages": [ + "rej-Latn" + ] + }, + { + "accuracy": 0.495778, + "f1": 0.473801, + "f1_weighted": 0.491071, + "scores_per_experiment": [ + { + "accuracy": 0.48, + "f1": 0.457285, + "f1_weighted": 0.473551 + }, + { + "accuracy": 0.533333, + "f1": 0.517682, + "f1_weighted": 0.531203 + }, + { + "accuracy": 0.49, + "f1": 0.471436, + "f1_weighted": 0.492853 + }, + { + "accuracy": 0.498889, + "f1": 0.476994, + "f1_weighted": 0.488351 + }, + { + "accuracy": 0.488889, + "f1": 0.45963, + "f1_weighted": 0.477088 + }, + { + "accuracy": 0.51, + "f1": 0.491568, + "f1_weighted": 0.510186 + }, + { + "accuracy": 0.46, + "f1": 0.443524, + "f1_weighted": 0.463129 + }, + { + "accuracy": 0.483333, + "f1": 0.459872, + "f1_weighted": 0.475343 + }, + { + "accuracy": 0.5, + "f1": 0.477769, + "f1_weighted": 0.495448 + }, + { + "accuracy": 0.513333, + "f1": 0.482249, + "f1_weighted": 0.503559 + } + ], + "main_score": 0.473801, + "hf_subset": "sun", + "languages": [ + "sun-Latn" + ] + } + ] + }, + "evaluation_time": 93.8737404346466, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/NusaX-senti.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/NusaX-senti.json new file mode 100644 index 0000000000..95c6149469 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/NusaX-senti.json @@ -0,0 +1,755 @@ +{ + "dataset_revision": "a450ba4b1b6d2216c3674d3e576b2e85ce729add", + "task_name": "NusaX-senti", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.532, + "f1": 0.515436, + "f1_weighted": 0.525895, + "scores_per_experiment": [ + { + "accuracy": 0.5, + "f1": 0.496885, + "f1_weighted": 0.497459 + }, + { + "accuracy": 0.5225, + "f1": 0.504403, + "f1_weighted": 0.520924 + }, + { + "accuracy": 0.5625, + "f1": 0.561615, + "f1_weighted": 0.56283 + }, + { + "accuracy": 0.5725, + "f1": 0.551678, + "f1_weighted": 0.560652 + }, + { + "accuracy": 0.54, + "f1": 0.51133, + "f1_weighted": 0.529526 + }, + { + "accuracy": 0.5075, + "f1": 0.480065, + "f1_weighted": 0.499413 + }, + { + "accuracy": 0.5275, + "f1": 0.49037, + "f1_weighted": 0.513808 + }, + { + "accuracy": 0.57, + "f1": 0.563799, + "f1_weighted": 0.568786 + }, + { + "accuracy": 0.5, + "f1": 0.486642, + "f1_weighted": 0.485546 + }, + { + "accuracy": 0.5175, + "f1": 0.507575, + "f1_weighted": 0.520007 + } + ], + "main_score": 0.532, + "hf_subset": "ace", + "languages": [ + "ace-Latn" + ] + }, + { + "accuracy": 0.54725, + "f1": 0.535372, + "f1_weighted": 0.541429, + "scores_per_experiment": [ + { + "accuracy": 0.5675, + "f1": 0.558604, + "f1_weighted": 0.562686 + }, + { + "accuracy": 0.555, + "f1": 0.533978, + "f1_weighted": 0.550453 + }, + { + "accuracy": 0.54, + "f1": 0.531908, + "f1_weighted": 0.534177 + }, + { + "accuracy": 0.5475, + "f1": 0.511364, + "f1_weighted": 0.52602 + }, + { + "accuracy": 0.55, + "f1": 0.544704, + "f1_weighted": 0.544453 + }, + { + "accuracy": 0.5275, + "f1": 0.520172, + "f1_weighted": 0.532832 + }, + { + "accuracy": 0.4725, + "f1": 0.452544, + "f1_weighted": 0.465918 + }, + { + "accuracy": 0.595, + "f1": 0.594754, + "f1_weighted": 0.59745 + }, + { + "accuracy": 0.555, + "f1": 0.548264, + "f1_weighted": 0.541183 + }, + { + "accuracy": 0.5625, + "f1": 0.557432, + "f1_weighted": 0.55912 + } + ], + "main_score": 0.54725, + "hf_subset": "ban", + "languages": [ + "ban-Latn" + ] + }, + { + "accuracy": 0.5275, + "f1": 0.507594, + "f1_weighted": 0.511123, + "scores_per_experiment": [ + { + "accuracy": 0.485, + "f1": 0.459846, + "f1_weighted": 0.466799 + }, + { + "accuracy": 0.5175, + "f1": 0.487097, + "f1_weighted": 0.492815 + }, + { + "accuracy": 0.57, + "f1": 0.551579, + "f1_weighted": 0.548136 + }, + { + "accuracy": 0.485, + "f1": 0.448055, + "f1_weighted": 0.448232 + }, + { + "accuracy": 0.5075, + "f1": 0.478584, + "f1_weighted": 0.477221 + }, + { + "accuracy": 0.5175, + "f1": 0.510101, + "f1_weighted": 0.519811 + }, + { + "accuracy": 0.57, + "f1": 0.554458, + "f1_weighted": 0.564337 + }, + { + "accuracy": 0.5575, + "f1": 0.555089, + "f1_weighted": 0.556255 + }, + { + "accuracy": 0.535, + "f1": 0.510586, + "f1_weighted": 0.512238 + }, + { + "accuracy": 0.53, + "f1": 0.520542, + "f1_weighted": 0.525381 + } + ], + "main_score": 0.5275, + "hf_subset": "bjn", + "languages": [ + "bjn-Latn" + ] + }, + { + "accuracy": 0.55725, + "f1": 0.539734, + "f1_weighted": 0.542673, + "scores_per_experiment": [ + { + "accuracy": 0.5725, + "f1": 0.555519, + "f1_weighted": 0.561282 + }, + { + "accuracy": 0.5375, + "f1": 0.512059, + "f1_weighted": 0.519372 + }, + { + "accuracy": 0.585, + "f1": 0.567262, + "f1_weighted": 0.563168 + }, + { + "accuracy": 0.5925, + "f1": 0.575158, + "f1_weighted": 0.582358 + }, + { + "accuracy": 0.5475, + "f1": 0.52491, + "f1_weighted": 0.523316 + }, + { + "accuracy": 0.515, + "f1": 0.50063, + "f1_weighted": 0.504919 + }, + { + "accuracy": 0.5475, + "f1": 0.522849, + "f1_weighted": 0.529802 + }, + { + "accuracy": 0.5775, + "f1": 0.574415, + "f1_weighted": 0.574395 + }, + { + "accuracy": 0.5425, + "f1": 0.52235, + "f1_weighted": 0.524443 + }, + { + "accuracy": 0.555, + "f1": 0.542192, + "f1_weighted": 0.543675 + } + ], + "main_score": 0.55725, + "hf_subset": "bug", + "languages": [ + "bug-Latn" + ] + }, + { + "accuracy": 0.71875, + "f1": 0.70232, + "f1_weighted": 0.713532, + "scores_per_experiment": [ + { + "accuracy": 0.755, + "f1": 0.743739, + "f1_weighted": 0.750926 + }, + { + "accuracy": 0.6875, + "f1": 0.660775, + "f1_weighted": 0.678557 + }, + { + "accuracy": 0.76, + "f1": 0.744127, + "f1_weighted": 0.757263 + }, + { + "accuracy": 0.73, + "f1": 0.711147, + "f1_weighted": 0.721207 + }, + { + "accuracy": 0.625, + "f1": 0.606338, + "f1_weighted": 0.60564 + }, + { + "accuracy": 0.6725, + "f1": 0.654737, + "f1_weighted": 0.675724 + }, + { + "accuracy": 0.73, + "f1": 0.711241, + "f1_weighted": 0.724335 + }, + { + "accuracy": 0.7325, + "f1": 0.723403, + "f1_weighted": 0.730079 + }, + { + "accuracy": 0.7275, + "f1": 0.706383, + "f1_weighted": 0.721096 + }, + { + "accuracy": 0.7675, + "f1": 0.761312, + "f1_weighted": 0.770489 + } + ], + "main_score": 0.71875, + "hf_subset": "eng", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.5445, + "f1": 0.527041, + "f1_weighted": 0.532961, + "scores_per_experiment": [ + { + "accuracy": 0.5175, + "f1": 0.51265, + "f1_weighted": 0.518893 + }, + { + "accuracy": 0.5325, + "f1": 0.496283, + "f1_weighted": 0.49789 + }, + { + "accuracy": 0.565, + "f1": 0.553821, + "f1_weighted": 0.562002 + }, + { + "accuracy": 0.5575, + "f1": 0.534865, + "f1_weighted": 0.540388 + }, + { + "accuracy": 0.5725, + "f1": 0.557157, + "f1_weighted": 0.562463 + }, + { + "accuracy": 0.5275, + "f1": 0.498516, + "f1_weighted": 0.506178 + }, + { + "accuracy": 0.5375, + "f1": 0.504982, + "f1_weighted": 0.525638 + }, + { + "accuracy": 0.5975, + "f1": 0.586205, + "f1_weighted": 0.595422 + }, + { + "accuracy": 0.48, + "f1": 0.473104, + "f1_weighted": 0.467973 + }, + { + "accuracy": 0.5575, + "f1": 0.552826, + "f1_weighted": 0.552764 + } + ], + "main_score": 0.5445, + "hf_subset": "ind", + "languages": [ + "ind-Latn" + ] + }, + { + "accuracy": 0.53625, + "f1": 0.519293, + "f1_weighted": 0.524599, + "scores_per_experiment": [ + { + "accuracy": 0.535, + "f1": 0.522285, + "f1_weighted": 0.523154 + }, + { + "accuracy": 0.4975, + "f1": 0.477457, + "f1_weighted": 0.487737 + }, + { + "accuracy": 0.5825, + "f1": 0.570447, + "f1_weighted": 0.575696 + }, + { + "accuracy": 0.555, + "f1": 0.529155, + "f1_weighted": 0.529699 + }, + { + "accuracy": 0.5775, + "f1": 0.570603, + "f1_weighted": 0.570243 + }, + { + "accuracy": 0.5475, + "f1": 0.53607, + "f1_weighted": 0.546102 + }, + { + "accuracy": 0.5, + "f1": 0.477558, + "f1_weighted": 0.49705 + }, + { + "accuracy": 0.5425, + "f1": 0.53691, + "f1_weighted": 0.537887 + }, + { + "accuracy": 0.5175, + "f1": 0.481831, + "f1_weighted": 0.488505 + }, + { + "accuracy": 0.5075, + "f1": 0.490615, + "f1_weighted": 0.489921 + } + ], + "main_score": 0.53625, + "hf_subset": "jav", + "languages": [ + "jav-Latn" + ] + }, + { + "accuracy": 0.52, + "f1": 0.5062, + "f1_weighted": 0.510949, + "scores_per_experiment": [ + { + "accuracy": 0.5275, + "f1": 0.527741, + "f1_weighted": 0.526589 + }, + { + "accuracy": 0.4875, + "f1": 0.471522, + "f1_weighted": 0.483078 + }, + { + "accuracy": 0.545, + "f1": 0.537539, + "f1_weighted": 0.533112 + }, + { + "accuracy": 0.5025, + "f1": 0.462771, + "f1_weighted": 0.462534 + }, + { + "accuracy": 0.5175, + "f1": 0.506108, + "f1_weighted": 0.514926 + }, + { + "accuracy": 0.525, + "f1": 0.502496, + "f1_weighted": 0.5222 + }, + { + "accuracy": 0.4325, + "f1": 0.414846, + "f1_weighted": 0.417246 + }, + { + "accuracy": 0.575, + "f1": 0.5673, + "f1_weighted": 0.569722 + }, + { + "accuracy": 0.5725, + "f1": 0.562441, + "f1_weighted": 0.560465 + }, + { + "accuracy": 0.515, + "f1": 0.509236, + "f1_weighted": 0.519621 + } + ], + "main_score": 0.52, + "hf_subset": "mad", + "languages": [ + "mad-Latn" + ] + }, + { + "accuracy": 0.52025, + "f1": 0.49376, + "f1_weighted": 0.50163, + "scores_per_experiment": [ + { + "accuracy": 0.51, + "f1": 0.497419, + "f1_weighted": 0.504024 + }, + { + "accuracy": 0.4825, + "f1": 0.439702, + "f1_weighted": 0.447729 + }, + { + "accuracy": 0.555, + "f1": 0.545186, + "f1_weighted": 0.553697 + }, + { + "accuracy": 0.5225, + "f1": 0.487011, + "f1_weighted": 0.495642 + }, + { + "accuracy": 0.5325, + "f1": 0.497815, + "f1_weighted": 0.496466 + }, + { + "accuracy": 0.495, + "f1": 0.47438, + "f1_weighted": 0.486143 + }, + { + "accuracy": 0.4975, + "f1": 0.465497, + "f1_weighted": 0.484789 + }, + { + "accuracy": 0.58, + "f1": 0.563566, + "f1_weighted": 0.566928 + }, + { + "accuracy": 0.48, + "f1": 0.432302, + "f1_weighted": 0.435483 + }, + { + "accuracy": 0.5475, + "f1": 0.534723, + "f1_weighted": 0.545403 + } + ], + "main_score": 0.52025, + "hf_subset": "min", + "languages": [ + "min-Latn" + ] + }, + { + "accuracy": 0.50725, + "f1": 0.486123, + "f1_weighted": 0.494119, + "scores_per_experiment": [ + { + "accuracy": 0.54, + "f1": 0.526473, + "f1_weighted": 0.532843 + }, + { + "accuracy": 0.4725, + "f1": 0.450897, + "f1_weighted": 0.46129 + }, + { + "accuracy": 0.5, + "f1": 0.493897, + "f1_weighted": 0.501405 + }, + { + "accuracy": 0.55, + "f1": 0.520697, + "f1_weighted": 0.536034 + }, + { + "accuracy": 0.5025, + "f1": 0.488576, + "f1_weighted": 0.485916 + }, + { + "accuracy": 0.43, + "f1": 0.423868, + "f1_weighted": 0.427313 + }, + { + "accuracy": 0.4825, + "f1": 0.447769, + "f1_weighted": 0.458298 + }, + { + "accuracy": 0.5275, + "f1": 0.51524, + "f1_weighted": 0.527523 + }, + { + "accuracy": 0.535, + "f1": 0.494818, + "f1_weighted": 0.506815 + }, + { + "accuracy": 0.5325, + "f1": 0.498997, + "f1_weighted": 0.503749 + } + ], + "main_score": 0.50725, + "hf_subset": "nij", + "languages": [ + "nij-Latn" + ] + }, + { + "accuracy": 0.55275, + "f1": 0.533242, + "f1_weighted": 0.53588, + "scores_per_experiment": [ + { + "accuracy": 0.56, + "f1": 0.543889, + "f1_weighted": 0.55118 + }, + { + "accuracy": 0.53, + "f1": 0.501541, + "f1_weighted": 0.500572 + }, + { + "accuracy": 0.605, + "f1": 0.598903, + "f1_weighted": 0.599479 + }, + { + "accuracy": 0.495, + "f1": 0.465436, + "f1_weighted": 0.467835 + }, + { + "accuracy": 0.5875, + "f1": 0.574841, + "f1_weighted": 0.5732 + }, + { + "accuracy": 0.5325, + "f1": 0.522278, + "f1_weighted": 0.529718 + }, + { + "accuracy": 0.5075, + "f1": 0.466035, + "f1_weighted": 0.483296 + }, + { + "accuracy": 0.6225, + "f1": 0.610049, + "f1_weighted": 0.616401 + }, + { + "accuracy": 0.54, + "f1": 0.508498, + "f1_weighted": 0.499881 + }, + { + "accuracy": 0.5475, + "f1": 0.540949, + "f1_weighted": 0.537241 + } + ], + "main_score": 0.55275, + "hf_subset": "sun", + "languages": [ + "sun-Latn" + ] + }, + { + "accuracy": 0.533, + "f1": 0.518861, + "f1_weighted": 0.522745, + "scores_per_experiment": [ + { + "accuracy": 0.555, + "f1": 0.547104, + "f1_weighted": 0.558629 + }, + { + "accuracy": 0.4825, + "f1": 0.470597, + "f1_weighted": 0.475317 + }, + { + "accuracy": 0.5875, + "f1": 0.581525, + "f1_weighted": 0.584901 + }, + { + "accuracy": 0.5325, + "f1": 0.511497, + "f1_weighted": 0.516722 + }, + { + "accuracy": 0.515, + "f1": 0.481893, + "f1_weighted": 0.476322 + }, + { + "accuracy": 0.5025, + "f1": 0.493447, + "f1_weighted": 0.506172 + }, + { + "accuracy": 0.53, + "f1": 0.502411, + "f1_weighted": 0.506073 + }, + { + "accuracy": 0.59, + "f1": 0.586977, + "f1_weighted": 0.583837 + }, + { + "accuracy": 0.5275, + "f1": 0.509292, + "f1_weighted": 0.512621 + }, + { + "accuracy": 0.5075, + "f1": 0.50387, + "f1_weighted": 0.506855 + } + ], + "main_score": 0.533, + "hf_subset": "bbc", + "languages": [ + "bbc-Latn" + ] + } + ] + }, + "evaluation_time": 17.95746874809265, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115DataRetentionLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115DataRetentionLegalBenchClassification.json new file mode 100644 index 0000000000..a399254a8e --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115DataRetentionLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "OPP115DataRetentionLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.625, + "f1": 0.606664, + "f1_weighted": 0.606664, + "ap": 0.59, + "ap_weighted": 0.59, + "scores_per_experiment": [ + { + "accuracy": 0.625, + "f1": 0.606664, + "f1_weighted": 0.606664, + "ap": 0.59, + "ap_weighted": 0.59 + }, + { + "accuracy": 0.625, + "f1": 0.606664, + "f1_weighted": 0.606664, + "ap": 0.59, + "ap_weighted": 0.59 + }, + { + "accuracy": 0.625, + "f1": 0.606664, + "f1_weighted": 0.606664, + "ap": 0.59, + "ap_weighted": 0.59 + }, + { + "accuracy": 0.625, + "f1": 0.606664, + "f1_weighted": 0.606664, + "ap": 0.59, + "ap_weighted": 0.59 + }, + { + "accuracy": 0.625, + "f1": 0.606664, + "f1_weighted": 0.606664, + "ap": 0.59, + "ap_weighted": 0.59 + }, + { + "accuracy": 0.625, + "f1": 0.606664, + "f1_weighted": 0.606664, + "ap": 0.59, + "ap_weighted": 0.59 + }, + { + "accuracy": 0.625, + "f1": 0.606664, + "f1_weighted": 0.606664, + "ap": 0.59, + "ap_weighted": 0.59 + }, + { + "accuracy": 0.625, + "f1": 0.606664, + "f1_weighted": 0.606664, + "ap": 0.59, + "ap_weighted": 0.59 + }, + { + "accuracy": 0.625, + "f1": 0.606664, + "f1_weighted": 0.606664, + "ap": 0.59, + "ap_weighted": 0.59 + }, + { + "accuracy": 0.625, + "f1": 0.606664, + "f1_weighted": 0.606664, + "ap": 0.59, + "ap_weighted": 0.59 + } + ], + "main_score": 0.625, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.6780602931976318, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115DataSecurityLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115DataSecurityLegalBenchClassification.json new file mode 100644 index 0000000000..04a70b88e7 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115DataSecurityLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "OPP115DataSecurityLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.765367, + "f1": 0.758869, + "f1_weighted": 0.75875, + "ap": 0.739824, + "ap_weighted": 0.739824, + "scores_per_experiment": [ + { + "accuracy": 0.765367, + "f1": 0.758869, + "f1_weighted": 0.75875, + "ap": 0.739824, + "ap_weighted": 0.739824 + }, + { + "accuracy": 0.765367, + "f1": 0.758869, + "f1_weighted": 0.75875, + "ap": 0.739824, + "ap_weighted": 0.739824 + }, + { + "accuracy": 0.765367, + "f1": 0.758869, + "f1_weighted": 0.75875, + "ap": 0.739824, + "ap_weighted": 0.739824 + }, + { + "accuracy": 0.765367, + "f1": 0.758869, + "f1_weighted": 0.75875, + "ap": 0.739824, + "ap_weighted": 0.739824 + }, + { + "accuracy": 0.765367, + "f1": 0.758869, + "f1_weighted": 0.75875, + "ap": 0.739824, + "ap_weighted": 0.739824 + }, + { + "accuracy": 0.765367, + "f1": 0.758869, + "f1_weighted": 0.75875, + "ap": 0.739824, + "ap_weighted": 0.739824 + }, + { + "accuracy": 0.765367, + "f1": 0.758869, + "f1_weighted": 0.75875, + "ap": 0.739824, + "ap_weighted": 0.739824 + }, + { + "accuracy": 0.765367, + "f1": 0.758869, + "f1_weighted": 0.75875, + "ap": 0.739824, + "ap_weighted": 0.739824 + }, + { + "accuracy": 0.765367, + "f1": 0.758869, + "f1_weighted": 0.75875, + "ap": 0.739824, + "ap_weighted": 0.739824 + }, + { + "accuracy": 0.765367, + "f1": 0.758869, + "f1_weighted": 0.75875, + "ap": 0.739824, + "ap_weighted": 0.739824 + } + ], + "main_score": 0.765367, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.2730674743652344, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115DoNotTrackLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115DoNotTrackLegalBenchClassification.json new file mode 100644 index 0000000000..316648ed94 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115DoNotTrackLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "OPP115DoNotTrackLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.954545, + "f1": 0.954542, + "f1_weighted": 0.954542, + "ap": 0.93771, + "ap_weighted": 0.93771, + "scores_per_experiment": [ + { + "accuracy": 0.954545, + "f1": 0.954542, + "f1_weighted": 0.954542, + "ap": 0.93771, + "ap_weighted": 0.93771 + }, + { + "accuracy": 0.954545, + "f1": 0.954542, + "f1_weighted": 0.954542, + "ap": 0.93771, + "ap_weighted": 0.93771 + }, + { + "accuracy": 0.954545, + "f1": 0.954542, + "f1_weighted": 0.954542, + "ap": 0.93771, + "ap_weighted": 0.93771 + }, + { + "accuracy": 0.954545, + "f1": 0.954542, + "f1_weighted": 0.954542, + "ap": 0.93771, + "ap_weighted": 0.93771 + }, + { + "accuracy": 0.954545, + "f1": 0.954542, + "f1_weighted": 0.954542, + "ap": 0.93771, + "ap_weighted": 0.93771 + }, + { + "accuracy": 0.954545, + "f1": 0.954542, + "f1_weighted": 0.954542, + "ap": 0.93771, + "ap_weighted": 0.93771 + }, + { + "accuracy": 0.954545, + "f1": 0.954542, + "f1_weighted": 0.954542, + "ap": 0.93771, + "ap_weighted": 0.93771 + }, + { + "accuracy": 0.954545, + "f1": 0.954542, + "f1_weighted": 0.954542, + "ap": 0.93771, + "ap_weighted": 0.93771 + }, + { + "accuracy": 0.954545, + "f1": 0.954542, + "f1_weighted": 0.954542, + "ap": 0.93771, + "ap_weighted": 0.93771 + }, + { + "accuracy": 0.954545, + "f1": 0.954542, + "f1_weighted": 0.954542, + "ap": 0.93771, + "ap_weighted": 0.93771 + } + ], + "main_score": 0.954545, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.7851336002349854, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115FirstPartyCollectionUseLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115FirstPartyCollectionUseLegalBenchClassification.json new file mode 100644 index 0000000000..03f7443911 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115FirstPartyCollectionUseLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "OPP115FirstPartyCollectionUseLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.632311, + "f1": 0.629939, + "f1_weighted": 0.629882, + "ap": 0.588092, + "ap_weighted": 0.588092, + "scores_per_experiment": [ + { + "accuracy": 0.632311, + "f1": 0.629939, + "f1_weighted": 0.629882, + "ap": 0.588092, + "ap_weighted": 0.588092 + }, + { + "accuracy": 0.632311, + "f1": 0.629939, + "f1_weighted": 0.629882, + "ap": 0.588092, + "ap_weighted": 0.588092 + }, + { + "accuracy": 0.632311, + "f1": 0.629939, + "f1_weighted": 0.629882, + "ap": 0.588092, + "ap_weighted": 0.588092 + }, + { + "accuracy": 0.632311, + "f1": 0.629939, + "f1_weighted": 0.629882, + "ap": 0.588092, + "ap_weighted": 0.588092 + }, + { + "accuracy": 0.632311, + "f1": 0.629939, + "f1_weighted": 0.629882, + "ap": 0.588092, + "ap_weighted": 0.588092 + }, + { + "accuracy": 0.632311, + "f1": 0.629939, + "f1_weighted": 0.629882, + "ap": 0.588092, + "ap_weighted": 0.588092 + }, + { + "accuracy": 0.632311, + "f1": 0.629939, + "f1_weighted": 0.629882, + "ap": 0.588092, + "ap_weighted": 0.588092 + }, + { + "accuracy": 0.632311, + "f1": 0.629939, + "f1_weighted": 0.629882, + "ap": 0.588092, + "ap_weighted": 0.588092 + }, + { + "accuracy": 0.632311, + "f1": 0.629939, + "f1_weighted": 0.629882, + "ap": 0.588092, + "ap_weighted": 0.588092 + }, + { + "accuracy": 0.632311, + "f1": 0.629939, + "f1_weighted": 0.629882, + "ap": 0.588092, + "ap_weighted": 0.588092 + } + ], + "main_score": 0.632311, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.93768572807312, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115InternationalAndSpecificAudiencesLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115InternationalAndSpecificAudiencesLegalBenchClassification.json new file mode 100644 index 0000000000..913f30994c --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115InternationalAndSpecificAudiencesLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "OPP115InternationalAndSpecificAudiencesLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.794898, + "f1": 0.794296, + "f1_weighted": 0.794342, + "ap": 0.742627, + "ap_weighted": 0.742627, + "scores_per_experiment": [ + { + "accuracy": 0.794898, + "f1": 0.794296, + "f1_weighted": 0.794342, + "ap": 0.742627, + "ap_weighted": 0.742627 + }, + { + "accuracy": 0.794898, + "f1": 0.794296, + "f1_weighted": 0.794342, + "ap": 0.742627, + "ap_weighted": 0.742627 + }, + { + "accuracy": 0.794898, + "f1": 0.794296, + "f1_weighted": 0.794342, + "ap": 0.742627, + "ap_weighted": 0.742627 + }, + { + "accuracy": 0.794898, + "f1": 0.794296, + "f1_weighted": 0.794342, + "ap": 0.742627, + "ap_weighted": 0.742627 + }, + { + "accuracy": 0.794898, + "f1": 0.794296, + "f1_weighted": 0.794342, + "ap": 0.742627, + "ap_weighted": 0.742627 + }, + { + "accuracy": 0.794898, + "f1": 0.794296, + "f1_weighted": 0.794342, + "ap": 0.742627, + "ap_weighted": 0.742627 + }, + { + "accuracy": 0.794898, + "f1": 0.794296, + "f1_weighted": 0.794342, + "ap": 0.742627, + "ap_weighted": 0.742627 + }, + { + "accuracy": 0.794898, + "f1": 0.794296, + "f1_weighted": 0.794342, + "ap": 0.742627, + "ap_weighted": 0.742627 + }, + { + "accuracy": 0.794898, + "f1": 0.794296, + "f1_weighted": 0.794342, + "ap": 0.742627, + "ap_weighted": 0.742627 + }, + { + "accuracy": 0.794898, + "f1": 0.794296, + "f1_weighted": 0.794342, + "ap": 0.742627, + "ap_weighted": 0.742627 + } + ], + "main_score": 0.794898, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.1893739700317383, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115PolicyChangeLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115PolicyChangeLegalBenchClassification.json new file mode 100644 index 0000000000..221b5c1af8 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115PolicyChangeLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "OPP115PolicyChangeLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.87239, + "f1": 0.87239, + "f1_weighted": 0.87239, + "ap": 0.823713, + "ap_weighted": 0.823713, + "scores_per_experiment": [ + { + "accuracy": 0.87239, + "f1": 0.87239, + "f1_weighted": 0.87239, + "ap": 0.823713, + "ap_weighted": 0.823713 + }, + { + "accuracy": 0.87239, + "f1": 0.87239, + "f1_weighted": 0.87239, + "ap": 0.823713, + "ap_weighted": 0.823713 + }, + { + "accuracy": 0.87239, + "f1": 0.87239, + "f1_weighted": 0.87239, + "ap": 0.823713, + "ap_weighted": 0.823713 + }, + { + "accuracy": 0.87239, + "f1": 0.87239, + "f1_weighted": 0.87239, + "ap": 0.823713, + "ap_weighted": 0.823713 + }, + { + "accuracy": 0.87239, + "f1": 0.87239, + "f1_weighted": 0.87239, + "ap": 0.823713, + "ap_weighted": 0.823713 + }, + { + "accuracy": 0.87239, + "f1": 0.87239, + "f1_weighted": 0.87239, + "ap": 0.823713, + "ap_weighted": 0.823713 + }, + { + "accuracy": 0.87239, + "f1": 0.87239, + "f1_weighted": 0.87239, + "ap": 0.823713, + "ap_weighted": 0.823713 + }, + { + "accuracy": 0.87239, + "f1": 0.87239, + "f1_weighted": 0.87239, + "ap": 0.823713, + "ap_weighted": 0.823713 + }, + { + "accuracy": 0.87239, + "f1": 0.87239, + "f1_weighted": 0.87239, + "ap": 0.823713, + "ap_weighted": 0.823713 + }, + { + "accuracy": 0.87239, + "f1": 0.87239, + "f1_weighted": 0.87239, + "ap": 0.823713, + "ap_weighted": 0.823713 + } + ], + "main_score": 0.87239, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.1181046962738037, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115ThirdPartySharingCollectionLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115ThirdPartySharingCollectionLegalBenchClassification.json new file mode 100644 index 0000000000..1f55728816 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115ThirdPartySharingCollectionLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "OPP115ThirdPartySharingCollectionLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.654717, + "f1": 0.616993, + "f1_weighted": 0.61654, + "ap": 0.645547, + "ap_weighted": 0.645547, + "scores_per_experiment": [ + { + "accuracy": 0.654717, + "f1": 0.616993, + "f1_weighted": 0.61654, + "ap": 0.645547, + "ap_weighted": 0.645547 + }, + { + "accuracy": 0.654717, + "f1": 0.616993, + "f1_weighted": 0.61654, + "ap": 0.645547, + "ap_weighted": 0.645547 + }, + { + "accuracy": 0.654717, + "f1": 0.616993, + "f1_weighted": 0.61654, + "ap": 0.645547, + "ap_weighted": 0.645547 + }, + { + "accuracy": 0.654717, + "f1": 0.616993, + "f1_weighted": 0.61654, + "ap": 0.645547, + "ap_weighted": 0.645547 + }, + { + "accuracy": 0.654717, + "f1": 0.616993, + "f1_weighted": 0.61654, + "ap": 0.645547, + "ap_weighted": 0.645547 + }, + { + "accuracy": 0.654717, + "f1": 0.616993, + "f1_weighted": 0.61654, + "ap": 0.645547, + "ap_weighted": 0.645547 + }, + { + "accuracy": 0.654717, + "f1": 0.616993, + "f1_weighted": 0.61654, + "ap": 0.645547, + "ap_weighted": 0.645547 + }, + { + "accuracy": 0.654717, + "f1": 0.616993, + "f1_weighted": 0.61654, + "ap": 0.645547, + "ap_weighted": 0.645547 + }, + { + "accuracy": 0.654717, + "f1": 0.616993, + "f1_weighted": 0.61654, + "ap": 0.645547, + "ap_weighted": 0.645547 + }, + { + "accuracy": 0.654717, + "f1": 0.616993, + "f1_weighted": 0.61654, + "ap": 0.645547, + "ap_weighted": 0.645547 + } + ], + "main_score": 0.654717, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.5025980472564697, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115UserAccessEditAndDeletionLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115UserAccessEditAndDeletionLegalBenchClassification.json new file mode 100644 index 0000000000..008b0ba299 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115UserAccessEditAndDeletionLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "OPP115UserAccessEditAndDeletionLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.768398, + "f1": 0.765976, + "f1_weighted": 0.765873, + "ap": 0.727433, + "ap_weighted": 0.727433, + "scores_per_experiment": [ + { + "accuracy": 0.768398, + "f1": 0.765976, + "f1_weighted": 0.765873, + "ap": 0.727433, + "ap_weighted": 0.727433 + }, + { + "accuracy": 0.768398, + "f1": 0.765976, + "f1_weighted": 0.765873, + "ap": 0.727433, + "ap_weighted": 0.727433 + }, + { + "accuracy": 0.768398, + "f1": 0.765976, + "f1_weighted": 0.765873, + "ap": 0.727433, + "ap_weighted": 0.727433 + }, + { + "accuracy": 0.768398, + "f1": 0.765976, + "f1_weighted": 0.765873, + "ap": 0.727433, + "ap_weighted": 0.727433 + }, + { + "accuracy": 0.768398, + "f1": 0.765976, + "f1_weighted": 0.765873, + "ap": 0.727433, + "ap_weighted": 0.727433 + }, + { + "accuracy": 0.768398, + "f1": 0.765976, + "f1_weighted": 0.765873, + "ap": 0.727433, + "ap_weighted": 0.727433 + }, + { + "accuracy": 0.768398, + "f1": 0.765976, + "f1_weighted": 0.765873, + "ap": 0.727433, + "ap_weighted": 0.727433 + }, + { + "accuracy": 0.768398, + "f1": 0.765976, + "f1_weighted": 0.765873, + "ap": 0.727433, + "ap_weighted": 0.727433 + }, + { + "accuracy": 0.768398, + "f1": 0.765976, + "f1_weighted": 0.765873, + "ap": 0.727433, + "ap_weighted": 0.727433 + }, + { + "accuracy": 0.768398, + "f1": 0.765976, + "f1_weighted": 0.765873, + "ap": 0.727433, + "ap_weighted": 0.727433 + } + ], + "main_score": 0.768398, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.116210699081421, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115UserChoiceControlLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115UserChoiceControlLegalBenchClassification.json new file mode 100644 index 0000000000..5ba556c671 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/OPP115UserChoiceControlLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "OPP115UserChoiceControlLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.776843, + "f1": 0.775645, + "f1_weighted": 0.775687, + "ap": 0.706379, + "ap_weighted": 0.706379, + "scores_per_experiment": [ + { + "accuracy": 0.776843, + "f1": 0.775645, + "f1_weighted": 0.775687, + "ap": 0.706379, + "ap_weighted": 0.706379 + }, + { + "accuracy": 0.776843, + "f1": 0.775645, + "f1_weighted": 0.775687, + "ap": 0.706379, + "ap_weighted": 0.706379 + }, + { + "accuracy": 0.776843, + "f1": 0.775645, + "f1_weighted": 0.775687, + "ap": 0.706379, + "ap_weighted": 0.706379 + }, + { + "accuracy": 0.776843, + "f1": 0.775645, + "f1_weighted": 0.775687, + "ap": 0.706379, + "ap_weighted": 0.706379 + }, + { + "accuracy": 0.776843, + "f1": 0.775645, + "f1_weighted": 0.775687, + "ap": 0.706379, + "ap_weighted": 0.706379 + }, + { + "accuracy": 0.776843, + "f1": 0.775645, + "f1_weighted": 0.775687, + "ap": 0.706379, + "ap_weighted": 0.706379 + }, + { + "accuracy": 0.776843, + "f1": 0.775645, + "f1_weighted": 0.775687, + "ap": 0.706379, + "ap_weighted": 0.706379 + }, + { + "accuracy": 0.776843, + "f1": 0.775645, + "f1_weighted": 0.775687, + "ap": 0.706379, + "ap_weighted": 0.706379 + }, + { + "accuracy": 0.776843, + "f1": 0.775645, + "f1_weighted": 0.775687, + "ap": 0.706379, + "ap_weighted": 0.706379 + }, + { + "accuracy": 0.776843, + "f1": 0.775645, + "f1_weighted": 0.775687, + "ap": 0.706379, + "ap_weighted": 0.706379 + } + ], + "main_score": 0.776843, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.4511358737945557, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/OdiaNewsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/OdiaNewsClassification.json new file mode 100644 index 0000000000..eb1e691da1 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/OdiaNewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "ffb8a34c9637fb20256e8c7be02504d16af4bd6b", + "task_name": "OdiaNewsClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.605322, + "f1": 0.606853, + "f1_weighted": 0.599111, + "scores_per_experiment": [ + { + "accuracy": 0.577637, + "f1": 0.599534, + "f1_weighted": 0.585842 + }, + { + "accuracy": 0.551758, + "f1": 0.571249, + "f1_weighted": 0.558442 + }, + { + "accuracy": 0.564941, + "f1": 0.587189, + "f1_weighted": 0.567789 + }, + { + "accuracy": 0.609375, + "f1": 0.585377, + "f1_weighted": 0.586607 + }, + { + "accuracy": 0.645508, + "f1": 0.561793, + "f1_weighted": 0.570579 + }, + { + "accuracy": 0.643555, + "f1": 0.649535, + "f1_weighted": 0.643794 + }, + { + "accuracy": 0.584961, + "f1": 0.605896, + "f1_weighted": 0.588528 + }, + { + "accuracy": 0.643555, + "f1": 0.649952, + "f1_weighted": 0.645606 + }, + { + "accuracy": 0.630371, + "f1": 0.639937, + "f1_weighted": 0.634712 + }, + { + "accuracy": 0.601562, + "f1": 0.618068, + "f1_weighted": 0.609207 + } + ], + "main_score": 0.606853, + "hf_subset": "default", + "languages": [ + "ory-Orya" + ] + } + ] + }, + "evaluation_time": 2.499537944793701, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/OnlineShopping.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/OnlineShopping.json new file mode 100644 index 0000000000..febac93443 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/OnlineShopping.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "e610f2ebd179a8fda30ae534c3878750a96db120", + "task_name": "OnlineShopping", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.5413, + "f1": 0.532936, + "f1_weighted": 0.534087, + "ap": 0.545695, + "ap_weighted": 0.545695, + "scores_per_experiment": [ + { + "accuracy": 0.472, + "f1": 0.452655, + "f1_weighted": 0.447715, + "ap": 0.515614, + "ap_weighted": 0.515614 + }, + { + "accuracy": 0.569, + "f1": 0.567494, + "f1_weighted": 0.568719, + "ap": 0.561702, + "ap_weighted": 0.561702 + }, + { + "accuracy": 0.563, + "f1": 0.538367, + "f1_weighted": 0.543485, + "ap": 0.552432, + "ap_weighted": 0.552432 + }, + { + "accuracy": 0.545, + "f1": 0.540496, + "f1_weighted": 0.54268, + "ap": 0.546122, + "ap_weighted": 0.546122 + }, + { + "accuracy": 0.547, + "f1": 0.543217, + "f1_weighted": 0.545213, + "ap": 0.547505, + "ap_weighted": 0.547505 + }, + { + "accuracy": 0.553, + "f1": 0.552172, + "f1_weighted": 0.553096, + "ap": 0.552538, + "ap_weighted": 0.552538 + }, + { + "accuracy": 0.522, + "f1": 0.520602, + "f1_weighted": 0.519359, + "ap": 0.537621, + "ap_weighted": 0.537621 + }, + { + "accuracy": 0.537, + "f1": 0.524991, + "f1_weighted": 0.528617, + "ap": 0.53996, + "ap_weighted": 0.53996 + }, + { + "accuracy": 0.544, + "f1": 0.542565, + "f1_weighted": 0.541336, + "ap": 0.55052, + "ap_weighted": 0.55052 + }, + { + "accuracy": 0.561, + "f1": 0.546802, + "f1_weighted": 0.550652, + "ap": 0.552931, + "ap_weighted": 0.552931 + } + ], + "main_score": 0.5413, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 1.8583898544311523, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/OnlineStoreReviewSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/OnlineStoreReviewSentimentClassification.json new file mode 100644 index 0000000000..13bdeb3f58 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/OnlineStoreReviewSentimentClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "fb63ba1255f57054d411fe02bb5cec25cd6b150c", + "task_name": "OnlineStoreReviewSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.258496, + "f1": 0.147422, + "f1_weighted": 0.23219, + "scores_per_experiment": [ + { + "accuracy": 0.378906, + "f1": 0.160764, + "f1_weighted": 0.360562 + }, + { + "accuracy": 0.100586, + "f1": 0.115101, + "f1_weighted": 0.064183 + }, + { + "accuracy": 0.123535, + "f1": 0.110679, + "f1_weighted": 0.109631 + }, + { + "accuracy": 0.391602, + "f1": 0.191093, + "f1_weighted": 0.387792 + }, + { + "accuracy": 0.089844, + "f1": 0.088726, + "f1_weighted": 0.039864 + }, + { + "accuracy": 0.45459, + "f1": 0.202412, + "f1_weighted": 0.419596 + }, + { + "accuracy": 0.459961, + "f1": 0.207831, + "f1_weighted": 0.426158 + }, + { + "accuracy": 0.346191, + "f1": 0.190886, + "f1_weighted": 0.356267 + }, + { + "accuracy": 0.123047, + "f1": 0.104795, + "f1_weighted": 0.096102 + }, + { + "accuracy": 0.116699, + "f1": 0.101936, + "f1_weighted": 0.061743 + } + ], + "main_score": 0.258496, + "hf_subset": "default", + "languages": [ + "ara-Arab" + ] + } + ] + }, + "evaluation_time": 2.7035741806030273, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/OralArgumentQuestionPurposeLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/OralArgumentQuestionPurposeLegalBenchClassification.json new file mode 100644 index 0000000000..52fac7d785 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/OralArgumentQuestionPurposeLegalBenchClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "OralArgumentQuestionPurposeLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.195513, + "f1": 0.186493, + "f1_weighted": 0.186276, + "scores_per_experiment": [ + { + "accuracy": 0.195513, + "f1": 0.186493, + "f1_weighted": 0.186276 + }, + { + "accuracy": 0.195513, + "f1": 0.186493, + "f1_weighted": 0.186276 + }, + { + "accuracy": 0.195513, + "f1": 0.186493, + "f1_weighted": 0.186276 + }, + { + "accuracy": 0.195513, + "f1": 0.186493, + "f1_weighted": 0.186276 + }, + { + "accuracy": 0.195513, + "f1": 0.186493, + "f1_weighted": 0.186276 + }, + { + "accuracy": 0.195513, + "f1": 0.186493, + "f1_weighted": 0.186276 + }, + { + "accuracy": 0.195513, + "f1": 0.186493, + "f1_weighted": 0.186276 + }, + { + "accuracy": 0.195513, + "f1": 0.186493, + "f1_weighted": 0.186276 + }, + { + "accuracy": 0.195513, + "f1": 0.186493, + "f1_weighted": 0.186276 + }, + { + "accuracy": 0.195513, + "f1": 0.186493, + "f1_weighted": 0.186276 + } + ], + "main_score": 0.195513, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.279799222946167, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/OverrulingLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/OverrulingLegalBenchClassification.json new file mode 100644 index 0000000000..cc4e50e58b --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/OverrulingLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "OverrulingLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.745605, + "f1": 0.742663, + "f1_weighted": 0.742233, + "ap": 0.709725, + "ap_weighted": 0.709725, + "scores_per_experiment": [ + { + "accuracy": 0.745605, + "f1": 0.742663, + "f1_weighted": 0.742233, + "ap": 0.709725, + "ap_weighted": 0.709725 + }, + { + "accuracy": 0.745605, + "f1": 0.742663, + "f1_weighted": 0.742233, + "ap": 0.709725, + "ap_weighted": 0.709725 + }, + { + "accuracy": 0.745605, + "f1": 0.742663, + "f1_weighted": 0.742233, + "ap": 0.709725, + "ap_weighted": 0.709725 + }, + { + "accuracy": 0.745605, + "f1": 0.742663, + "f1_weighted": 0.742233, + "ap": 0.709725, + "ap_weighted": 0.709725 + }, + { + "accuracy": 0.745605, + "f1": 0.742663, + "f1_weighted": 0.742233, + "ap": 0.709725, + "ap_weighted": 0.709725 + }, + { + "accuracy": 0.745605, + "f1": 0.742663, + "f1_weighted": 0.742233, + "ap": 0.709725, + "ap_weighted": 0.709725 + }, + { + "accuracy": 0.745605, + "f1": 0.742663, + "f1_weighted": 0.742233, + "ap": 0.709725, + "ap_weighted": 0.709725 + }, + { + "accuracy": 0.745605, + "f1": 0.742663, + "f1_weighted": 0.742233, + "ap": 0.709725, + "ap_weighted": 0.709725 + }, + { + "accuracy": 0.745605, + "f1": 0.742663, + "f1_weighted": 0.742233, + "ap": 0.709725, + "ap_weighted": 0.709725 + }, + { + "accuracy": 0.745605, + "f1": 0.742663, + "f1_weighted": 0.742233, + "ap": 0.709725, + "ap_weighted": 0.709725 + } + ], + "main_score": 0.745605, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.5579707622528076, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/PAC.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/PAC.json new file mode 100644 index 0000000000..6328e49a04 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/PAC.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "fc69d1c153a8ccdcf1eef52f4e2a27f88782f543", + "task_name": "PAC", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.572632, + "f1": 0.548038, + "f1_weighted": 0.575814, + "ap": 0.716514, + "ap_weighted": 0.716514, + "scores_per_experiment": [ + { + "accuracy": 0.544454, + "f1": 0.54441, + "f1_weighted": 0.542834, + "ap": 0.744588, + "ap_weighted": 0.744588 + }, + { + "accuracy": 0.500145, + "f1": 0.494371, + "f1_weighted": 0.475389, + "ap": 0.742143, + "ap_weighted": 0.742143 + }, + { + "accuracy": 0.641182, + "f1": 0.607319, + "f1_weighted": 0.647827, + "ap": 0.731461, + "ap_weighted": 0.731461 + }, + { + "accuracy": 0.536635, + "f1": 0.525925, + "f1_weighted": 0.550956, + "ap": 0.700029, + "ap_weighted": 0.700029 + }, + { + "accuracy": 0.622937, + "f1": 0.579569, + "f1_weighted": 0.627004, + "ap": 0.714508, + "ap_weighted": 0.714508 + }, + { + "accuracy": 0.550246, + "f1": 0.533192, + "f1_weighted": 0.564536, + "ap": 0.699487, + "ap_weighted": 0.699487 + }, + { + "accuracy": 0.530843, + "f1": 0.528697, + "f1_weighted": 0.539869, + "ap": 0.71454, + "ap_weighted": 0.71454 + }, + { + "accuracy": 0.622357, + "f1": 0.549203, + "f1_weighted": 0.612996, + "ap": 0.697839, + "ap_weighted": 0.697839 + }, + { + "accuracy": 0.604981, + "f1": 0.588799, + "f1_weighted": 0.617455, + "ap": 0.730801, + "ap_weighted": 0.730801 + }, + { + "accuracy": 0.572546, + "f1": 0.528902, + "f1_weighted": 0.579273, + "ap": 0.689746, + "ap_weighted": 0.689746 + } + ], + "main_score": 0.572632, + "hf_subset": "default", + "languages": [ + "pol-Latn" + ] + } + ] + }, + "evaluation_time": 8.277080059051514, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/PROALegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/PROALegalBenchClassification.json new file mode 100644 index 0000000000..2113fdd53b --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/PROALegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "PROALegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.494737, + "f1": 0.330986, + "f1_weighted": 0.327502, + "ap": 0.494737, + "ap_weighted": 0.494737, + "scores_per_experiment": [ + { + "accuracy": 0.494737, + "f1": 0.330986, + "f1_weighted": 0.327502, + "ap": 0.494737, + "ap_weighted": 0.494737 + }, + { + "accuracy": 0.494737, + "f1": 0.330986, + "f1_weighted": 0.327502, + "ap": 0.494737, + "ap_weighted": 0.494737 + }, + { + "accuracy": 0.494737, + "f1": 0.330986, + "f1_weighted": 0.327502, + "ap": 0.494737, + "ap_weighted": 0.494737 + }, + { + "accuracy": 0.494737, + "f1": 0.330986, + "f1_weighted": 0.327502, + "ap": 0.494737, + "ap_weighted": 0.494737 + }, + { + "accuracy": 0.494737, + "f1": 0.330986, + "f1_weighted": 0.327502, + "ap": 0.494737, + "ap_weighted": 0.494737 + }, + { + "accuracy": 0.494737, + "f1": 0.330986, + "f1_weighted": 0.327502, + "ap": 0.494737, + "ap_weighted": 0.494737 + }, + { + "accuracy": 0.494737, + "f1": 0.330986, + "f1_weighted": 0.327502, + "ap": 0.494737, + "ap_weighted": 0.494737 + }, + { + "accuracy": 0.494737, + "f1": 0.330986, + "f1_weighted": 0.327502, + "ap": 0.494737, + "ap_weighted": 0.494737 + }, + { + "accuracy": 0.494737, + "f1": 0.330986, + "f1_weighted": 0.327502, + "ap": 0.494737, + "ap_weighted": 0.494737 + }, + { + "accuracy": 0.494737, + "f1": 0.330986, + "f1_weighted": 0.327502, + "ap": 0.494737, + "ap_weighted": 0.494737 + } + ], + "main_score": 0.494737, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.6141514778137207, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/PatentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/PatentClassification.json new file mode 100644 index 0000000000..abbb98aaa4 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/PatentClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "2f38a1dfdecfacee0184d74eaeafd3c0fb49d2a6", + "task_name": "PatentClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.328223, + "f1": 0.277196, + "f1_weighted": 0.313154, + "scores_per_experiment": [ + { + "accuracy": 0.333496, + "f1": 0.294191, + "f1_weighted": 0.321099 + }, + { + "accuracy": 0.330566, + "f1": 0.276013, + "f1_weighted": 0.318284 + }, + { + "accuracy": 0.358887, + "f1": 0.293302, + "f1_weighted": 0.336796 + }, + { + "accuracy": 0.322754, + "f1": 0.276682, + "f1_weighted": 0.312348 + }, + { + "accuracy": 0.348633, + "f1": 0.297152, + "f1_weighted": 0.339806 + }, + { + "accuracy": 0.312012, + "f1": 0.26396, + "f1_weighted": 0.299842 + }, + { + "accuracy": 0.317871, + "f1": 0.260913, + "f1_weighted": 0.284293 + }, + { + "accuracy": 0.323242, + "f1": 0.272443, + "f1_weighted": 0.306692 + }, + { + "accuracy": 0.32373, + "f1": 0.275663, + "f1_weighted": 0.312299 + }, + { + "accuracy": 0.311035, + "f1": 0.261642, + "f1_weighted": 0.300084 + } + ], + "main_score": 0.328223, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 87.55868482589722, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/PersianFoodSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/PersianFoodSentimentClassification.json new file mode 100644 index 0000000000..c434ebbaec --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/PersianFoodSentimentClassification.json @@ -0,0 +1,181 @@ +{ + "dataset_revision": "92ba517dfd22f6334111ad84154d16a2890f5b1d", + "task_name": "PersianFoodSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "validation": [ + { + "accuracy": 0.510938, + "f1": 0.485305, + "f1_weighted": 0.485305, + "ap": 0.508536, + "ap_weighted": 0.508536, + "scores_per_experiment": [ + { + "accuracy": 0.529297, + "f1": 0.529275, + "f1_weighted": 0.529275, + "ap": 0.515495, + "ap_weighted": 0.515495 + }, + { + "accuracy": 0.483398, + "f1": 0.389827, + "f1_weighted": 0.389827, + "ap": 0.492971, + "ap_weighted": 0.492971 + }, + { + "accuracy": 0.463867, + "f1": 0.409389, + "f1_weighted": 0.409389, + "ap": 0.485259, + "ap_weighted": 0.485259 + }, + { + "accuracy": 0.587402, + "f1": 0.586986, + "f1_weighted": 0.586986, + "ap": 0.551858, + "ap_weighted": 0.551858 + }, + { + "accuracy": 0.51123, + "f1": 0.457041, + "f1_weighted": 0.457041, + "ap": 0.505958, + "ap_weighted": 0.505958 + }, + { + "accuracy": 0.481934, + "f1": 0.442137, + "f1_weighted": 0.442137, + "ap": 0.491667, + "ap_weighted": 0.491667 + }, + { + "accuracy": 0.581055, + "f1": 0.578079, + "f1_weighted": 0.578079, + "ap": 0.548424, + "ap_weighted": 0.548424 + }, + { + "accuracy": 0.448242, + "f1": 0.447768, + "f1_weighted": 0.447768, + "ap": 0.476652, + "ap_weighted": 0.476652 + }, + { + "accuracy": 0.568848, + "f1": 0.561393, + "f1_weighted": 0.561393, + "ap": 0.538184, + "ap_weighted": 0.538184 + }, + { + "accuracy": 0.454102, + "f1": 0.451157, + "f1_weighted": 0.451157, + "ap": 0.478888, + "ap_weighted": 0.478888 + } + ], + "main_score": 0.510938, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ], + "test": [ + { + "accuracy": 0.508838, + "f1": 0.481712, + "f1_weighted": 0.481712, + "ap": 0.507102, + "ap_weighted": 0.507102, + "scores_per_experiment": [ + { + "accuracy": 0.536133, + "f1": 0.53576, + "f1_weighted": 0.53576, + "ap": 0.519302, + "ap_weighted": 0.519302 + }, + { + "accuracy": 0.473633, + "f1": 0.373722, + "f1_weighted": 0.373722, + "ap": 0.490272, + "ap_weighted": 0.490272 + }, + { + "accuracy": 0.469238, + "f1": 0.422085, + "f1_weighted": 0.422085, + "ap": 0.486826, + "ap_weighted": 0.486826 + }, + { + "accuracy": 0.57373, + "f1": 0.573029, + "f1_weighted": 0.573029, + "ap": 0.542781, + "ap_weighted": 0.542781 + }, + { + "accuracy": 0.5, + "f1": 0.437637, + "f1_weighted": 0.437637, + "ap": 0.5, + "ap_weighted": 0.5 + }, + { + "accuracy": 0.475098, + "f1": 0.428464, + "f1_weighted": 0.428464, + "ap": 0.488995, + "ap_weighted": 0.488995 + }, + { + "accuracy": 0.574707, + "f1": 0.571506, + "f1_weighted": 0.571506, + "ap": 0.544101, + "ap_weighted": 0.544101 + }, + { + "accuracy": 0.468262, + "f1": 0.467692, + "f1_weighted": 0.467692, + "ap": 0.485076, + "ap_weighted": 0.485076 + }, + { + "accuracy": 0.5625, + "f1": 0.5562, + "f1_weighted": 0.5562, + "ap": 0.534405, + "ap_weighted": 0.534405 + }, + { + "accuracy": 0.455078, + "f1": 0.451024, + "f1_weighted": 0.451024, + "ap": 0.479261, + "ap_weighted": 0.479261 + } + ], + "main_score": 0.508838, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 8.81991195678711, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/PersianTextEmotion.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/PersianTextEmotion.json new file mode 100644 index 0000000000..71fb282f1b --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/PersianTextEmotion.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "518fcd2c8b89917c7696770672688217a2eabf88", + "task_name": "PersianTextEmotion", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.289856, + "f1": 0.258942, + "f1_weighted": 0.258973, + "scores_per_experiment": [ + { + "accuracy": 0.280576, + "f1": 0.248816, + "f1_weighted": 0.248828 + }, + { + "accuracy": 0.280576, + "f1": 0.269687, + "f1_weighted": 0.269788 + }, + { + "accuracy": 0.290647, + "f1": 0.245624, + "f1_weighted": 0.245565 + }, + { + "accuracy": 0.28705, + "f1": 0.261338, + "f1_weighted": 0.261355 + }, + { + "accuracy": 0.299281, + "f1": 0.263357, + "f1_weighted": 0.263389 + }, + { + "accuracy": 0.305036, + "f1": 0.264879, + "f1_weighted": 0.264862 + }, + { + "accuracy": 0.294245, + "f1": 0.281514, + "f1_weighted": 0.2816 + }, + { + "accuracy": 0.281295, + "f1": 0.244294, + "f1_weighted": 0.244278 + }, + { + "accuracy": 0.292806, + "f1": 0.272814, + "f1_weighted": 0.272894 + }, + { + "accuracy": 0.28705, + "f1": 0.237093, + "f1_weighted": 0.237173 + } + ], + "main_score": 0.289856, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 15.379451036453247, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/PersianTextTone.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/PersianTextTone.json new file mode 100644 index 0000000000..7b9dc97383 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/PersianTextTone.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "7144f4c6bdd77911df0dfc5a8bd44dba17e27e3a", + "task_name": "PersianTextTone", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.421311, + "f1": 0.444718, + "f1_weighted": 0.412529, + "scores_per_experiment": [ + { + "accuracy": 0.488388, + "f1": 0.495341, + "f1_weighted": 0.502546 + }, + { + "accuracy": 0.40403, + "f1": 0.453555, + "f1_weighted": 0.3962 + }, + { + "accuracy": 0.438183, + "f1": 0.465591, + "f1_weighted": 0.446134 + }, + { + "accuracy": 0.420082, + "f1": 0.428632, + "f1_weighted": 0.408329 + }, + { + "accuracy": 0.498634, + "f1": 0.468554, + "f1_weighted": 0.501244 + }, + { + "accuracy": 0.393784, + "f1": 0.41147, + "f1_weighted": 0.365111 + }, + { + "accuracy": 0.447746, + "f1": 0.469567, + "f1_weighted": 0.447111 + }, + { + "accuracy": 0.415984, + "f1": 0.431854, + "f1_weighted": 0.403771 + }, + { + "accuracy": 0.346653, + "f1": 0.416767, + "f1_weighted": 0.320704 + }, + { + "accuracy": 0.359631, + "f1": 0.405851, + "f1_weighted": 0.334138 + } + ], + "main_score": 0.421311, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 9.051206350326538, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/PersonalJurisdictionLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/PersonalJurisdictionLegalBenchClassification.json new file mode 100644 index 0000000000..464a18a6ae --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/PersonalJurisdictionLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "PersonalJurisdictionLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.52, + "f1": 0.519231, + "f1_weighted": 0.516154, + "ap": 0.441075, + "ap_weighted": 0.441075, + "scores_per_experiment": [ + { + "accuracy": 0.52, + "f1": 0.519231, + "f1_weighted": 0.516154, + "ap": 0.441075, + "ap_weighted": 0.441075 + }, + { + "accuracy": 0.52, + "f1": 0.519231, + "f1_weighted": 0.516154, + "ap": 0.441075, + "ap_weighted": 0.441075 + }, + { + "accuracy": 0.52, + "f1": 0.519231, + "f1_weighted": 0.516154, + "ap": 0.441075, + "ap_weighted": 0.441075 + }, + { + "accuracy": 0.52, + "f1": 0.519231, + "f1_weighted": 0.516154, + "ap": 0.441075, + "ap_weighted": 0.441075 + }, + { + "accuracy": 0.52, + "f1": 0.519231, + "f1_weighted": 0.516154, + "ap": 0.441075, + "ap_weighted": 0.441075 + }, + { + "accuracy": 0.52, + "f1": 0.519231, + "f1_weighted": 0.516154, + "ap": 0.441075, + "ap_weighted": 0.441075 + }, + { + "accuracy": 0.52, + "f1": 0.519231, + "f1_weighted": 0.516154, + "ap": 0.441075, + "ap_weighted": 0.441075 + }, + { + "accuracy": 0.52, + "f1": 0.519231, + "f1_weighted": 0.516154, + "ap": 0.441075, + "ap_weighted": 0.441075 + }, + { + "accuracy": 0.52, + "f1": 0.519231, + "f1_weighted": 0.516154, + "ap": 0.441075, + "ap_weighted": 0.441075 + }, + { + "accuracy": 0.52, + "f1": 0.519231, + "f1_weighted": 0.516154, + "ap": 0.441075, + "ap_weighted": 0.441075 + } + ], + "main_score": 0.52, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.5507400035858154, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/PoemSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/PoemSentimentClassification.json new file mode 100644 index 0000000000..18c9985908 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/PoemSentimentClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "329d529d875a00c47ec71954a1a96ae167584770", + "task_name": "PoemSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "validation": [ + { + "accuracy": 0.430476, + "f1": 0.315979, + "f1_weighted": 0.489561, + "scores_per_experiment": [ + { + "accuracy": 0.409524, + "f1": 0.313036, + "f1_weighted": 0.470569 + }, + { + "accuracy": 0.428571, + "f1": 0.338569, + "f1_weighted": 0.479427 + }, + { + "accuracy": 0.409524, + "f1": 0.320904, + "f1_weighted": 0.483917 + }, + { + "accuracy": 0.466667, + "f1": 0.366471, + "f1_weighted": 0.549958 + }, + { + "accuracy": 0.409524, + "f1": 0.28633, + "f1_weighted": 0.467913 + }, + { + "accuracy": 0.409524, + "f1": 0.286162, + "f1_weighted": 0.446896 + }, + { + "accuracy": 0.495238, + "f1": 0.343045, + "f1_weighted": 0.553022 + }, + { + "accuracy": 0.4, + "f1": 0.273706, + "f1_weighted": 0.450627 + }, + { + "accuracy": 0.4, + "f1": 0.270342, + "f1_weighted": 0.466158 + }, + { + "accuracy": 0.47619, + "f1": 0.361222, + "f1_weighted": 0.527119 + } + ], + "main_score": 0.430476, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test": [ + { + "accuracy": 0.426923, + "f1": 0.318097, + "f1_weighted": 0.48638, + "scores_per_experiment": [ + { + "accuracy": 0.451923, + "f1": 0.321725, + "f1_weighted": 0.524868 + }, + { + "accuracy": 0.471154, + "f1": 0.385636, + "f1_weighted": 0.501383 + }, + { + "accuracy": 0.423077, + "f1": 0.315855, + "f1_weighted": 0.500653 + }, + { + "accuracy": 0.442308, + "f1": 0.358165, + "f1_weighted": 0.528992 + }, + { + "accuracy": 0.480769, + "f1": 0.314326, + "f1_weighted": 0.530857 + }, + { + "accuracy": 0.346154, + "f1": 0.27132, + "f1_weighted": 0.37214 + }, + { + "accuracy": 0.490385, + "f1": 0.349008, + "f1_weighted": 0.562235 + }, + { + "accuracy": 0.394231, + "f1": 0.297658, + "f1_weighted": 0.45094 + }, + { + "accuracy": 0.365385, + "f1": 0.263388, + "f1_weighted": 0.430184 + }, + { + "accuracy": 0.403846, + "f1": 0.303891, + "f1_weighted": 0.461544 + } + ], + "main_score": 0.426923, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.9417047500610352, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/PolEmo2.0-IN.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/PolEmo2.0-IN.json new file mode 100644 index 0000000000..d1adc67cac --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/PolEmo2.0-IN.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "d90724373c70959f17d2331ad51fb60c71176b03", + "task_name": "PolEmo2.0-IN", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.397922, + "f1": 0.388682, + "f1_weighted": 0.389421, + "scores_per_experiment": [ + { + "accuracy": 0.375346, + "f1": 0.357778, + "f1_weighted": 0.36397 + }, + { + "accuracy": 0.401662, + "f1": 0.379269, + "f1_weighted": 0.384477 + }, + { + "accuracy": 0.421053, + "f1": 0.411945, + "f1_weighted": 0.403826 + }, + { + "accuracy": 0.369806, + "f1": 0.347765, + "f1_weighted": 0.360481 + }, + { + "accuracy": 0.416898, + "f1": 0.419456, + "f1_weighted": 0.407753 + }, + { + "accuracy": 0.315789, + "f1": 0.298619, + "f1_weighted": 0.292804 + }, + { + "accuracy": 0.409972, + "f1": 0.414985, + "f1_weighted": 0.414817 + }, + { + "accuracy": 0.383657, + "f1": 0.37833, + "f1_weighted": 0.382318 + }, + { + "accuracy": 0.451524, + "f1": 0.446795, + "f1_weighted": 0.450463 + }, + { + "accuracy": 0.433518, + "f1": 0.431878, + "f1_weighted": 0.433304 + } + ], + "main_score": 0.397922, + "hf_subset": "default", + "languages": [ + "pol-Latn" + ] + } + ] + }, + "evaluation_time": 8.416445016860962, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/PolEmo2.0-OUT.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/PolEmo2.0-OUT.json new file mode 100644 index 0000000000..171d1b6ec8 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/PolEmo2.0-OUT.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "6a21ab8716e255ab1867265f8b396105e8aa63d4", + "task_name": "PolEmo2.0-OUT", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.198988, + "f1": 0.181956, + "f1_weighted": 0.239836, + "scores_per_experiment": [ + { + "accuracy": 0.133603, + "f1": 0.138957, + "f1_weighted": 0.187553 + }, + { + "accuracy": 0.196356, + "f1": 0.194748, + "f1_weighted": 0.246942 + }, + { + "accuracy": 0.287449, + "f1": 0.250903, + "f1_weighted": 0.328772 + }, + { + "accuracy": 0.186235, + "f1": 0.193669, + "f1_weighted": 0.254967 + }, + { + "accuracy": 0.17004, + "f1": 0.164706, + "f1_weighted": 0.215902 + }, + { + "accuracy": 0.050607, + "f1": 0.066673, + "f1_weighted": 0.087911 + }, + { + "accuracy": 0.224696, + "f1": 0.207692, + "f1_weighted": 0.268742 + }, + { + "accuracy": 0.283401, + "f1": 0.249726, + "f1_weighted": 0.336125 + }, + { + "accuracy": 0.309717, + "f1": 0.222234, + "f1_weighted": 0.296354 + }, + { + "accuracy": 0.147773, + "f1": 0.130252, + "f1_weighted": 0.175095 + } + ], + "main_score": 0.198988, + "hf_subset": "default", + "languages": [ + "pol-Latn" + ] + } + ] + }, + "evaluation_time": 6.448662519454956, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/PunjabiNewsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/PunjabiNewsClassification.json new file mode 100644 index 0000000000..c8ba61fb32 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/PunjabiNewsClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "cec3923e16519efe51d535497e711932b8f1dc44", + "task_name": "PunjabiNewsClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.60828, + "f1": 0.543512, + "f1_weighted": 0.649639, + "ap": 0.239275, + "ap_weighted": 0.239275, + "scores_per_experiment": [ + { + "accuracy": 0.700637, + "f1": 0.609018, + "f1_weighted": 0.735597, + "ap": 0.259423, + "ap_weighted": 0.259423 + }, + { + "accuracy": 0.630573, + "f1": 0.578894, + "f1_weighted": 0.677555, + "ap": 0.270417, + "ap_weighted": 0.270417 + }, + { + "accuracy": 0.694268, + "f1": 0.615667, + "f1_weighted": 0.731907, + "ap": 0.275996, + "ap_weighted": 0.275996 + }, + { + "accuracy": 0.433121, + "f1": 0.419557, + "f1_weighted": 0.478899, + "ap": 0.199453, + "ap_weighted": 0.199453 + }, + { + "accuracy": 0.515924, + "f1": 0.490608, + "f1_weighted": 0.566555, + "ap": 0.231047, + "ap_weighted": 0.231047 + }, + { + "accuracy": 0.55414, + "f1": 0.514233, + "f1_weighted": 0.60735, + "ap": 0.229074, + "ap_weighted": 0.229074 + }, + { + "accuracy": 0.605096, + "f1": 0.558509, + "f1_weighted": 0.654423, + "ap": 0.25817, + "ap_weighted": 0.25817 + }, + { + "accuracy": 0.764331, + "f1": 0.580062, + "f1_weighted": 0.766103, + "ap": 0.205818, + "ap_weighted": 0.205818 + }, + { + "accuracy": 0.656051, + "f1": 0.567625, + "f1_weighted": 0.698396, + "ap": 0.227797, + "ap_weighted": 0.227797 + }, + { + "accuracy": 0.528662, + "f1": 0.500945, + "f1_weighted": 0.579603, + "ap": 0.235557, + "ap_weighted": 0.235557 + } + ], + "main_score": 0.60828, + "hf_subset": "default", + "languages": [ + "pan-Guru" + ] + } + ] + }, + "evaluation_time": 4.884253263473511, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/QuoraRetrieval.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/QuoraRetrieval.json deleted file mode 100644 index fb60993e12..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/QuoraRetrieval.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "test": { - "evaluation_time": 414.64, - "map_at_1": 0.70412, - "map_at_10": 0.84226, - "map_at_100": 0.84879, - "map_at_1000": 0.84896, - "map_at_3": 0.81315, - "map_at_5": 0.83123, - "ndcg_at_1": 0.8104, - "ndcg_at_10": 0.87977, - "ndcg_at_100": 0.89248, - "ndcg_at_1000": 0.89365, - "ndcg_at_3": 0.8513, - "ndcg_at_5": 0.86647, - "precision_at_1": 0.8104, - "precision_at_10": 0.13339, - "precision_at_100": 0.01528, - "precision_at_1000": 0.00157, - "precision_at_3": 0.3718, - "precision_at_5": 0.24422, - "recall_at_1": 0.70412, - "recall_at_10": 0.95112, - "recall_at_100": 0.99426, - "recall_at_1000": 0.99971, - "recall_at_3": 0.86902, - "recall_at_5": 0.91238 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "QuoraRetrieval", - "dataset_revision": "6205996560df11e3a3da9ab4f926788fc30a7db4" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/RedditClustering.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/RedditClustering.json deleted file mode 100644 index 6e4f08a261..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/RedditClustering.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "test": { - "evaluation_time": 301.36, - "v_measure": 0.5613269289874916, - "v_measure_std": 0.04814425068842244 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "RedditClustering", - "dataset_revision": "b2805658ae38990172679479369a78b86de8c390" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/RedditClusteringP2P.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/RedditClusteringP2P.json deleted file mode 100644 index 621a629eaf..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/RedditClusteringP2P.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "mteb_version": "0.0.2", - "test": { - "evaluation_time": 2466.24, - "v_measure": 0.585344949362798, - "v_measure_std": 0.11545088428737668 - }, - "mteb_dataset_name": "RedditClusteringP2P", - "dataset_revision": "385e3cb46b4cfa89021f56c4380204149d0efe33" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/RestaurantReviewSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/RestaurantReviewSentimentClassification.json new file mode 100644 index 0000000000..4f2736393a --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/RestaurantReviewSentimentClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "d51bf2435d030e0041344f576c5e8d7154828977", + "task_name": "RestaurantReviewSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.499805, + "f1": 0.473383, + "f1_weighted": 0.517018, + "ap": 0.713142, + "ap_weighted": 0.713142, + "scores_per_experiment": [ + { + "accuracy": 0.42041, + "f1": 0.418029, + "f1_weighted": 0.433734, + "ap": 0.702453, + "ap_weighted": 0.702453 + }, + { + "accuracy": 0.486816, + "f1": 0.463035, + "f1_weighted": 0.510708, + "ap": 0.704567, + "ap_weighted": 0.704567 + }, + { + "accuracy": 0.519043, + "f1": 0.494071, + "f1_weighted": 0.54149, + "ap": 0.718231, + "ap_weighted": 0.718231 + }, + { + "accuracy": 0.406738, + "f1": 0.406426, + "f1_weighted": 0.412172, + "ap": 0.70418, + "ap_weighted": 0.70418 + }, + { + "accuracy": 0.499023, + "f1": 0.460388, + "f1_weighted": 0.521302, + "ap": 0.698892, + "ap_weighted": 0.698892 + }, + { + "accuracy": 0.537598, + "f1": 0.467077, + "f1_weighted": 0.548862, + "ap": 0.697887, + "ap_weighted": 0.697887 + }, + { + "accuracy": 0.54248, + "f1": 0.511852, + "f1_weighted": 0.563437, + "ap": 0.724269, + "ap_weighted": 0.724269 + }, + { + "accuracy": 0.57959, + "f1": 0.539102, + "f1_weighted": 0.596732, + "ap": 0.733847, + "ap_weighted": 0.733847 + }, + { + "accuracy": 0.541016, + "f1": 0.511676, + "f1_weighted": 0.562173, + "ap": 0.724701, + "ap_weighted": 0.724701 + }, + { + "accuracy": 0.465332, + "f1": 0.462171, + "f1_weighted": 0.479565, + "ap": 0.722392, + "ap_weighted": 0.722392 + } + ], + "main_score": 0.499805, + "hf_subset": "default", + "languages": [ + "ara-Arab" + ] + } + ] + }, + "evaluation_time": 5.379749774932861, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/RomanianReviewsSentiment.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/RomanianReviewsSentiment.json new file mode 100644 index 0000000000..0fda32d4fe --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/RomanianReviewsSentiment.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "358bcc95aeddd5d07a4524ee416f03d993099b23", + "task_name": "RomanianReviewsSentiment", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.35083, + "f1": 0.302565, + "f1_weighted": 0.376847, + "scores_per_experiment": [ + { + "accuracy": 0.399414, + "f1": 0.326209, + "f1_weighted": 0.422249 + }, + { + "accuracy": 0.319336, + "f1": 0.279834, + "f1_weighted": 0.338758 + }, + { + "accuracy": 0.320312, + "f1": 0.298115, + "f1_weighted": 0.363399 + }, + { + "accuracy": 0.405762, + "f1": 0.336871, + "f1_weighted": 0.438091 + }, + { + "accuracy": 0.339844, + "f1": 0.297876, + "f1_weighted": 0.369353 + }, + { + "accuracy": 0.326172, + "f1": 0.265145, + "f1_weighted": 0.341452 + }, + { + "accuracy": 0.35791, + "f1": 0.311238, + "f1_weighted": 0.394387 + }, + { + "accuracy": 0.408691, + "f1": 0.341324, + "f1_weighted": 0.422522 + }, + { + "accuracy": 0.358887, + "f1": 0.309147, + "f1_weighted": 0.377058 + }, + { + "accuracy": 0.271973, + "f1": 0.25989, + "f1_weighted": 0.3012 + } + ], + "main_score": 0.35083, + "hf_subset": "default", + "languages": [ + "ron-Latn" + ] + } + ] + }, + "evaluation_time": 5.457130670547485, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/RomanianSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/RomanianSentimentClassification.json new file mode 100644 index 0000000000..95b1f41ec2 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/RomanianSentimentClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "155048684cea7a6d6af1ddbfeb9a04820311ce93", + "task_name": "RomanianSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.540039, + "f1": 0.511222, + "f1_weighted": 0.50507, + "ap": 0.599247, + "ap_weighted": 0.599247, + "scores_per_experiment": [ + { + "accuracy": 0.567383, + "f1": 0.553903, + "f1_weighted": 0.563445, + "ap": 0.590392, + "ap_weighted": 0.590392 + }, + { + "accuracy": 0.550293, + "f1": 0.517357, + "f1_weighted": 0.501843, + "ap": 0.626876, + "ap_weighted": 0.626876 + }, + { + "accuracy": 0.510742, + "f1": 0.450121, + "f1_weighted": 0.427655, + "ap": 0.60575, + "ap_weighted": 0.60575 + }, + { + "accuracy": 0.608887, + "f1": 0.605455, + "f1_weighted": 0.600927, + "ap": 0.646598, + "ap_weighted": 0.646598 + }, + { + "accuracy": 0.461426, + "f1": 0.3803, + "f1_weighted": 0.352711, + "ap": 0.569343, + "ap_weighted": 0.569343 + }, + { + "accuracy": 0.549316, + "f1": 0.517358, + "f1_weighted": 0.502076, + "ap": 0.625069, + "ap_weighted": 0.625069 + }, + { + "accuracy": 0.529297, + "f1": 0.509843, + "f1_weighted": 0.521859, + "ap": 0.567807, + "ap_weighted": 0.567807 + }, + { + "accuracy": 0.523438, + "f1": 0.511664, + "f1_weighted": 0.502334, + "ap": 0.590528, + "ap_weighted": 0.590528 + }, + { + "accuracy": 0.500488, + "f1": 0.499365, + "f1_weighted": 0.496448, + "ap": 0.568686, + "ap_weighted": 0.568686 + }, + { + "accuracy": 0.599121, + "f1": 0.566851, + "f1_weighted": 0.581398, + "ap": 0.601421, + "ap_weighted": 0.601421 + } + ], + "main_score": 0.540039, + "hf_subset": "default", + "languages": [ + "ron-Latn" + ] + } + ] + }, + "evaluation_time": 9.549221277236938, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/RuReviewsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/RuReviewsClassification.json new file mode 100644 index 0000000000..eaf3c46a63 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/RuReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a", + "task_name": "RuReviewsClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.380273, + "f1": 0.360355, + "f1_weighted": 0.360353, + "scores_per_experiment": [ + { + "accuracy": 0.416504, + "f1": 0.390697, + "f1_weighted": 0.390631 + }, + { + "accuracy": 0.376953, + "f1": 0.364155, + "f1_weighted": 0.364147 + }, + { + "accuracy": 0.328125, + "f1": 0.31761, + "f1_weighted": 0.317643 + }, + { + "accuracy": 0.378418, + "f1": 0.352164, + "f1_weighted": 0.352255 + }, + { + "accuracy": 0.419434, + "f1": 0.414268, + "f1_weighted": 0.414241 + }, + { + "accuracy": 0.418945, + "f1": 0.374464, + "f1_weighted": 0.374398 + }, + { + "accuracy": 0.364746, + "f1": 0.320352, + "f1_weighted": 0.320271 + }, + { + "accuracy": 0.415527, + "f1": 0.412315, + "f1_weighted": 0.412322 + }, + { + "accuracy": 0.323242, + "f1": 0.317591, + "f1_weighted": 0.317645 + }, + { + "accuracy": 0.36084, + "f1": 0.339928, + "f1_weighted": 0.339979 + } + ], + "main_score": 0.380273, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 6.673748731613159, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/RuSciBenchGRNTIClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/RuSciBenchGRNTIClassification.json new file mode 100644 index 0000000000..aa087ecbbe --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/RuSciBenchGRNTIClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "task_name": "RuSciBenchGRNTIClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.10083, + "f1": 0.075743, + "f1_weighted": 0.075731, + "scores_per_experiment": [ + { + "accuracy": 0.101562, + "f1": 0.080004, + "f1_weighted": 0.079996 + }, + { + "accuracy": 0.101562, + "f1": 0.077666, + "f1_weighted": 0.077616 + }, + { + "accuracy": 0.100586, + "f1": 0.072666, + "f1_weighted": 0.072676 + }, + { + "accuracy": 0.109863, + "f1": 0.080207, + "f1_weighted": 0.080181 + }, + { + "accuracy": 0.088379, + "f1": 0.077662, + "f1_weighted": 0.077575 + }, + { + "accuracy": 0.108398, + "f1": 0.086821, + "f1_weighted": 0.086792 + }, + { + "accuracy": 0.095703, + "f1": 0.076581, + "f1_weighted": 0.076626 + }, + { + "accuracy": 0.097656, + "f1": 0.061573, + "f1_weighted": 0.06163 + }, + { + "accuracy": 0.109375, + "f1": 0.084389, + "f1_weighted": 0.084385 + }, + { + "accuracy": 0.095215, + "f1": 0.059864, + "f1_weighted": 0.059829 + } + ], + "main_score": 0.10083, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 44.25600218772888, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/RuSciBenchOECDClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/RuSciBenchOECDClassification.json new file mode 100644 index 0000000000..07cb58c3db --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/RuSciBenchOECDClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "task_name": "RuSciBenchOECDClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.090625, + "f1": 0.061111, + "f1_weighted": 0.061186, + "scores_per_experiment": [ + { + "accuracy": 0.083008, + "f1": 0.057425, + "f1_weighted": 0.057445 + }, + { + "accuracy": 0.098633, + "f1": 0.058975, + "f1_weighted": 0.059061 + }, + { + "accuracy": 0.099121, + "f1": 0.069795, + "f1_weighted": 0.06985 + }, + { + "accuracy": 0.087891, + "f1": 0.050896, + "f1_weighted": 0.050923 + }, + { + "accuracy": 0.081543, + "f1": 0.060761, + "f1_weighted": 0.060742 + }, + { + "accuracy": 0.091797, + "f1": 0.063, + "f1_weighted": 0.06313 + }, + { + "accuracy": 0.095703, + "f1": 0.066162, + "f1_weighted": 0.066315 + }, + { + "accuracy": 0.098145, + "f1": 0.057857, + "f1_weighted": 0.058012 + }, + { + "accuracy": 0.081543, + "f1": 0.059839, + "f1_weighted": 0.059926 + }, + { + "accuracy": 0.088867, + "f1": 0.066402, + "f1_weighted": 0.066458 + } + ], + "main_score": 0.090625, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ] + } + ] + }, + "evaluation_time": 43.81903028488159, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDBPAccountabilityLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDBPAccountabilityLegalBenchClassification.json new file mode 100644 index 0000000000..7f3dbca416 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDBPAccountabilityLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "SCDBPAccountabilityLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.530343, + "f1": 0.462094, + "f1_weighted": 0.602131, + "ap": 0.884791, + "ap_weighted": 0.884791, + "scores_per_experiment": [ + { + "accuracy": 0.530343, + "f1": 0.462094, + "f1_weighted": 0.602131, + "ap": 0.884791, + "ap_weighted": 0.884791 + }, + { + "accuracy": 0.530343, + "f1": 0.462094, + "f1_weighted": 0.602131, + "ap": 0.884791, + "ap_weighted": 0.884791 + }, + { + "accuracy": 0.530343, + "f1": 0.462094, + "f1_weighted": 0.602131, + "ap": 0.884791, + "ap_weighted": 0.884791 + }, + { + "accuracy": 0.530343, + "f1": 0.462094, + "f1_weighted": 0.602131, + "ap": 0.884791, + "ap_weighted": 0.884791 + }, + { + "accuracy": 0.530343, + "f1": 0.462094, + "f1_weighted": 0.602131, + "ap": 0.884791, + "ap_weighted": 0.884791 + }, + { + "accuracy": 0.530343, + "f1": 0.462094, + "f1_weighted": 0.602131, + "ap": 0.884791, + "ap_weighted": 0.884791 + }, + { + "accuracy": 0.530343, + "f1": 0.462094, + "f1_weighted": 0.602131, + "ap": 0.884791, + "ap_weighted": 0.884791 + }, + { + "accuracy": 0.530343, + "f1": 0.462094, + "f1_weighted": 0.602131, + "ap": 0.884791, + "ap_weighted": 0.884791 + }, + { + "accuracy": 0.530343, + "f1": 0.462094, + "f1_weighted": 0.602131, + "ap": 0.884791, + "ap_weighted": 0.884791 + }, + { + "accuracy": 0.530343, + "f1": 0.462094, + "f1_weighted": 0.602131, + "ap": 0.884791, + "ap_weighted": 0.884791 + } + ], + "main_score": 0.530343, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 5.272130489349365, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDBPAuditsLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDBPAuditsLegalBenchClassification.json new file mode 100644 index 0000000000..c279514b52 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDBPAuditsLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "SCDBPAuditsLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.596306, + "f1": 0.574392, + "f1_weighted": 0.610321, + "ap": 0.730992, + "ap_weighted": 0.730992, + "scores_per_experiment": [ + { + "accuracy": 0.596306, + "f1": 0.574392, + "f1_weighted": 0.610321, + "ap": 0.730992, + "ap_weighted": 0.730992 + }, + { + "accuracy": 0.596306, + "f1": 0.574392, + "f1_weighted": 0.610321, + "ap": 0.730992, + "ap_weighted": 0.730992 + }, + { + "accuracy": 0.596306, + "f1": 0.574392, + "f1_weighted": 0.610321, + "ap": 0.730992, + "ap_weighted": 0.730992 + }, + { + "accuracy": 0.596306, + "f1": 0.574392, + "f1_weighted": 0.610321, + "ap": 0.730992, + "ap_weighted": 0.730992 + }, + { + "accuracy": 0.596306, + "f1": 0.574392, + "f1_weighted": 0.610321, + "ap": 0.730992, + "ap_weighted": 0.730992 + }, + { + "accuracy": 0.596306, + "f1": 0.574392, + "f1_weighted": 0.610321, + "ap": 0.730992, + "ap_weighted": 0.730992 + }, + { + "accuracy": 0.596306, + "f1": 0.574392, + "f1_weighted": 0.610321, + "ap": 0.730992, + "ap_weighted": 0.730992 + }, + { + "accuracy": 0.596306, + "f1": 0.574392, + "f1_weighted": 0.610321, + "ap": 0.730992, + "ap_weighted": 0.730992 + }, + { + "accuracy": 0.596306, + "f1": 0.574392, + "f1_weighted": 0.610321, + "ap": 0.730992, + "ap_weighted": 0.730992 + }, + { + "accuracy": 0.596306, + "f1": 0.574392, + "f1_weighted": 0.610321, + "ap": 0.730992, + "ap_weighted": 0.730992 + } + ], + "main_score": 0.596306, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 5.397128105163574, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDBPCertificationLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDBPCertificationLegalBenchClassification.json new file mode 100644 index 0000000000..423e1ee2bc --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDBPCertificationLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "SCDBPCertificationLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.494709, + "f1": 0.47163, + "f1_weighted": 0.527136, + "ap": 0.759501, + "ap_weighted": 0.759501, + "scores_per_experiment": [ + { + "accuracy": 0.494709, + "f1": 0.47163, + "f1_weighted": 0.527136, + "ap": 0.759501, + "ap_weighted": 0.759501 + }, + { + "accuracy": 0.494709, + "f1": 0.47163, + "f1_weighted": 0.527136, + "ap": 0.759501, + "ap_weighted": 0.759501 + }, + { + "accuracy": 0.494709, + "f1": 0.47163, + "f1_weighted": 0.527136, + "ap": 0.759501, + "ap_weighted": 0.759501 + }, + { + "accuracy": 0.494709, + "f1": 0.47163, + "f1_weighted": 0.527136, + "ap": 0.759501, + "ap_weighted": 0.759501 + }, + { + "accuracy": 0.494709, + "f1": 0.47163, + "f1_weighted": 0.527136, + "ap": 0.759501, + "ap_weighted": 0.759501 + }, + { + "accuracy": 0.494709, + "f1": 0.47163, + "f1_weighted": 0.527136, + "ap": 0.759501, + "ap_weighted": 0.759501 + }, + { + "accuracy": 0.494709, + "f1": 0.47163, + "f1_weighted": 0.527136, + "ap": 0.759501, + "ap_weighted": 0.759501 + }, + { + "accuracy": 0.494709, + "f1": 0.47163, + "f1_weighted": 0.527136, + "ap": 0.759501, + "ap_weighted": 0.759501 + }, + { + "accuracy": 0.494709, + "f1": 0.47163, + "f1_weighted": 0.527136, + "ap": 0.759501, + "ap_weighted": 0.759501 + }, + { + "accuracy": 0.494709, + "f1": 0.47163, + "f1_weighted": 0.527136, + "ap": 0.759501, + "ap_weighted": 0.759501 + } + ], + "main_score": 0.494709, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 5.289391994476318, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDBPTrainingLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDBPTrainingLegalBenchClassification.json new file mode 100644 index 0000000000..f8f1c57d60 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDBPTrainingLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "SCDBPTrainingLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.522427, + "f1": 0.520169, + "f1_weighted": 0.5345, + "ap": 0.768913, + "ap_weighted": 0.768913, + "scores_per_experiment": [ + { + "accuracy": 0.522427, + "f1": 0.520169, + "f1_weighted": 0.5345, + "ap": 0.768913, + "ap_weighted": 0.768913 + }, + { + "accuracy": 0.522427, + "f1": 0.520169, + "f1_weighted": 0.5345, + "ap": 0.768913, + "ap_weighted": 0.768913 + }, + { + "accuracy": 0.522427, + "f1": 0.520169, + "f1_weighted": 0.5345, + "ap": 0.768913, + "ap_weighted": 0.768913 + }, + { + "accuracy": 0.522427, + "f1": 0.520169, + "f1_weighted": 0.5345, + "ap": 0.768913, + "ap_weighted": 0.768913 + }, + { + "accuracy": 0.522427, + "f1": 0.520169, + "f1_weighted": 0.5345, + "ap": 0.768913, + "ap_weighted": 0.768913 + }, + { + "accuracy": 0.522427, + "f1": 0.520169, + "f1_weighted": 0.5345, + "ap": 0.768913, + "ap_weighted": 0.768913 + }, + { + "accuracy": 0.522427, + "f1": 0.520169, + "f1_weighted": 0.5345, + "ap": 0.768913, + "ap_weighted": 0.768913 + }, + { + "accuracy": 0.522427, + "f1": 0.520169, + "f1_weighted": 0.5345, + "ap": 0.768913, + "ap_weighted": 0.768913 + }, + { + "accuracy": 0.522427, + "f1": 0.520169, + "f1_weighted": 0.5345, + "ap": 0.768913, + "ap_weighted": 0.768913 + }, + { + "accuracy": 0.522427, + "f1": 0.520169, + "f1_weighted": 0.5345, + "ap": 0.768913, + "ap_weighted": 0.768913 + } + ], + "main_score": 0.522427, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 5.335339307785034, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDBPVerificationLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDBPVerificationLegalBenchClassification.json new file mode 100644 index 0000000000..da03845c0b --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDBPVerificationLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "SCDBPVerificationLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.548813, + "f1": 0.540821, + "f1_weighted": 0.548333, + "ap": 0.583334, + "ap_weighted": 0.583334, + "scores_per_experiment": [ + { + "accuracy": 0.548813, + "f1": 0.540821, + "f1_weighted": 0.548333, + "ap": 0.583334, + "ap_weighted": 0.583334 + }, + { + "accuracy": 0.548813, + "f1": 0.540821, + "f1_weighted": 0.548333, + "ap": 0.583334, + "ap_weighted": 0.583334 + }, + { + "accuracy": 0.548813, + "f1": 0.540821, + "f1_weighted": 0.548333, + "ap": 0.583334, + "ap_weighted": 0.583334 + }, + { + "accuracy": 0.548813, + "f1": 0.540821, + "f1_weighted": 0.548333, + "ap": 0.583334, + "ap_weighted": 0.583334 + }, + { + "accuracy": 0.548813, + "f1": 0.540821, + "f1_weighted": 0.548333, + "ap": 0.583334, + "ap_weighted": 0.583334 + }, + { + "accuracy": 0.548813, + "f1": 0.540821, + "f1_weighted": 0.548333, + "ap": 0.583334, + "ap_weighted": 0.583334 + }, + { + "accuracy": 0.548813, + "f1": 0.540821, + "f1_weighted": 0.548333, + "ap": 0.583334, + "ap_weighted": 0.583334 + }, + { + "accuracy": 0.548813, + "f1": 0.540821, + "f1_weighted": 0.548333, + "ap": 0.583334, + "ap_weighted": 0.583334 + }, + { + "accuracy": 0.548813, + "f1": 0.540821, + "f1_weighted": 0.548333, + "ap": 0.583334, + "ap_weighted": 0.583334 + }, + { + "accuracy": 0.548813, + "f1": 0.540821, + "f1_weighted": 0.548333, + "ap": 0.583334, + "ap_weighted": 0.583334 + } + ], + "main_score": 0.548813, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 5.414256572723389, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDDAccountabilityLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDDAccountabilityLegalBenchClassification.json new file mode 100644 index 0000000000..4a054021d4 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDDAccountabilityLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "SCDDAccountabilityLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.484127, + "f1": 0.376581, + "f1_weighted": 0.606744, + "ap": 0.949498, + "ap_weighted": 0.949498, + "scores_per_experiment": [ + { + "accuracy": 0.484127, + "f1": 0.376581, + "f1_weighted": 0.606744, + "ap": 0.949498, + "ap_weighted": 0.949498 + }, + { + "accuracy": 0.484127, + "f1": 0.376581, + "f1_weighted": 0.606744, + "ap": 0.949498, + "ap_weighted": 0.949498 + }, + { + "accuracy": 0.484127, + "f1": 0.376581, + "f1_weighted": 0.606744, + "ap": 0.949498, + "ap_weighted": 0.949498 + }, + { + "accuracy": 0.484127, + "f1": 0.376581, + "f1_weighted": 0.606744, + "ap": 0.949498, + "ap_weighted": 0.949498 + }, + { + "accuracy": 0.484127, + "f1": 0.376581, + "f1_weighted": 0.606744, + "ap": 0.949498, + "ap_weighted": 0.949498 + }, + { + "accuracy": 0.484127, + "f1": 0.376581, + "f1_weighted": 0.606744, + "ap": 0.949498, + "ap_weighted": 0.949498 + }, + { + "accuracy": 0.484127, + "f1": 0.376581, + "f1_weighted": 0.606744, + "ap": 0.949498, + "ap_weighted": 0.949498 + }, + { + "accuracy": 0.484127, + "f1": 0.376581, + "f1_weighted": 0.606744, + "ap": 0.949498, + "ap_weighted": 0.949498 + }, + { + "accuracy": 0.484127, + "f1": 0.376581, + "f1_weighted": 0.606744, + "ap": 0.949498, + "ap_weighted": 0.949498 + }, + { + "accuracy": 0.484127, + "f1": 0.376581, + "f1_weighted": 0.606744, + "ap": 0.949498, + "ap_weighted": 0.949498 + } + ], + "main_score": 0.484127, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 5.230846166610718, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDDAuditsLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDDAuditsLegalBenchClassification.json new file mode 100644 index 0000000000..757aa4bf39 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDDAuditsLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "SCDDAuditsLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.630607, + "f1": 0.504853, + "f1_weighted": 0.708298, + "ap": 0.934949, + "ap_weighted": 0.934949, + "scores_per_experiment": [ + { + "accuracy": 0.630607, + "f1": 0.504853, + "f1_weighted": 0.708298, + "ap": 0.934949, + "ap_weighted": 0.934949 + }, + { + "accuracy": 0.630607, + "f1": 0.504853, + "f1_weighted": 0.708298, + "ap": 0.934949, + "ap_weighted": 0.934949 + }, + { + "accuracy": 0.630607, + "f1": 0.504853, + "f1_weighted": 0.708298, + "ap": 0.934949, + "ap_weighted": 0.934949 + }, + { + "accuracy": 0.630607, + "f1": 0.504853, + "f1_weighted": 0.708298, + "ap": 0.934949, + "ap_weighted": 0.934949 + }, + { + "accuracy": 0.630607, + "f1": 0.504853, + "f1_weighted": 0.708298, + "ap": 0.934949, + "ap_weighted": 0.934949 + }, + { + "accuracy": 0.630607, + "f1": 0.504853, + "f1_weighted": 0.708298, + "ap": 0.934949, + "ap_weighted": 0.934949 + }, + { + "accuracy": 0.630607, + "f1": 0.504853, + "f1_weighted": 0.708298, + "ap": 0.934949, + "ap_weighted": 0.934949 + }, + { + "accuracy": 0.630607, + "f1": 0.504853, + "f1_weighted": 0.708298, + "ap": 0.934949, + "ap_weighted": 0.934949 + }, + { + "accuracy": 0.630607, + "f1": 0.504853, + "f1_weighted": 0.708298, + "ap": 0.934949, + "ap_weighted": 0.934949 + }, + { + "accuracy": 0.630607, + "f1": 0.504853, + "f1_weighted": 0.708298, + "ap": 0.934949, + "ap_weighted": 0.934949 + } + ], + "main_score": 0.630607, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 5.370135545730591, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDDCertificationLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDDCertificationLegalBenchClassification.json new file mode 100644 index 0000000000..d89f34782e --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDDCertificationLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "SCDDCertificationLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.531746, + "f1": 0.427973, + "f1_weighted": 0.626495, + "ap": 0.915883, + "ap_weighted": 0.915883, + "scores_per_experiment": [ + { + "accuracy": 0.531746, + "f1": 0.427973, + "f1_weighted": 0.626495, + "ap": 0.915883, + "ap_weighted": 0.915883 + }, + { + "accuracy": 0.531746, + "f1": 0.427973, + "f1_weighted": 0.626495, + "ap": 0.915883, + "ap_weighted": 0.915883 + }, + { + "accuracy": 0.531746, + "f1": 0.427973, + "f1_weighted": 0.626495, + "ap": 0.915883, + "ap_weighted": 0.915883 + }, + { + "accuracy": 0.531746, + "f1": 0.427973, + "f1_weighted": 0.626495, + "ap": 0.915883, + "ap_weighted": 0.915883 + }, + { + "accuracy": 0.531746, + "f1": 0.427973, + "f1_weighted": 0.626495, + "ap": 0.915883, + "ap_weighted": 0.915883 + }, + { + "accuracy": 0.531746, + "f1": 0.427973, + "f1_weighted": 0.626495, + "ap": 0.915883, + "ap_weighted": 0.915883 + }, + { + "accuracy": 0.531746, + "f1": 0.427973, + "f1_weighted": 0.626495, + "ap": 0.915883, + "ap_weighted": 0.915883 + }, + { + "accuracy": 0.531746, + "f1": 0.427973, + "f1_weighted": 0.626495, + "ap": 0.915883, + "ap_weighted": 0.915883 + }, + { + "accuracy": 0.531746, + "f1": 0.427973, + "f1_weighted": 0.626495, + "ap": 0.915883, + "ap_weighted": 0.915883 + }, + { + "accuracy": 0.531746, + "f1": 0.427973, + "f1_weighted": 0.626495, + "ap": 0.915883, + "ap_weighted": 0.915883 + } + ], + "main_score": 0.531746, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 5.154891490936279, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDDTrainingLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDDTrainingLegalBenchClassification.json new file mode 100644 index 0000000000..319efbce7c --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDDTrainingLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "SCDDTrainingLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.459103, + "f1": 0.411299, + "f1_weighted": 0.533907, + "ap": 0.874616, + "ap_weighted": 0.874616, + "scores_per_experiment": [ + { + "accuracy": 0.459103, + "f1": 0.411299, + "f1_weighted": 0.533907, + "ap": 0.874616, + "ap_weighted": 0.874616 + }, + { + "accuracy": 0.459103, + "f1": 0.411299, + "f1_weighted": 0.533907, + "ap": 0.874616, + "ap_weighted": 0.874616 + }, + { + "accuracy": 0.459103, + "f1": 0.411299, + "f1_weighted": 0.533907, + "ap": 0.874616, + "ap_weighted": 0.874616 + }, + { + "accuracy": 0.459103, + "f1": 0.411299, + "f1_weighted": 0.533907, + "ap": 0.874616, + "ap_weighted": 0.874616 + }, + { + "accuracy": 0.459103, + "f1": 0.411299, + "f1_weighted": 0.533907, + "ap": 0.874616, + "ap_weighted": 0.874616 + }, + { + "accuracy": 0.459103, + "f1": 0.411299, + "f1_weighted": 0.533907, + "ap": 0.874616, + "ap_weighted": 0.874616 + }, + { + "accuracy": 0.459103, + "f1": 0.411299, + "f1_weighted": 0.533907, + "ap": 0.874616, + "ap_weighted": 0.874616 + }, + { + "accuracy": 0.459103, + "f1": 0.411299, + "f1_weighted": 0.533907, + "ap": 0.874616, + "ap_weighted": 0.874616 + }, + { + "accuracy": 0.459103, + "f1": 0.411299, + "f1_weighted": 0.533907, + "ap": 0.874616, + "ap_weighted": 0.874616 + }, + { + "accuracy": 0.459103, + "f1": 0.411299, + "f1_weighted": 0.533907, + "ap": 0.874616, + "ap_weighted": 0.874616 + } + ], + "main_score": 0.459103, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 5.425621271133423, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDDVerificationLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDDVerificationLegalBenchClassification.json new file mode 100644 index 0000000000..7115f5adb5 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SCDDVerificationLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "SCDDVerificationLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.577836, + "f1": 0.466723, + "f1_weighted": 0.584259, + "ap": 0.728746, + "ap_weighted": 0.728746, + "scores_per_experiment": [ + { + "accuracy": 0.577836, + "f1": 0.466723, + "f1_weighted": 0.584259, + "ap": 0.728746, + "ap_weighted": 0.728746 + }, + { + "accuracy": 0.577836, + "f1": 0.466723, + "f1_weighted": 0.584259, + "ap": 0.728746, + "ap_weighted": 0.728746 + }, + { + "accuracy": 0.577836, + "f1": 0.466723, + "f1_weighted": 0.584259, + "ap": 0.728746, + "ap_weighted": 0.728746 + }, + { + "accuracy": 0.577836, + "f1": 0.466723, + "f1_weighted": 0.584259, + "ap": 0.728746, + "ap_weighted": 0.728746 + }, + { + "accuracy": 0.577836, + "f1": 0.466723, + "f1_weighted": 0.584259, + "ap": 0.728746, + "ap_weighted": 0.728746 + }, + { + "accuracy": 0.577836, + "f1": 0.466723, + "f1_weighted": 0.584259, + "ap": 0.728746, + "ap_weighted": 0.728746 + }, + { + "accuracy": 0.577836, + "f1": 0.466723, + "f1_weighted": 0.584259, + "ap": 0.728746, + "ap_weighted": 0.728746 + }, + { + "accuracy": 0.577836, + "f1": 0.466723, + "f1_weighted": 0.584259, + "ap": 0.728746, + "ap_weighted": 0.728746 + }, + { + "accuracy": 0.577836, + "f1": 0.466723, + "f1_weighted": 0.584259, + "ap": 0.728746, + "ap_weighted": 0.728746 + }, + { + "accuracy": 0.577836, + "f1": 0.466723, + "f1_weighted": 0.584259, + "ap": 0.728746, + "ap_weighted": 0.728746 + } + ], + "main_score": 0.577836, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 5.335377931594849, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SCIDOCS.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SCIDOCS.json deleted file mode 100644 index 27a68c68f8..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/SCIDOCS.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "test": { - "evaluation_time": 175.55, - "map_at_1": 0.03328, - "map_at_10": 0.07955, - "map_at_100": 0.09321, - "map_at_1000": 0.09552, - "map_at_3": 0.05926, - "map_at_5": 0.06936, - "ndcg_at_1": 0.164, - "ndcg_at_10": 0.14, - "ndcg_at_100": 0.19835, - "ndcg_at_1000": 0.24525, - "ndcg_at_3": 0.13592, - "ndcg_at_5": 0.11823, - "precision_at_1": 0.164, - "precision_at_10": 0.0716, - "precision_at_100": 0.01557, - "precision_at_1000": 0.00269, - "precision_at_3": 0.12733, - "precision_at_5": 0.1038, - "recall_at_1": 0.03328, - "recall_at_10": 0.1452, - "recall_at_100": 0.31618, - "recall_at_1000": 0.54547, - "recall_at_3": 0.07752, - "recall_at_5": 0.10527 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "SCIDOCS", - "dataset_revision": "5c59ef3e437a0a9651c8fe6fde943e7dce59fba5" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SDSEyeProtectionClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SDSEyeProtectionClassification.json new file mode 100644 index 0000000000..7394036ae8 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SDSEyeProtectionClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "35cbe5ee544dd26e343238a333de4568e6f77819", + "task_name": "SDSEyeProtectionClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.65915, + "f1": 0.399279, + "f1_weighted": 0.787985, + "ap": 0.998098, + "ap_weighted": 0.998098, + "scores_per_experiment": [ + { + "accuracy": 0.699, + "f1": 0.414614, + "f1_weighted": 0.820588, + "ap": 0.997749, + "ap_weighted": 0.997749 + }, + { + "accuracy": 0.578, + "f1": 0.369561, + "f1_weighted": 0.730251, + "ap": 0.997944, + "ap_weighted": 0.997944 + }, + { + "accuracy": 0.59, + "f1": 0.373334, + "f1_weighted": 0.739971, + "ap": 0.997476, + "ap_weighted": 0.997476 + }, + { + "accuracy": 0.648, + "f1": 0.398562, + "f1_weighted": 0.783951, + "ap": 0.998618, + "ap_weighted": 0.998618 + }, + { + "accuracy": 0.7625, + "f1": 0.438746, + "f1_weighted": 0.862887, + "ap": 0.998406, + "ap_weighted": 0.998406 + }, + { + "accuracy": 0.7385, + "f1": 0.430333, + "f1_weighted": 0.847228, + "ap": 0.998346, + "ap_weighted": 0.998346 + }, + { + "accuracy": 0.537, + "f1": 0.353272, + "f1_weighted": 0.696254, + "ap": 0.99834, + "ap_weighted": 0.99834 + }, + { + "accuracy": 0.805, + "f1": 0.448504, + "f1_weighted": 0.88969, + "ap": 0.997516, + "ap_weighted": 0.997516 + }, + { + "accuracy": 0.5045, + "f1": 0.338892, + "f1_weighted": 0.668122, + "ap": 0.998259, + "ap_weighted": 0.998259 + }, + { + "accuracy": 0.729, + "f1": 0.426969, + "f1_weighted": 0.840909, + "ap": 0.998322, + "ap_weighted": 0.998322 + } + ], + "main_score": 0.65915, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 25.88741397857666, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SDSGlovesClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SDSGlovesClassification.json new file mode 100644 index 0000000000..c2d3147806 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SDSGlovesClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "c723236c5ec417d79512e6104aca9d2cd88168f6", + "task_name": "SDSGlovesClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.608, + "f1": 0.380725, + "f1_weighted": 0.74559, + "ap": 0.99688, + "ap_weighted": 0.99688, + "scores_per_experiment": [ + { + "accuracy": 0.751, + "f1": 0.438537, + "f1_weighted": 0.854038, + "ap": 0.997503, + "ap_weighted": 0.997503 + }, + { + "accuracy": 0.7345, + "f1": 0.430709, + "f1_weighted": 0.843249, + "ap": 0.996939, + "ap_weighted": 0.996939 + }, + { + "accuracy": 0.3955, + "f1": 0.28739, + "f1_weighted": 0.562731, + "ap": 0.996575, + "ap_weighted": 0.996575 + }, + { + "accuracy": 0.5155, + "f1": 0.34473, + "f1_weighted": 0.676569, + "ap": 0.996559, + "ap_weighted": 0.996559 + }, + { + "accuracy": 0.7455, + "f1": 0.434668, + "f1_weighted": 0.850508, + "ap": 0.996983, + "ap_weighted": 0.996983 + }, + { + "accuracy": 0.5665, + "f1": 0.365851, + "f1_weighted": 0.719706, + "ap": 0.996266, + "ap_weighted": 0.996266 + }, + { + "accuracy": 0.4895, + "f1": 0.334588, + "f1_weighted": 0.653081, + "ap": 0.99745, + "ap_weighted": 0.99745 + }, + { + "accuracy": 0.5925, + "f1": 0.377714, + "f1_weighted": 0.740382, + "ap": 0.996867, + "ap_weighted": 0.996867 + }, + { + "accuracy": 0.571, + "f1": 0.369836, + "f1_weighted": 0.72303, + "ap": 0.997279, + "ap_weighted": 0.997279 + }, + { + "accuracy": 0.7185, + "f1": 0.423226, + "f1_weighted": 0.832606, + "ap": 0.996378, + "ap_weighted": 0.996378 + } + ], + "main_score": 0.608, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 25.944142818450928, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SIB200Classification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SIB200Classification.json new file mode 100644 index 0000000000..4e6f14c49d --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SIB200Classification.json @@ -0,0 +1,36657 @@ +{ + "dataset_revision": "a74d7350ea12af010cfb1c21e34f1f81fd2e615b", + "task_name": "SIB200Classification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.399144, + "f1": 0.390204, + "f1_weighted": 0.406548, + "scores_per_experiment": [ + { + "accuracy": 0.456491, + "f1": 0.44729, + "f1_weighted": 0.46382 + }, + { + "accuracy": 0.365193, + "f1": 0.351775, + "f1_weighted": 0.366711 + }, + { + "accuracy": 0.39087, + "f1": 0.387146, + "f1_weighted": 0.397935 + }, + { + "accuracy": 0.388017, + "f1": 0.392003, + "f1_weighted": 0.407932 + }, + { + "accuracy": 0.399429, + "f1": 0.385026, + "f1_weighted": 0.402228 + }, + { + "accuracy": 0.338088, + "f1": 0.329529, + "f1_weighted": 0.346458 + }, + { + "accuracy": 0.425107, + "f1": 0.400643, + "f1_weighted": 0.426738 + }, + { + "accuracy": 0.410842, + "f1": 0.408673, + "f1_weighted": 0.423436 + }, + { + "accuracy": 0.402282, + "f1": 0.396422, + "f1_weighted": 0.409668 + }, + { + "accuracy": 0.415121, + "f1": 0.403531, + "f1_weighted": 0.420556 + } + ], + "main_score": 0.399144, + "hf_subset": "ace_Latn", + "languages": [ + "ace-Latn" + ] + }, + { + "accuracy": 0.214408, + "f1": 0.176034, + "f1_weighted": 0.168922, + "scores_per_experiment": [ + { + "accuracy": 0.269615, + "f1": 0.214037, + "f1_weighted": 0.20942 + }, + { + "accuracy": 0.172611, + "f1": 0.150256, + "f1_weighted": 0.142558 + }, + { + "accuracy": 0.166904, + "f1": 0.149782, + "f1_weighted": 0.131771 + }, + { + "accuracy": 0.256776, + "f1": 0.175972, + "f1_weighted": 0.173463 + }, + { + "accuracy": 0.233951, + "f1": 0.183439, + "f1_weighted": 0.179647 + }, + { + "accuracy": 0.186876, + "f1": 0.163467, + "f1_weighted": 0.175695 + }, + { + "accuracy": 0.231098, + "f1": 0.164808, + "f1_weighted": 0.170949 + }, + { + "accuracy": 0.225392, + "f1": 0.208983, + "f1_weighted": 0.195919 + }, + { + "accuracy": 0.184023, + "f1": 0.18263, + "f1_weighted": 0.148833 + }, + { + "accuracy": 0.216833, + "f1": 0.166966, + "f1_weighted": 0.16097 + } + ], + "main_score": 0.214408, + "hf_subset": "acm_Arab", + "languages": [ + "acm-Arab" + ] + }, + { + "accuracy": 0.209843, + "f1": 0.169879, + "f1_weighted": 0.16135, + "scores_per_experiment": [ + { + "accuracy": 0.259629, + "f1": 0.201564, + "f1_weighted": 0.196725 + }, + { + "accuracy": 0.21826, + "f1": 0.152138, + "f1_weighted": 0.157041 + }, + { + "accuracy": 0.154066, + "f1": 0.132702, + "f1_weighted": 0.105444 + }, + { + "accuracy": 0.245364, + "f1": 0.16721, + "f1_weighted": 0.161167 + }, + { + "accuracy": 0.221113, + "f1": 0.173579, + "f1_weighted": 0.174506 + }, + { + "accuracy": 0.165478, + "f1": 0.159386, + "f1_weighted": 0.153493 + }, + { + "accuracy": 0.223966, + "f1": 0.156935, + "f1_weighted": 0.164733 + }, + { + "accuracy": 0.21826, + "f1": 0.2053, + "f1_weighted": 0.189901 + }, + { + "accuracy": 0.179743, + "f1": 0.176498, + "f1_weighted": 0.142116 + }, + { + "accuracy": 0.212553, + "f1": 0.173482, + "f1_weighted": 0.16837 + } + ], + "main_score": 0.209843, + "hf_subset": "acq_Arab", + "languages": [ + "acq-Arab" + ] + }, + { + "accuracy": 0.213267, + "f1": 0.170367, + "f1_weighted": 0.167816, + "scores_per_experiment": [ + { + "accuracy": 0.263909, + "f1": 0.186498, + "f1_weighted": 0.193468 + }, + { + "accuracy": 0.223966, + "f1": 0.154822, + "f1_weighted": 0.158336 + }, + { + "accuracy": 0.156919, + "f1": 0.141133, + "f1_weighted": 0.120284 + }, + { + "accuracy": 0.252496, + "f1": 0.188983, + "f1_weighted": 0.189422 + }, + { + "accuracy": 0.21826, + "f1": 0.171733, + "f1_weighted": 0.174924 + }, + { + "accuracy": 0.195435, + "f1": 0.176574, + "f1_weighted": 0.191572 + }, + { + "accuracy": 0.241084, + "f1": 0.168866, + "f1_weighted": 0.18045 + }, + { + "accuracy": 0.205421, + "f1": 0.183534, + "f1_weighted": 0.172468 + }, + { + "accuracy": 0.171184, + "f1": 0.167448, + "f1_weighted": 0.13817 + }, + { + "accuracy": 0.203994, + "f1": 0.16408, + "f1_weighted": 0.159069 + } + ], + "main_score": 0.213267, + "hf_subset": "aeb_Arab", + "languages": [ + "aeb-Arab" + ] + }, + { + "accuracy": 0.402282, + "f1": 0.395707, + "f1_weighted": 0.401749, + "scores_per_experiment": [ + { + "accuracy": 0.433666, + "f1": 0.430057, + "f1_weighted": 0.444242 + }, + { + "accuracy": 0.400856, + "f1": 0.396155, + "f1_weighted": 0.402057 + }, + { + "accuracy": 0.383738, + "f1": 0.378704, + "f1_weighted": 0.383392 + }, + { + "accuracy": 0.429387, + "f1": 0.408368, + "f1_weighted": 0.420825 + }, + { + "accuracy": 0.386591, + "f1": 0.374785, + "f1_weighted": 0.389471 + }, + { + "accuracy": 0.375178, + "f1": 0.367327, + "f1_weighted": 0.368022 + }, + { + "accuracy": 0.389444, + "f1": 0.385795, + "f1_weighted": 0.37874 + }, + { + "accuracy": 0.403709, + "f1": 0.405467, + "f1_weighted": 0.406654 + }, + { + "accuracy": 0.412268, + "f1": 0.407, + "f1_weighted": 0.409754 + }, + { + "accuracy": 0.407989, + "f1": 0.403416, + "f1_weighted": 0.414332 + } + ], + "main_score": 0.402282, + "hf_subset": "afr_Latn", + "languages": [ + "afr-Latn" + ] + }, + { + "accuracy": 0.20214, + "f1": 0.161372, + "f1_weighted": 0.156098, + "scores_per_experiment": [ + { + "accuracy": 0.253923, + "f1": 0.191854, + "f1_weighted": 0.190789 + }, + { + "accuracy": 0.236805, + "f1": 0.141583, + "f1_weighted": 0.161653 + }, + { + "accuracy": 0.156919, + "f1": 0.132454, + "f1_weighted": 0.10336 + }, + { + "accuracy": 0.228245, + "f1": 0.145539, + "f1_weighted": 0.140686 + }, + { + "accuracy": 0.21826, + "f1": 0.184955, + "f1_weighted": 0.174066 + }, + { + "accuracy": 0.152639, + "f1": 0.149884, + "f1_weighted": 0.147564 + }, + { + "accuracy": 0.195435, + "f1": 0.166496, + "f1_weighted": 0.15862 + }, + { + "accuracy": 0.189729, + "f1": 0.167767, + "f1_weighted": 0.15422 + }, + { + "accuracy": 0.184023, + "f1": 0.159998, + "f1_weighted": 0.154222 + }, + { + "accuracy": 0.205421, + "f1": 0.173192, + "f1_weighted": 0.175802 + } + ], + "main_score": 0.20214, + "hf_subset": "ajp_Arab", + "languages": [ + "ajp-Arab" + ] + }, + { + "accuracy": 0.420542, + "f1": 0.417182, + "f1_weighted": 0.422547, + "scores_per_experiment": [ + { + "accuracy": 0.410842, + "f1": 0.409144, + "f1_weighted": 0.415529 + }, + { + "accuracy": 0.396576, + "f1": 0.394458, + "f1_weighted": 0.395095 + }, + { + "accuracy": 0.396576, + "f1": 0.398774, + "f1_weighted": 0.403934 + }, + { + "accuracy": 0.447932, + "f1": 0.438222, + "f1_weighted": 0.45119 + }, + { + "accuracy": 0.433666, + "f1": 0.428904, + "f1_weighted": 0.435082 + }, + { + "accuracy": 0.348074, + "f1": 0.350528, + "f1_weighted": 0.341541 + }, + { + "accuracy": 0.447932, + "f1": 0.445305, + "f1_weighted": 0.447884 + }, + { + "accuracy": 0.470756, + "f1": 0.459372, + "f1_weighted": 0.469524 + }, + { + "accuracy": 0.419401, + "f1": 0.413398, + "f1_weighted": 0.421929 + }, + { + "accuracy": 0.433666, + "f1": 0.433717, + "f1_weighted": 0.44376 + } + ], + "main_score": 0.420542, + "hf_subset": "aka_Latn", + "languages": [ + "aka-Latn" + ] + }, + { + "accuracy": 0.38602, + "f1": 0.379131, + "f1_weighted": 0.388655, + "scores_per_experiment": [ + { + "accuracy": 0.405136, + "f1": 0.398104, + "f1_weighted": 0.415236 + }, + { + "accuracy": 0.368046, + "f1": 0.35802, + "f1_weighted": 0.366708 + }, + { + "accuracy": 0.373752, + "f1": 0.371413, + "f1_weighted": 0.371551 + }, + { + "accuracy": 0.413695, + "f1": 0.403595, + "f1_weighted": 0.415097 + }, + { + "accuracy": 0.350927, + "f1": 0.349603, + "f1_weighted": 0.359774 + }, + { + "accuracy": 0.406562, + "f1": 0.405747, + "f1_weighted": 0.413539 + }, + { + "accuracy": 0.407989, + "f1": 0.395926, + "f1_weighted": 0.405266 + }, + { + "accuracy": 0.399429, + "f1": 0.394678, + "f1_weighted": 0.405435 + }, + { + "accuracy": 0.372325, + "f1": 0.362889, + "f1_weighted": 0.369109 + }, + { + "accuracy": 0.36234, + "f1": 0.351333, + "f1_weighted": 0.364831 + } + ], + "main_score": 0.38602, + "hf_subset": "als_Latn", + "languages": [ + "als-Latn" + ] + }, + { + "accuracy": 0.219401, + "f1": 0.173531, + "f1_weighted": 0.173229, + "scores_per_experiment": [ + { + "accuracy": 0.262482, + "f1": 0.188832, + "f1_weighted": 0.201843 + }, + { + "accuracy": 0.172611, + "f1": 0.148751, + "f1_weighted": 0.134682 + }, + { + "accuracy": 0.158345, + "f1": 0.15298, + "f1_weighted": 0.132085 + }, + { + "accuracy": 0.24679, + "f1": 0.188863, + "f1_weighted": 0.184102 + }, + { + "accuracy": 0.2097, + "f1": 0.1712, + "f1_weighted": 0.171148 + }, + { + "accuracy": 0.281027, + "f1": 0.169062, + "f1_weighted": 0.19375 + }, + { + "accuracy": 0.174037, + "f1": 0.134059, + "f1_weighted": 0.127595 + }, + { + "accuracy": 0.300999, + "f1": 0.246231, + "f1_weighted": 0.272419 + }, + { + "accuracy": 0.169757, + "f1": 0.144486, + "f1_weighted": 0.125935 + }, + { + "accuracy": 0.21826, + "f1": 0.190843, + "f1_weighted": 0.18873 + } + ], + "main_score": 0.219401, + "hf_subset": "amh_Ethi", + "languages": [ + "amh-Ethi" + ] + }, + { + "accuracy": 0.214408, + "f1": 0.180202, + "f1_weighted": 0.169475, + "scores_per_experiment": [ + { + "accuracy": 0.272468, + "f1": 0.199607, + "f1_weighted": 0.200789 + }, + { + "accuracy": 0.236805, + "f1": 0.176219, + "f1_weighted": 0.17484 + }, + { + "accuracy": 0.164051, + "f1": 0.138851, + "f1_weighted": 0.114337 + }, + { + "accuracy": 0.258203, + "f1": 0.191298, + "f1_weighted": 0.175407 + }, + { + "accuracy": 0.222539, + "f1": 0.190776, + "f1_weighted": 0.177232 + }, + { + "accuracy": 0.186876, + "f1": 0.177356, + "f1_weighted": 0.174204 + }, + { + "accuracy": 0.221113, + "f1": 0.205183, + "f1_weighted": 0.207423 + }, + { + "accuracy": 0.191155, + "f1": 0.186566, + "f1_weighted": 0.162026 + }, + { + "accuracy": 0.186876, + "f1": 0.161345, + "f1_weighted": 0.148715 + }, + { + "accuracy": 0.203994, + "f1": 0.174815, + "f1_weighted": 0.159776 + } + ], + "main_score": 0.214408, + "hf_subset": "apc_Arab", + "languages": [ + "apc-Arab" + ] + }, + { + "accuracy": 0.330813, + "f1": 0.319015, + "f1_weighted": 0.331815, + "scores_per_experiment": [ + { + "accuracy": 0.360913, + "f1": 0.347725, + "f1_weighted": 0.367447 + }, + { + "accuracy": 0.315264, + "f1": 0.304499, + "f1_weighted": 0.307546 + }, + { + "accuracy": 0.308131, + "f1": 0.305469, + "f1_weighted": 0.311698 + }, + { + "accuracy": 0.339515, + "f1": 0.324692, + "f1_weighted": 0.340897 + }, + { + "accuracy": 0.305278, + "f1": 0.301612, + "f1_weighted": 0.30711 + }, + { + "accuracy": 0.312411, + "f1": 0.301255, + "f1_weighted": 0.322416 + }, + { + "accuracy": 0.332382, + "f1": 0.307128, + "f1_weighted": 0.332803 + }, + { + "accuracy": 0.356633, + "f1": 0.345676, + "f1_weighted": 0.353724 + }, + { + "accuracy": 0.346648, + "f1": 0.323806, + "f1_weighted": 0.339364 + }, + { + "accuracy": 0.330956, + "f1": 0.328285, + "f1_weighted": 0.335147 + } + ], + "main_score": 0.330813, + "hf_subset": "arb_Latn", + "languages": [ + "arb-Latn" + ] + }, + { + "accuracy": 0.212981, + "f1": 0.167457, + "f1_weighted": 0.16433, + "scores_per_experiment": [ + { + "accuracy": 0.25535, + "f1": 0.184943, + "f1_weighted": 0.186742 + }, + { + "accuracy": 0.206847, + "f1": 0.1409, + "f1_weighted": 0.148263 + }, + { + "accuracy": 0.165478, + "f1": 0.148382, + "f1_weighted": 0.130217 + }, + { + "accuracy": 0.253923, + "f1": 0.181031, + "f1_weighted": 0.181696 + }, + { + "accuracy": 0.223966, + "f1": 0.174768, + "f1_weighted": 0.177114 + }, + { + "accuracy": 0.175464, + "f1": 0.154944, + "f1_weighted": 0.160731 + }, + { + "accuracy": 0.236805, + "f1": 0.159253, + "f1_weighted": 0.168409 + }, + { + "accuracy": 0.21826, + "f1": 0.192202, + "f1_weighted": 0.179751 + }, + { + "accuracy": 0.18117, + "f1": 0.174017, + "f1_weighted": 0.15218 + }, + { + "accuracy": 0.212553, + "f1": 0.164133, + "f1_weighted": 0.1582 + } + ], + "main_score": 0.212981, + "hf_subset": "ars_Arab", + "languages": [ + "ars-Arab" + ] + }, + { + "accuracy": 0.205278, + "f1": 0.164227, + "f1_weighted": 0.164213, + "scores_per_experiment": [ + { + "accuracy": 0.248217, + "f1": 0.178977, + "f1_weighted": 0.182477 + }, + { + "accuracy": 0.152639, + "f1": 0.108329, + "f1_weighted": 0.094643 + }, + { + "accuracy": 0.154066, + "f1": 0.141025, + "f1_weighted": 0.13112 + }, + { + "accuracy": 0.25535, + "f1": 0.186207, + "f1_weighted": 0.199226 + }, + { + "accuracy": 0.201141, + "f1": 0.191999, + "f1_weighted": 0.183465 + }, + { + "accuracy": 0.191155, + "f1": 0.1852, + "f1_weighted": 0.187373 + }, + { + "accuracy": 0.216833, + "f1": 0.148178, + "f1_weighted": 0.159147 + }, + { + "accuracy": 0.24679, + "f1": 0.190833, + "f1_weighted": 0.191108 + }, + { + "accuracy": 0.179743, + "f1": 0.146869, + "f1_weighted": 0.155227 + }, + { + "accuracy": 0.206847, + "f1": 0.164654, + "f1_weighted": 0.158347 + } + ], + "main_score": 0.205278, + "hf_subset": "ary_Arab", + "languages": [ + "ary-Arab" + ] + }, + { + "accuracy": 0.202568, + "f1": 0.161139, + "f1_weighted": 0.159513, + "scores_per_experiment": [ + { + "accuracy": 0.256776, + "f1": 0.191903, + "f1_weighted": 0.195665 + }, + { + "accuracy": 0.221113, + "f1": 0.14962, + "f1_weighted": 0.155524 + }, + { + "accuracy": 0.161198, + "f1": 0.143485, + "f1_weighted": 0.125978 + }, + { + "accuracy": 0.242511, + "f1": 0.153601, + "f1_weighted": 0.14955 + }, + { + "accuracy": 0.226819, + "f1": 0.198814, + "f1_weighted": 0.213768 + }, + { + "accuracy": 0.145506, + "f1": 0.139515, + "f1_weighted": 0.137201 + }, + { + "accuracy": 0.189729, + "f1": 0.151579, + "f1_weighted": 0.147708 + }, + { + "accuracy": 0.212553, + "f1": 0.18865, + "f1_weighted": 0.178055 + }, + { + "accuracy": 0.161198, + "f1": 0.137255, + "f1_weighted": 0.133793 + }, + { + "accuracy": 0.208274, + "f1": 0.156964, + "f1_weighted": 0.157884 + } + ], + "main_score": 0.202568, + "hf_subset": "arz_Arab", + "languages": [ + "arz-Arab" + ] + }, + { + "accuracy": 0.178745, + "f1": 0.152546, + "f1_weighted": 0.160064, + "scores_per_experiment": [ + { + "accuracy": 0.189729, + "f1": 0.156163, + "f1_weighted": 0.171927 + }, + { + "accuracy": 0.171184, + "f1": 0.16309, + "f1_weighted": 0.180457 + }, + { + "accuracy": 0.1398, + "f1": 0.128584, + "f1_weighted": 0.110987 + }, + { + "accuracy": 0.166904, + "f1": 0.13578, + "f1_weighted": 0.128696 + }, + { + "accuracy": 0.164051, + "f1": 0.150712, + "f1_weighted": 0.146527 + }, + { + "accuracy": 0.221113, + "f1": 0.186476, + "f1_weighted": 0.214623 + }, + { + "accuracy": 0.152639, + "f1": 0.139845, + "f1_weighted": 0.133235 + }, + { + "accuracy": 0.275321, + "f1": 0.206117, + "f1_weighted": 0.245159 + }, + { + "accuracy": 0.142653, + "f1": 0.131419, + "f1_weighted": 0.13033 + }, + { + "accuracy": 0.164051, + "f1": 0.127269, + "f1_weighted": 0.138697 + } + ], + "main_score": 0.178745, + "hf_subset": "asm_Beng", + "languages": [ + "asm-Beng" + ] + }, + { + "accuracy": 0.533381, + "f1": 0.515771, + "f1_weighted": 0.536396, + "scores_per_experiment": [ + { + "accuracy": 0.560628, + "f1": 0.536758, + "f1_weighted": 0.556348 + }, + { + "accuracy": 0.527817, + "f1": 0.511436, + "f1_weighted": 0.533525 + }, + { + "accuracy": 0.514979, + "f1": 0.498281, + "f1_weighted": 0.519058 + }, + { + "accuracy": 0.557775, + "f1": 0.53387, + "f1_weighted": 0.564738 + }, + { + "accuracy": 0.53923, + "f1": 0.523826, + "f1_weighted": 0.543716 + }, + { + "accuracy": 0.503566, + "f1": 0.486852, + "f1_weighted": 0.511997 + }, + { + "accuracy": 0.57204, + "f1": 0.553591, + "f1_weighted": 0.565705 + }, + { + "accuracy": 0.503566, + "f1": 0.494904, + "f1_weighted": 0.512864 + }, + { + "accuracy": 0.519258, + "f1": 0.50294, + "f1_weighted": 0.515134 + }, + { + "accuracy": 0.53495, + "f1": 0.515256, + "f1_weighted": 0.540871 + } + ], + "main_score": 0.533381, + "hf_subset": "ast_Latn", + "languages": [ + "ast-Latn" + ] + }, + { + "accuracy": 0.199144, + "f1": 0.167655, + "f1_weighted": 0.164581, + "scores_per_experiment": [ + { + "accuracy": 0.196862, + "f1": 0.160693, + "f1_weighted": 0.164339 + }, + { + "accuracy": 0.191155, + "f1": 0.138035, + "f1_weighted": 0.143843 + }, + { + "accuracy": 0.178317, + "f1": 0.177651, + "f1_weighted": 0.165832 + }, + { + "accuracy": 0.191155, + "f1": 0.160507, + "f1_weighted": 0.145054 + }, + { + "accuracy": 0.196862, + "f1": 0.178179, + "f1_weighted": 0.174076 + }, + { + "accuracy": 0.194009, + "f1": 0.178583, + "f1_weighted": 0.16745 + }, + { + "accuracy": 0.222539, + "f1": 0.150262, + "f1_weighted": 0.160133 + }, + { + "accuracy": 0.253923, + "f1": 0.227301, + "f1_weighted": 0.238139 + }, + { + "accuracy": 0.175464, + "f1": 0.159382, + "f1_weighted": 0.142525 + }, + { + "accuracy": 0.191155, + "f1": 0.145956, + "f1_weighted": 0.144417 + } + ], + "main_score": 0.199144, + "hf_subset": "awa_Deva", + "languages": [ + "awa-Deva" + ] + }, + { + "accuracy": 0.328103, + "f1": 0.322767, + "f1_weighted": 0.328031, + "scores_per_experiment": [ + { + "accuracy": 0.335235, + "f1": 0.339946, + "f1_weighted": 0.331905 + }, + { + "accuracy": 0.305278, + "f1": 0.294001, + "f1_weighted": 0.307702 + }, + { + "accuracy": 0.328103, + "f1": 0.317823, + "f1_weighted": 0.32953 + }, + { + "accuracy": 0.345221, + "f1": 0.341995, + "f1_weighted": 0.344125 + }, + { + "accuracy": 0.338088, + "f1": 0.336741, + "f1_weighted": 0.341015 + }, + { + "accuracy": 0.312411, + "f1": 0.301539, + "f1_weighted": 0.309135 + }, + { + "accuracy": 0.319544, + "f1": 0.30715, + "f1_weighted": 0.315839 + }, + { + "accuracy": 0.318117, + "f1": 0.324744, + "f1_weighted": 0.319271 + }, + { + "accuracy": 0.369472, + "f1": 0.358231, + "f1_weighted": 0.371623 + }, + { + "accuracy": 0.309558, + "f1": 0.305499, + "f1_weighted": 0.31017 + } + ], + "main_score": 0.328103, + "hf_subset": "ayr_Latn", + "languages": [ + "ayr-Latn" + ] + }, + { + "accuracy": 0.235235, + "f1": 0.162964, + "f1_weighted": 0.185352, + "scores_per_experiment": [ + { + "accuracy": 0.233951, + "f1": 0.165688, + "f1_weighted": 0.202989 + }, + { + "accuracy": 0.243937, + "f1": 0.132285, + "f1_weighted": 0.163106 + }, + { + "accuracy": 0.159772, + "f1": 0.162031, + "f1_weighted": 0.150323 + }, + { + "accuracy": 0.238231, + "f1": 0.136897, + "f1_weighted": 0.151232 + }, + { + "accuracy": 0.265335, + "f1": 0.214427, + "f1_weighted": 0.242996 + }, + { + "accuracy": 0.21826, + "f1": 0.171579, + "f1_weighted": 0.207237 + }, + { + "accuracy": 0.219686, + "f1": 0.141141, + "f1_weighted": 0.151831 + }, + { + "accuracy": 0.28816, + "f1": 0.207365, + "f1_weighted": 0.246999 + }, + { + "accuracy": 0.25535, + "f1": 0.134464, + "f1_weighted": 0.173577 + }, + { + "accuracy": 0.229672, + "f1": 0.163759, + "f1_weighted": 0.163226 + } + ], + "main_score": 0.235235, + "hf_subset": "azb_Arab", + "languages": [ + "azb-Arab" + ] + }, + { + "accuracy": 0.343224, + "f1": 0.335321, + "f1_weighted": 0.345645, + "scores_per_experiment": [ + { + "accuracy": 0.383738, + "f1": 0.372439, + "f1_weighted": 0.38695 + }, + { + "accuracy": 0.340942, + "f1": 0.339212, + "f1_weighted": 0.340407 + }, + { + "accuracy": 0.308131, + "f1": 0.307423, + "f1_weighted": 0.313117 + }, + { + "accuracy": 0.350927, + "f1": 0.345655, + "f1_weighted": 0.35731 + }, + { + "accuracy": 0.32525, + "f1": 0.309789, + "f1_weighted": 0.316723 + }, + { + "accuracy": 0.293866, + "f1": 0.289667, + "f1_weighted": 0.29833 + }, + { + "accuracy": 0.36234, + "f1": 0.353827, + "f1_weighted": 0.363887 + }, + { + "accuracy": 0.370899, + "f1": 0.358958, + "f1_weighted": 0.381599 + }, + { + "accuracy": 0.333809, + "f1": 0.323905, + "f1_weighted": 0.335606 + }, + { + "accuracy": 0.36234, + "f1": 0.352337, + "f1_weighted": 0.362526 + } + ], + "main_score": 0.343224, + "hf_subset": "azj_Latn", + "languages": [ + "azj-Latn" + ] + }, + { + "accuracy": 0.247218, + "f1": 0.232581, + "f1_weighted": 0.241774, + "scores_per_experiment": [ + { + "accuracy": 0.299572, + "f1": 0.260137, + "f1_weighted": 0.284268 + }, + { + "accuracy": 0.201141, + "f1": 0.182681, + "f1_weighted": 0.193227 + }, + { + "accuracy": 0.238231, + "f1": 0.246412, + "f1_weighted": 0.250438 + }, + { + "accuracy": 0.275321, + "f1": 0.266142, + "f1_weighted": 0.274383 + }, + { + "accuracy": 0.21826, + "f1": 0.212631, + "f1_weighted": 0.210412 + }, + { + "accuracy": 0.269615, + "f1": 0.263054, + "f1_weighted": 0.277154 + }, + { + "accuracy": 0.184023, + "f1": 0.174136, + "f1_weighted": 0.162688 + }, + { + "accuracy": 0.310984, + "f1": 0.265368, + "f1_weighted": 0.293198 + }, + { + "accuracy": 0.215407, + "f1": 0.210512, + "f1_weighted": 0.212316 + }, + { + "accuracy": 0.259629, + "f1": 0.24474, + "f1_weighted": 0.25966 + } + ], + "main_score": 0.247218, + "hf_subset": "bak_Cyrl", + "languages": [ + "bak-Cyrl" + ] + }, + { + "accuracy": 0.338802, + "f1": 0.330822, + "f1_weighted": 0.340841, + "scores_per_experiment": [ + { + "accuracy": 0.303852, + "f1": 0.29837, + "f1_weighted": 0.305068 + }, + { + "accuracy": 0.339515, + "f1": 0.32633, + "f1_weighted": 0.339559 + }, + { + "accuracy": 0.336662, + "f1": 0.327566, + "f1_weighted": 0.336254 + }, + { + "accuracy": 0.372325, + "f1": 0.360662, + "f1_weighted": 0.376979 + }, + { + "accuracy": 0.349501, + "f1": 0.342919, + "f1_weighted": 0.350901 + }, + { + "accuracy": 0.339515, + "f1": 0.331302, + "f1_weighted": 0.339922 + }, + { + "accuracy": 0.359486, + "f1": 0.346703, + "f1_weighted": 0.364195 + }, + { + "accuracy": 0.345221, + "f1": 0.337809, + "f1_weighted": 0.348136 + }, + { + "accuracy": 0.326676, + "f1": 0.317765, + "f1_weighted": 0.324547 + }, + { + "accuracy": 0.315264, + "f1": 0.318795, + "f1_weighted": 0.322852 + } + ], + "main_score": 0.338802, + "hf_subset": "bam_Latn", + "languages": [ + "bam-Latn" + ] + }, + { + "accuracy": 0.37418, + "f1": 0.367372, + "f1_weighted": 0.382369, + "scores_per_experiment": [ + { + "accuracy": 0.386591, + "f1": 0.384235, + "f1_weighted": 0.398523 + }, + { + "accuracy": 0.365193, + "f1": 0.353047, + "f1_weighted": 0.36875 + }, + { + "accuracy": 0.335235, + "f1": 0.330903, + "f1_weighted": 0.342212 + }, + { + "accuracy": 0.396576, + "f1": 0.381578, + "f1_weighted": 0.403224 + }, + { + "accuracy": 0.372325, + "f1": 0.367536, + "f1_weighted": 0.379003 + }, + { + "accuracy": 0.363766, + "f1": 0.35778, + "f1_weighted": 0.366757 + }, + { + "accuracy": 0.345221, + "f1": 0.337408, + "f1_weighted": 0.357585 + }, + { + "accuracy": 0.413695, + "f1": 0.402641, + "f1_weighted": 0.426743 + }, + { + "accuracy": 0.360913, + "f1": 0.363726, + "f1_weighted": 0.368778 + }, + { + "accuracy": 0.402282, + "f1": 0.394868, + "f1_weighted": 0.412113 + } + ], + "main_score": 0.37418, + "hf_subset": "ban_Latn", + "languages": [ + "ban-Latn" + ] + }, + { + "accuracy": 0.247218, + "f1": 0.231809, + "f1_weighted": 0.240134, + "scores_per_experiment": [ + { + "accuracy": 0.285307, + "f1": 0.23919, + "f1_weighted": 0.263231 + }, + { + "accuracy": 0.192582, + "f1": 0.169855, + "f1_weighted": 0.172411 + }, + { + "accuracy": 0.211127, + "f1": 0.220648, + "f1_weighted": 0.212631 + }, + { + "accuracy": 0.236805, + "f1": 0.23322, + "f1_weighted": 0.248321 + }, + { + "accuracy": 0.241084, + "f1": 0.23501, + "f1_weighted": 0.236782 + }, + { + "accuracy": 0.231098, + "f1": 0.22747, + "f1_weighted": 0.232226 + }, + { + "accuracy": 0.233951, + "f1": 0.216942, + "f1_weighted": 0.216483 + }, + { + "accuracy": 0.300999, + "f1": 0.277504, + "f1_weighted": 0.294614 + }, + { + "accuracy": 0.28816, + "f1": 0.277582, + "f1_weighted": 0.28253 + }, + { + "accuracy": 0.25107, + "f1": 0.220664, + "f1_weighted": 0.242111 + } + ], + "main_score": 0.247218, + "hf_subset": "bel_Cyrl", + "languages": [ + "bel-Cyrl" + ] + }, + { + "accuracy": 0.384023, + "f1": 0.379201, + "f1_weighted": 0.38753, + "scores_per_experiment": [ + { + "accuracy": 0.42368, + "f1": 0.409254, + "f1_weighted": 0.427307 + }, + { + "accuracy": 0.329529, + "f1": 0.332584, + "f1_weighted": 0.335126 + }, + { + "accuracy": 0.365193, + "f1": 0.356969, + "f1_weighted": 0.362823 + }, + { + "accuracy": 0.360913, + "f1": 0.360664, + "f1_weighted": 0.365335 + }, + { + "accuracy": 0.402282, + "f1": 0.391649, + "f1_weighted": 0.397047 + }, + { + "accuracy": 0.35378, + "f1": 0.341984, + "f1_weighted": 0.35111 + }, + { + "accuracy": 0.429387, + "f1": 0.414642, + "f1_weighted": 0.428291 + }, + { + "accuracy": 0.416548, + "f1": 0.416726, + "f1_weighted": 0.42764 + }, + { + "accuracy": 0.379458, + "f1": 0.378762, + "f1_weighted": 0.389778 + }, + { + "accuracy": 0.379458, + "f1": 0.388774, + "f1_weighted": 0.390841 + } + ], + "main_score": 0.384023, + "hf_subset": "bem_Latn", + "languages": [ + "bem-Latn" + ] + }, + { + "accuracy": 0.181598, + "f1": 0.151713, + "f1_weighted": 0.158542, + "scores_per_experiment": [ + { + "accuracy": 0.182596, + "f1": 0.155547, + "f1_weighted": 0.174027 + }, + { + "accuracy": 0.149786, + "f1": 0.12376, + "f1_weighted": 0.112519 + }, + { + "accuracy": 0.191155, + "f1": 0.144284, + "f1_weighted": 0.157178 + }, + { + "accuracy": 0.168331, + "f1": 0.149737, + "f1_weighted": 0.140223 + }, + { + "accuracy": 0.235378, + "f1": 0.189796, + "f1_weighted": 0.220923 + }, + { + "accuracy": 0.21826, + "f1": 0.150999, + "f1_weighted": 0.182868 + }, + { + "accuracy": 0.155492, + "f1": 0.149879, + "f1_weighted": 0.144386 + }, + { + "accuracy": 0.179743, + "f1": 0.159897, + "f1_weighted": 0.165658 + }, + { + "accuracy": 0.166904, + "f1": 0.158584, + "f1_weighted": 0.152834 + }, + { + "accuracy": 0.168331, + "f1": 0.134645, + "f1_weighted": 0.134803 + } + ], + "main_score": 0.181598, + "hf_subset": "ben_Beng", + "languages": [ + "ben-Beng" + ] + }, + { + "accuracy": 0.244508, + "f1": 0.213131, + "f1_weighted": 0.225626, + "scores_per_experiment": [ + { + "accuracy": 0.222539, + "f1": 0.193222, + "f1_weighted": 0.197019 + }, + { + "accuracy": 0.249643, + "f1": 0.216391, + "f1_weighted": 0.241477 + }, + { + "accuracy": 0.195435, + "f1": 0.202823, + "f1_weighted": 0.193026 + }, + { + "accuracy": 0.278174, + "f1": 0.240654, + "f1_weighted": 0.264386 + }, + { + "accuracy": 0.231098, + "f1": 0.21842, + "f1_weighted": 0.210168 + }, + { + "accuracy": 0.203994, + "f1": 0.195864, + "f1_weighted": 0.19299 + }, + { + "accuracy": 0.238231, + "f1": 0.200849, + "f1_weighted": 0.207919 + }, + { + "accuracy": 0.315264, + "f1": 0.23209, + "f1_weighted": 0.274833 + }, + { + "accuracy": 0.266762, + "f1": 0.22652, + "f1_weighted": 0.260733 + }, + { + "accuracy": 0.243937, + "f1": 0.204477, + "f1_weighted": 0.213714 + } + ], + "main_score": 0.244508, + "hf_subset": "bho_Deva", + "languages": [ + "bho-Deva" + ] + }, + { + "accuracy": 0.377889, + "f1": 0.371178, + "f1_weighted": 0.382221, + "scores_per_experiment": [ + { + "accuracy": 0.422254, + "f1": 0.41141, + "f1_weighted": 0.427189 + }, + { + "accuracy": 0.346648, + "f1": 0.343932, + "f1_weighted": 0.35218 + }, + { + "accuracy": 0.372325, + "f1": 0.370254, + "f1_weighted": 0.371764 + }, + { + "accuracy": 0.403709, + "f1": 0.395693, + "f1_weighted": 0.408801 + }, + { + "accuracy": 0.35806, + "f1": 0.351195, + "f1_weighted": 0.35955 + }, + { + "accuracy": 0.365193, + "f1": 0.364275, + "f1_weighted": 0.373928 + }, + { + "accuracy": 0.359486, + "f1": 0.348402, + "f1_weighted": 0.361706 + }, + { + "accuracy": 0.39515, + "f1": 0.380628, + "f1_weighted": 0.399756 + }, + { + "accuracy": 0.398003, + "f1": 0.391215, + "f1_weighted": 0.402715 + }, + { + "accuracy": 0.35806, + "f1": 0.354774, + "f1_weighted": 0.364623 + } + ], + "main_score": 0.377889, + "hf_subset": "bjn_Latn", + "languages": [ + "bjn-Latn" + ] + }, + { + "accuracy": 0.201997, + "f1": 0.15498, + "f1_weighted": 0.162047, + "scores_per_experiment": [ + { + "accuracy": 0.229672, + "f1": 0.179837, + "f1_weighted": 0.183491 + }, + { + "accuracy": 0.216833, + "f1": 0.149985, + "f1_weighted": 0.169127 + }, + { + "accuracy": 0.169757, + "f1": 0.150742, + "f1_weighted": 0.142973 + }, + { + "accuracy": 0.211127, + "f1": 0.157899, + "f1_weighted": 0.159491 + }, + { + "accuracy": 0.221113, + "f1": 0.159775, + "f1_weighted": 0.183775 + }, + { + "accuracy": 0.1398, + "f1": 0.101636, + "f1_weighted": 0.104547 + }, + { + "accuracy": 0.196862, + "f1": 0.156213, + "f1_weighted": 0.155973 + }, + { + "accuracy": 0.222539, + "f1": 0.205209, + "f1_weighted": 0.201565 + }, + { + "accuracy": 0.223966, + "f1": 0.132525, + "f1_weighted": 0.165076 + }, + { + "accuracy": 0.188302, + "f1": 0.155982, + "f1_weighted": 0.154453 + } + ], + "main_score": 0.201997, + "hf_subset": "bod_Tibt", + "languages": [ + "bod-Tibt" + ] + }, + { + "accuracy": 0.406134, + "f1": 0.394072, + "f1_weighted": 0.40861, + "scores_per_experiment": [ + { + "accuracy": 0.426534, + "f1": 0.420695, + "f1_weighted": 0.434015 + }, + { + "accuracy": 0.349501, + "f1": 0.341659, + "f1_weighted": 0.355985 + }, + { + "accuracy": 0.400856, + "f1": 0.386293, + "f1_weighted": 0.402108 + }, + { + "accuracy": 0.426534, + "f1": 0.41691, + "f1_weighted": 0.431548 + }, + { + "accuracy": 0.393723, + "f1": 0.381376, + "f1_weighted": 0.39768 + }, + { + "accuracy": 0.433666, + "f1": 0.426101, + "f1_weighted": 0.438949 + }, + { + "accuracy": 0.412268, + "f1": 0.394374, + "f1_weighted": 0.398321 + }, + { + "accuracy": 0.442225, + "f1": 0.427943, + "f1_weighted": 0.44937 + }, + { + "accuracy": 0.419401, + "f1": 0.392189, + "f1_weighted": 0.420881 + }, + { + "accuracy": 0.356633, + "f1": 0.353182, + "f1_weighted": 0.35724 + } + ], + "main_score": 0.406134, + "hf_subset": "bos_Latn", + "languages": [ + "bos-Latn" + ] + }, + { + "accuracy": 0.367047, + "f1": 0.360153, + "f1_weighted": 0.37003, + "scores_per_experiment": [ + { + "accuracy": 0.378031, + "f1": 0.378341, + "f1_weighted": 0.379014 + }, + { + "accuracy": 0.379458, + "f1": 0.371726, + "f1_weighted": 0.38519 + }, + { + "accuracy": 0.329529, + "f1": 0.328233, + "f1_weighted": 0.334471 + }, + { + "accuracy": 0.370899, + "f1": 0.356596, + "f1_weighted": 0.373336 + }, + { + "accuracy": 0.345221, + "f1": 0.328306, + "f1_weighted": 0.347236 + }, + { + "accuracy": 0.346648, + "f1": 0.3488, + "f1_weighted": 0.348354 + }, + { + "accuracy": 0.383738, + "f1": 0.374171, + "f1_weighted": 0.382289 + }, + { + "accuracy": 0.376605, + "f1": 0.37344, + "f1_weighted": 0.383301 + }, + { + "accuracy": 0.382311, + "f1": 0.370389, + "f1_weighted": 0.386769 + }, + { + "accuracy": 0.378031, + "f1": 0.371526, + "f1_weighted": 0.380339 + } + ], + "main_score": 0.367047, + "hf_subset": "bug_Latn", + "languages": [ + "bug-Latn" + ] + }, + { + "accuracy": 0.236519, + "f1": 0.229633, + "f1_weighted": 0.234567, + "scores_per_experiment": [ + { + "accuracy": 0.309558, + "f1": 0.270587, + "f1_weighted": 0.298888 + }, + { + "accuracy": 0.201141, + "f1": 0.183881, + "f1_weighted": 0.192842 + }, + { + "accuracy": 0.228245, + "f1": 0.231503, + "f1_weighted": 0.225678 + }, + { + "accuracy": 0.241084, + "f1": 0.237266, + "f1_weighted": 0.24834 + }, + { + "accuracy": 0.196862, + "f1": 0.197848, + "f1_weighted": 0.199162 + }, + { + "accuracy": 0.238231, + "f1": 0.232731, + "f1_weighted": 0.242131 + }, + { + "accuracy": 0.233951, + "f1": 0.216248, + "f1_weighted": 0.217332 + }, + { + "accuracy": 0.24679, + "f1": 0.248708, + "f1_weighted": 0.25348 + }, + { + "accuracy": 0.228245, + "f1": 0.23554, + "f1_weighted": 0.225814 + }, + { + "accuracy": 0.241084, + "f1": 0.24202, + "f1_weighted": 0.242007 + } + ], + "main_score": 0.236519, + "hf_subset": "bul_Cyrl", + "languages": [ + "bul-Cyrl" + ] + }, + { + "accuracy": 0.523966, + "f1": 0.510809, + "f1_weighted": 0.52861, + "scores_per_experiment": [ + { + "accuracy": 0.556348, + "f1": 0.540857, + "f1_weighted": 0.565612 + }, + { + "accuracy": 0.517832, + "f1": 0.501666, + "f1_weighted": 0.524869 + }, + { + "accuracy": 0.507846, + "f1": 0.506641, + "f1_weighted": 0.50726 + }, + { + "accuracy": 0.570613, + "f1": 0.54991, + "f1_weighted": 0.575902 + }, + { + "accuracy": 0.53495, + "f1": 0.527159, + "f1_weighted": 0.541051 + }, + { + "accuracy": 0.452211, + "f1": 0.443022, + "f1_weighted": 0.456356 + }, + { + "accuracy": 0.53923, + "f1": 0.521158, + "f1_weighted": 0.538608 + }, + { + "accuracy": 0.496434, + "f1": 0.490298, + "f1_weighted": 0.506104 + }, + { + "accuracy": 0.517832, + "f1": 0.494346, + "f1_weighted": 0.522573 + }, + { + "accuracy": 0.546362, + "f1": 0.53303, + "f1_weighted": 0.547767 + } + ], + "main_score": 0.523966, + "hf_subset": "cat_Latn", + "languages": [ + "cat-Latn" + ] + }, + { + "accuracy": 0.489158, + "f1": 0.474416, + "f1_weighted": 0.496752, + "scores_per_experiment": [ + { + "accuracy": 0.527817, + "f1": 0.513713, + "f1_weighted": 0.534864 + }, + { + "accuracy": 0.492154, + "f1": 0.474794, + "f1_weighted": 0.497985 + }, + { + "accuracy": 0.442225, + "f1": 0.431605, + "f1_weighted": 0.451054 + }, + { + "accuracy": 0.49786, + "f1": 0.476435, + "f1_weighted": 0.506327 + }, + { + "accuracy": 0.452211, + "f1": 0.435261, + "f1_weighted": 0.456673 + }, + { + "accuracy": 0.486448, + "f1": 0.466643, + "f1_weighted": 0.490522 + }, + { + "accuracy": 0.489301, + "f1": 0.474474, + "f1_weighted": 0.500816 + }, + { + "accuracy": 0.499287, + "f1": 0.491963, + "f1_weighted": 0.507992 + }, + { + "accuracy": 0.489301, + "f1": 0.474443, + "f1_weighted": 0.49456 + }, + { + "accuracy": 0.514979, + "f1": 0.504833, + "f1_weighted": 0.526724 + } + ], + "main_score": 0.489158, + "hf_subset": "ceb_Latn", + "languages": [ + "ceb-Latn" + ] + }, + { + "accuracy": 0.391013, + "f1": 0.380562, + "f1_weighted": 0.393626, + "scores_per_experiment": [ + { + "accuracy": 0.449358, + "f1": 0.433633, + "f1_weighted": 0.453285 + }, + { + "accuracy": 0.350927, + "f1": 0.342398, + "f1_weighted": 0.353968 + }, + { + "accuracy": 0.345221, + "f1": 0.33923, + "f1_weighted": 0.348252 + }, + { + "accuracy": 0.416548, + "f1": 0.398791, + "f1_weighted": 0.422828 + }, + { + "accuracy": 0.406562, + "f1": 0.398877, + "f1_weighted": 0.4175 + }, + { + "accuracy": 0.396576, + "f1": 0.388079, + "f1_weighted": 0.403821 + }, + { + "accuracy": 0.413695, + "f1": 0.403238, + "f1_weighted": 0.413131 + }, + { + "accuracy": 0.420827, + "f1": 0.401857, + "f1_weighted": 0.426495 + }, + { + "accuracy": 0.349501, + "f1": 0.341128, + "f1_weighted": 0.337924 + }, + { + "accuracy": 0.360913, + "f1": 0.358389, + "f1_weighted": 0.359056 + } + ], + "main_score": 0.391013, + "hf_subset": "ces_Latn", + "languages": [ + "ces-Latn" + ] + }, + { + "accuracy": 0.33224, + "f1": 0.326637, + "f1_weighted": 0.334344, + "scores_per_experiment": [ + { + "accuracy": 0.35806, + "f1": 0.351305, + "f1_weighted": 0.359694 + }, + { + "accuracy": 0.313837, + "f1": 0.311695, + "f1_weighted": 0.313482 + }, + { + "accuracy": 0.292439, + "f1": 0.281749, + "f1_weighted": 0.298142 + }, + { + "accuracy": 0.306705, + "f1": 0.303887, + "f1_weighted": 0.304855 + }, + { + "accuracy": 0.315264, + "f1": 0.307119, + "f1_weighted": 0.312708 + }, + { + "accuracy": 0.343795, + "f1": 0.33652, + "f1_weighted": 0.352155 + }, + { + "accuracy": 0.318117, + "f1": 0.316019, + "f1_weighted": 0.320171 + }, + { + "accuracy": 0.35806, + "f1": 0.355978, + "f1_weighted": 0.359512 + }, + { + "accuracy": 0.389444, + "f1": 0.37649, + "f1_weighted": 0.39403 + }, + { + "accuracy": 0.326676, + "f1": 0.325606, + "f1_weighted": 0.328694 + } + ], + "main_score": 0.33224, + "hf_subset": "cjk_Latn", + "languages": [ + "cjk-Latn" + ] + }, + { + "accuracy": 0.185307, + "f1": 0.155642, + "f1_weighted": 0.163588, + "scores_per_experiment": [ + { + "accuracy": 0.199715, + "f1": 0.150872, + "f1_weighted": 0.163762 + }, + { + "accuracy": 0.164051, + "f1": 0.153673, + "f1_weighted": 0.15666 + }, + { + "accuracy": 0.158345, + "f1": 0.134599, + "f1_weighted": 0.139835 + }, + { + "accuracy": 0.175464, + "f1": 0.153837, + "f1_weighted": 0.168293 + }, + { + "accuracy": 0.235378, + "f1": 0.197846, + "f1_weighted": 0.223436 + }, + { + "accuracy": 0.164051, + "f1": 0.150223, + "f1_weighted": 0.159226 + }, + { + "accuracy": 0.146933, + "f1": 0.125616, + "f1_weighted": 0.116367 + }, + { + "accuracy": 0.266762, + "f1": 0.195379, + "f1_weighted": 0.226784 + }, + { + "accuracy": 0.148359, + "f1": 0.146556, + "f1_weighted": 0.125417 + }, + { + "accuracy": 0.194009, + "f1": 0.147817, + "f1_weighted": 0.156095 + } + ], + "main_score": 0.185307, + "hf_subset": "ckb_Arab", + "languages": [ + "ckb-Arab" + ] + }, + { + "accuracy": 0.394864, + "f1": 0.391335, + "f1_weighted": 0.395703, + "scores_per_experiment": [ + { + "accuracy": 0.413695, + "f1": 0.411623, + "f1_weighted": 0.421102 + }, + { + "accuracy": 0.366619, + "f1": 0.374191, + "f1_weighted": 0.364831 + }, + { + "accuracy": 0.332382, + "f1": 0.331669, + "f1_weighted": 0.324665 + }, + { + "accuracy": 0.400856, + "f1": 0.385283, + "f1_weighted": 0.399265 + }, + { + "accuracy": 0.392297, + "f1": 0.392165, + "f1_weighted": 0.390351 + }, + { + "accuracy": 0.400856, + "f1": 0.400065, + "f1_weighted": 0.404669 + }, + { + "accuracy": 0.447932, + "f1": 0.437052, + "f1_weighted": 0.446978 + }, + { + "accuracy": 0.422254, + "f1": 0.420549, + "f1_weighted": 0.428784 + }, + { + "accuracy": 0.370899, + "f1": 0.36883, + "f1_weighted": 0.371231 + }, + { + "accuracy": 0.400856, + "f1": 0.391918, + "f1_weighted": 0.405152 + } + ], + "main_score": 0.394864, + "hf_subset": "crh_Latn", + "languages": [ + "crh-Latn" + ] + }, + { + "accuracy": 0.32368, + "f1": 0.320744, + "f1_weighted": 0.32415, + "scores_per_experiment": [ + { + "accuracy": 0.365193, + "f1": 0.350295, + "f1_weighted": 0.367055 + }, + { + "accuracy": 0.32097, + "f1": 0.318584, + "f1_weighted": 0.321487 + }, + { + "accuracy": 0.262482, + "f1": 0.262344, + "f1_weighted": 0.253764 + }, + { + "accuracy": 0.336662, + "f1": 0.33312, + "f1_weighted": 0.339592 + }, + { + "accuracy": 0.300999, + "f1": 0.297603, + "f1_weighted": 0.300108 + }, + { + "accuracy": 0.310984, + "f1": 0.313537, + "f1_weighted": 0.311554 + }, + { + "accuracy": 0.352354, + "f1": 0.348862, + "f1_weighted": 0.358151 + }, + { + "accuracy": 0.310984, + "f1": 0.321407, + "f1_weighted": 0.320923 + }, + { + "accuracy": 0.338088, + "f1": 0.331873, + "f1_weighted": 0.334456 + }, + { + "accuracy": 0.338088, + "f1": 0.329816, + "f1_weighted": 0.334407 + } + ], + "main_score": 0.32368, + "hf_subset": "cym_Latn", + "languages": [ + "cym-Latn" + ] + }, + { + "accuracy": 0.488873, + "f1": 0.475577, + "f1_weighted": 0.491376, + "scores_per_experiment": [ + { + "accuracy": 0.513552, + "f1": 0.508949, + "f1_weighted": 0.515811 + }, + { + "accuracy": 0.482168, + "f1": 0.456213, + "f1_weighted": 0.485455 + }, + { + "accuracy": 0.450785, + "f1": 0.441742, + "f1_weighted": 0.449681 + }, + { + "accuracy": 0.492154, + "f1": 0.477383, + "f1_weighted": 0.493184 + }, + { + "accuracy": 0.483595, + "f1": 0.469592, + "f1_weighted": 0.488348 + }, + { + "accuracy": 0.499287, + "f1": 0.477717, + "f1_weighted": 0.499718 + }, + { + "accuracy": 0.485021, + "f1": 0.477959, + "f1_weighted": 0.485293 + }, + { + "accuracy": 0.503566, + "f1": 0.489402, + "f1_weighted": 0.510118 + }, + { + "accuracy": 0.487874, + "f1": 0.472809, + "f1_weighted": 0.487644 + }, + { + "accuracy": 0.490728, + "f1": 0.484007, + "f1_weighted": 0.498504 + } + ], + "main_score": 0.488873, + "hf_subset": "dan_Latn", + "languages": [ + "dan-Latn" + ] + }, + { + "accuracy": 0.640371, + "f1": 0.623366, + "f1_weighted": 0.64042, + "scores_per_experiment": [ + { + "accuracy": 0.681883, + "f1": 0.658315, + "f1_weighted": 0.686175 + }, + { + "accuracy": 0.620542, + "f1": 0.609004, + "f1_weighted": 0.621402 + }, + { + "accuracy": 0.590585, + "f1": 0.585933, + "f1_weighted": 0.587462 + }, + { + "accuracy": 0.644793, + "f1": 0.624203, + "f1_weighted": 0.643842 + }, + { + "accuracy": 0.657632, + "f1": 0.649592, + "f1_weighted": 0.656789 + }, + { + "accuracy": 0.657632, + "f1": 0.635164, + "f1_weighted": 0.658302 + }, + { + "accuracy": 0.661912, + "f1": 0.643161, + "f1_weighted": 0.658406 + }, + { + "accuracy": 0.623395, + "f1": 0.602901, + "f1_weighted": 0.625971 + }, + { + "accuracy": 0.631954, + "f1": 0.605023, + "f1_weighted": 0.632521 + }, + { + "accuracy": 0.633381, + "f1": 0.620367, + "f1_weighted": 0.633335 + } + ], + "main_score": 0.640371, + "hf_subset": "deu_Latn", + "languages": [ + "deu-Latn" + ] + }, + { + "accuracy": 0.380884, + "f1": 0.367126, + "f1_weighted": 0.388352, + "scores_per_experiment": [ + { + "accuracy": 0.36234, + "f1": 0.357425, + "f1_weighted": 0.377683 + }, + { + "accuracy": 0.332382, + "f1": 0.325124, + "f1_weighted": 0.331965 + }, + { + "accuracy": 0.356633, + "f1": 0.346057, + "f1_weighted": 0.360473 + }, + { + "accuracy": 0.389444, + "f1": 0.369048, + "f1_weighted": 0.396432 + }, + { + "accuracy": 0.399429, + "f1": 0.374768, + "f1_weighted": 0.403843 + }, + { + "accuracy": 0.373752, + "f1": 0.366314, + "f1_weighted": 0.378311 + }, + { + "accuracy": 0.386591, + "f1": 0.372929, + "f1_weighted": 0.390175 + }, + { + "accuracy": 0.410842, + "f1": 0.396989, + "f1_weighted": 0.422921 + }, + { + "accuracy": 0.409415, + "f1": 0.395388, + "f1_weighted": 0.421077 + }, + { + "accuracy": 0.388017, + "f1": 0.367221, + "f1_weighted": 0.400642 + } + ], + "main_score": 0.380884, + "hf_subset": "dik_Latn", + "languages": [ + "dik-Latn" + ] + }, + { + "accuracy": 0.327817, + "f1": 0.318419, + "f1_weighted": 0.331431, + "scores_per_experiment": [ + { + "accuracy": 0.342368, + "f1": 0.330252, + "f1_weighted": 0.352334 + }, + { + "accuracy": 0.300999, + "f1": 0.296454, + "f1_weighted": 0.298474 + }, + { + "accuracy": 0.329529, + "f1": 0.318184, + "f1_weighted": 0.332238 + }, + { + "accuracy": 0.313837, + "f1": 0.304164, + "f1_weighted": 0.31299 + }, + { + "accuracy": 0.342368, + "f1": 0.329919, + "f1_weighted": 0.349604 + }, + { + "accuracy": 0.329529, + "f1": 0.3182, + "f1_weighted": 0.341781 + }, + { + "accuracy": 0.319544, + "f1": 0.30997, + "f1_weighted": 0.32143 + }, + { + "accuracy": 0.348074, + "f1": 0.334646, + "f1_weighted": 0.351202 + }, + { + "accuracy": 0.355207, + "f1": 0.345081, + "f1_weighted": 0.359268 + }, + { + "accuracy": 0.296719, + "f1": 0.297323, + "f1_weighted": 0.294992 + } + ], + "main_score": 0.327817, + "hf_subset": "dyu_Latn", + "languages": [ + "dyu-Latn" + ] + }, + { + "accuracy": 0.156348, + "f1": 0.098479, + "f1_weighted": 0.108724, + "scores_per_experiment": [ + { + "accuracy": 0.154066, + "f1": 0.103319, + "f1_weighted": 0.108999 + }, + { + "accuracy": 0.216833, + "f1": 0.092781, + "f1_weighted": 0.13857 + }, + { + "accuracy": 0.116976, + "f1": 0.082799, + "f1_weighted": 0.075744 + }, + { + "accuracy": 0.14408, + "f1": 0.098051, + "f1_weighted": 0.098183 + }, + { + "accuracy": 0.168331, + "f1": 0.116758, + "f1_weighted": 0.137842 + }, + { + "accuracy": 0.158345, + "f1": 0.095577, + "f1_weighted": 0.090832 + }, + { + "accuracy": 0.1398, + "f1": 0.094297, + "f1_weighted": 0.095045 + }, + { + "accuracy": 0.148359, + "f1": 0.10149, + "f1_weighted": 0.106507 + }, + { + "accuracy": 0.156919, + "f1": 0.106273, + "f1_weighted": 0.116987 + }, + { + "accuracy": 0.159772, + "f1": 0.093441, + "f1_weighted": 0.118527 + } + ], + "main_score": 0.156348, + "hf_subset": "dzo_Tibt", + "languages": [ + "dzo-Tibt" + ] + }, + { + "accuracy": 0.206562, + "f1": 0.181426, + "f1_weighted": 0.177952, + "scores_per_experiment": [ + { + "accuracy": 0.236805, + "f1": 0.203607, + "f1_weighted": 0.20128 + }, + { + "accuracy": 0.172611, + "f1": 0.157833, + "f1_weighted": 0.150411 + }, + { + "accuracy": 0.166904, + "f1": 0.156704, + "f1_weighted": 0.13185 + }, + { + "accuracy": 0.226819, + "f1": 0.195569, + "f1_weighted": 0.208801 + }, + { + "accuracy": 0.174037, + "f1": 0.166042, + "f1_weighted": 0.137965 + }, + { + "accuracy": 0.169757, + "f1": 0.166291, + "f1_weighted": 0.169935 + }, + { + "accuracy": 0.25107, + "f1": 0.18308, + "f1_weighted": 0.184151 + }, + { + "accuracy": 0.242511, + "f1": 0.234787, + "f1_weighted": 0.244768 + }, + { + "accuracy": 0.215407, + "f1": 0.174179, + "f1_weighted": 0.177411 + }, + { + "accuracy": 0.2097, + "f1": 0.176169, + "f1_weighted": 0.172943 + } + ], + "main_score": 0.206562, + "hf_subset": "ell_Grek", + "languages": [ + "ell-Grek" + ] + }, + { + "accuracy": 0.690442, + "f1": 0.67484, + "f1_weighted": 0.689979, + "scores_per_experiment": [ + { + "accuracy": 0.699001, + "f1": 0.687911, + "f1_weighted": 0.702883 + }, + { + "accuracy": 0.666191, + "f1": 0.648453, + "f1_weighted": 0.665553 + }, + { + "accuracy": 0.657632, + "f1": 0.643121, + "f1_weighted": 0.656356 + }, + { + "accuracy": 0.727532, + "f1": 0.700232, + "f1_weighted": 0.725436 + }, + { + "accuracy": 0.718973, + "f1": 0.709117, + "f1_weighted": 0.719533 + }, + { + "accuracy": 0.676177, + "f1": 0.66584, + "f1_weighted": 0.677767 + }, + { + "accuracy": 0.703281, + "f1": 0.694881, + "f1_weighted": 0.699839 + }, + { + "accuracy": 0.654779, + "f1": 0.63362, + "f1_weighted": 0.654189 + }, + { + "accuracy": 0.669044, + "f1": 0.654905, + "f1_weighted": 0.668418 + }, + { + "accuracy": 0.731812, + "f1": 0.710317, + "f1_weighted": 0.72982 + } + ], + "main_score": 0.690442, + "hf_subset": "eng_Latn", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.481027, + "f1": 0.469959, + "f1_weighted": 0.483434, + "scores_per_experiment": [ + { + "accuracy": 0.547789, + "f1": 0.534482, + "f1_weighted": 0.553888 + }, + { + "accuracy": 0.46933, + "f1": 0.4557, + "f1_weighted": 0.475261 + }, + { + "accuracy": 0.450785, + "f1": 0.445693, + "f1_weighted": 0.450441 + }, + { + "accuracy": 0.522111, + "f1": 0.509829, + "f1_weighted": 0.526609 + }, + { + "accuracy": 0.463623, + "f1": 0.45662, + "f1_weighted": 0.465911 + }, + { + "accuracy": 0.43224, + "f1": 0.435022, + "f1_weighted": 0.43861 + }, + { + "accuracy": 0.470756, + "f1": 0.457757, + "f1_weighted": 0.465498 + }, + { + "accuracy": 0.490728, + "f1": 0.476665, + "f1_weighted": 0.492916 + }, + { + "accuracy": 0.475036, + "f1": 0.454615, + "f1_weighted": 0.474006 + }, + { + "accuracy": 0.487874, + "f1": 0.473204, + "f1_weighted": 0.491196 + } + ], + "main_score": 0.481027, + "hf_subset": "epo_Latn", + "languages": [ + "epo-Latn" + ] + }, + { + "accuracy": 0.343081, + "f1": 0.340605, + "f1_weighted": 0.345877, + "scores_per_experiment": [ + { + "accuracy": 0.369472, + "f1": 0.369826, + "f1_weighted": 0.372706 + }, + { + "accuracy": 0.319544, + "f1": 0.318895, + "f1_weighted": 0.320372 + }, + { + "accuracy": 0.330956, + "f1": 0.326179, + "f1_weighted": 0.327866 + }, + { + "accuracy": 0.355207, + "f1": 0.348545, + "f1_weighted": 0.363914 + }, + { + "accuracy": 0.355207, + "f1": 0.359249, + "f1_weighted": 0.362371 + }, + { + "accuracy": 0.31669, + "f1": 0.3139, + "f1_weighted": 0.314636 + }, + { + "accuracy": 0.339515, + "f1": 0.34141, + "f1_weighted": 0.3385 + }, + { + "accuracy": 0.365193, + "f1": 0.35741, + "f1_weighted": 0.372217 + }, + { + "accuracy": 0.322397, + "f1": 0.320414, + "f1_weighted": 0.324809 + }, + { + "accuracy": 0.356633, + "f1": 0.350219, + "f1_weighted": 0.361376 + } + ], + "main_score": 0.343081, + "hf_subset": "est_Latn", + "languages": [ + "est-Latn" + ] + }, + { + "accuracy": 0.382454, + "f1": 0.373537, + "f1_weighted": 0.385116, + "scores_per_experiment": [ + { + "accuracy": 0.425107, + "f1": 0.407695, + "f1_weighted": 0.428092 + }, + { + "accuracy": 0.39087, + "f1": 0.378157, + "f1_weighted": 0.394103 + }, + { + "accuracy": 0.36234, + "f1": 0.362302, + "f1_weighted": 0.363588 + }, + { + "accuracy": 0.403709, + "f1": 0.391074, + "f1_weighted": 0.406987 + }, + { + "accuracy": 0.375178, + "f1": 0.372409, + "f1_weighted": 0.375468 + }, + { + "accuracy": 0.340942, + "f1": 0.34522, + "f1_weighted": 0.34906 + }, + { + "accuracy": 0.372325, + "f1": 0.360209, + "f1_weighted": 0.376526 + }, + { + "accuracy": 0.400856, + "f1": 0.390776, + "f1_weighted": 0.405803 + }, + { + "accuracy": 0.366619, + "f1": 0.355104, + "f1_weighted": 0.361 + }, + { + "accuracy": 0.386591, + "f1": 0.372421, + "f1_weighted": 0.390535 + } + ], + "main_score": 0.382454, + "hf_subset": "eus_Latn", + "languages": [ + "eus-Latn" + ] + }, + { + "accuracy": 0.353923, + "f1": 0.350631, + "f1_weighted": 0.35412, + "scores_per_experiment": [ + { + "accuracy": 0.375178, + "f1": 0.373874, + "f1_weighted": 0.378236 + }, + { + "accuracy": 0.315264, + "f1": 0.318611, + "f1_weighted": 0.304675 + }, + { + "accuracy": 0.330956, + "f1": 0.325404, + "f1_weighted": 0.327786 + }, + { + "accuracy": 0.360913, + "f1": 0.355406, + "f1_weighted": 0.371738 + }, + { + "accuracy": 0.329529, + "f1": 0.331046, + "f1_weighted": 0.322239 + }, + { + "accuracy": 0.342368, + "f1": 0.343072, + "f1_weighted": 0.34293 + }, + { + "accuracy": 0.355207, + "f1": 0.331603, + "f1_weighted": 0.35432 + }, + { + "accuracy": 0.388017, + "f1": 0.38609, + "f1_weighted": 0.389256 + }, + { + "accuracy": 0.363766, + "f1": 0.356812, + "f1_weighted": 0.368044 + }, + { + "accuracy": 0.378031, + "f1": 0.384388, + "f1_weighted": 0.381971 + } + ], + "main_score": 0.353923, + "hf_subset": "ewe_Latn", + "languages": [ + "ewe-Latn" + ] + }, + { + "accuracy": 0.383452, + "f1": 0.368339, + "f1_weighted": 0.386053, + "scores_per_experiment": [ + { + "accuracy": 0.416548, + "f1": 0.397124, + "f1_weighted": 0.415123 + }, + { + "accuracy": 0.323823, + "f1": 0.315093, + "f1_weighted": 0.328536 + }, + { + "accuracy": 0.352354, + "f1": 0.340535, + "f1_weighted": 0.343795 + }, + { + "accuracy": 0.422254, + "f1": 0.412725, + "f1_weighted": 0.424191 + }, + { + "accuracy": 0.380884, + "f1": 0.356772, + "f1_weighted": 0.382758 + }, + { + "accuracy": 0.378031, + "f1": 0.364349, + "f1_weighted": 0.381039 + }, + { + "accuracy": 0.422254, + "f1": 0.391816, + "f1_weighted": 0.428738 + }, + { + "accuracy": 0.447932, + "f1": 0.422924, + "f1_weighted": 0.448166 + }, + { + "accuracy": 0.323823, + "f1": 0.326105, + "f1_weighted": 0.334939 + }, + { + "accuracy": 0.366619, + "f1": 0.355944, + "f1_weighted": 0.37324 + } + ], + "main_score": 0.383452, + "hf_subset": "fao_Latn", + "languages": [ + "fao-Latn" + ] + }, + { + "accuracy": 0.41826, + "f1": 0.401279, + "f1_weighted": 0.416969, + "scores_per_experiment": [ + { + "accuracy": 0.450785, + "f1": 0.434188, + "f1_weighted": 0.454336 + }, + { + "accuracy": 0.369472, + "f1": 0.357337, + "f1_weighted": 0.366585 + }, + { + "accuracy": 0.370899, + "f1": 0.362794, + "f1_weighted": 0.365443 + }, + { + "accuracy": 0.457917, + "f1": 0.432806, + "f1_weighted": 0.465326 + }, + { + "accuracy": 0.440799, + "f1": 0.42218, + "f1_weighted": 0.431633 + }, + { + "accuracy": 0.400856, + "f1": 0.386454, + "f1_weighted": 0.402698 + }, + { + "accuracy": 0.413695, + "f1": 0.392571, + "f1_weighted": 0.417401 + }, + { + "accuracy": 0.403709, + "f1": 0.38831, + "f1_weighted": 0.397759 + }, + { + "accuracy": 0.42368, + "f1": 0.404021, + "f1_weighted": 0.415427 + }, + { + "accuracy": 0.450785, + "f1": 0.432127, + "f1_weighted": 0.453083 + } + ], + "main_score": 0.41826, + "hf_subset": "fij_Latn", + "languages": [ + "fij-Latn" + ] + }, + { + "accuracy": 0.360628, + "f1": 0.352367, + "f1_weighted": 0.362425, + "scores_per_experiment": [ + { + "accuracy": 0.39087, + "f1": 0.377525, + "f1_weighted": 0.38991 + }, + { + "accuracy": 0.352354, + "f1": 0.346264, + "f1_weighted": 0.357755 + }, + { + "accuracy": 0.366619, + "f1": 0.35997, + "f1_weighted": 0.365451 + }, + { + "accuracy": 0.373752, + "f1": 0.363865, + "f1_weighted": 0.378859 + }, + { + "accuracy": 0.332382, + "f1": 0.330254, + "f1_weighted": 0.342298 + }, + { + "accuracy": 0.346648, + "f1": 0.338245, + "f1_weighted": 0.341339 + }, + { + "accuracy": 0.350927, + "f1": 0.338944, + "f1_weighted": 0.351139 + }, + { + "accuracy": 0.370899, + "f1": 0.357366, + "f1_weighted": 0.37764 + }, + { + "accuracy": 0.389444, + "f1": 0.382025, + "f1_weighted": 0.388965 + }, + { + "accuracy": 0.332382, + "f1": 0.329212, + "f1_weighted": 0.330897 + } + ], + "main_score": 0.360628, + "hf_subset": "fin_Latn", + "languages": [ + "fin-Latn" + ] + }, + { + "accuracy": 0.366762, + "f1": 0.355106, + "f1_weighted": 0.366577, + "scores_per_experiment": [ + { + "accuracy": 0.376605, + "f1": 0.362506, + "f1_weighted": 0.376099 + }, + { + "accuracy": 0.340942, + "f1": 0.334945, + "f1_weighted": 0.344566 + }, + { + "accuracy": 0.383738, + "f1": 0.368724, + "f1_weighted": 0.38589 + }, + { + "accuracy": 0.382311, + "f1": 0.363513, + "f1_weighted": 0.384644 + }, + { + "accuracy": 0.343795, + "f1": 0.32837, + "f1_weighted": 0.335379 + }, + { + "accuracy": 0.342368, + "f1": 0.331773, + "f1_weighted": 0.343573 + }, + { + "accuracy": 0.383738, + "f1": 0.370642, + "f1_weighted": 0.385405 + }, + { + "accuracy": 0.39515, + "f1": 0.389502, + "f1_weighted": 0.394722 + }, + { + "accuracy": 0.363766, + "f1": 0.358426, + "f1_weighted": 0.361559 + }, + { + "accuracy": 0.355207, + "f1": 0.342655, + "f1_weighted": 0.353927 + } + ], + "main_score": 0.366762, + "hf_subset": "fon_Latn", + "languages": [ + "fon-Latn" + ] + }, + { + "accuracy": 0.652496, + "f1": 0.638953, + "f1_weighted": 0.652589, + "scores_per_experiment": [ + { + "accuracy": 0.664765, + "f1": 0.649889, + "f1_weighted": 0.667418 + }, + { + "accuracy": 0.644793, + "f1": 0.624195, + "f1_weighted": 0.645469 + }, + { + "accuracy": 0.580599, + "f1": 0.575337, + "f1_weighted": 0.574649 + }, + { + "accuracy": 0.671897, + "f1": 0.650524, + "f1_weighted": 0.670782 + }, + { + "accuracy": 0.669044, + "f1": 0.664835, + "f1_weighted": 0.673863 + }, + { + "accuracy": 0.654779, + "f1": 0.641288, + "f1_weighted": 0.657363 + }, + { + "accuracy": 0.650499, + "f1": 0.639894, + "f1_weighted": 0.647926 + }, + { + "accuracy": 0.639087, + "f1": 0.625591, + "f1_weighted": 0.640159 + }, + { + "accuracy": 0.677603, + "f1": 0.661777, + "f1_weighted": 0.67672 + }, + { + "accuracy": 0.671897, + "f1": 0.656203, + "f1_weighted": 0.671544 + } + ], + "main_score": 0.652496, + "hf_subset": "fra_Latn", + "languages": [ + "fra-Latn" + ] + }, + { + "accuracy": 0.496434, + "f1": 0.480361, + "f1_weighted": 0.501252, + "scores_per_experiment": [ + { + "accuracy": 0.542083, + "f1": 0.522908, + "f1_weighted": 0.546472 + }, + { + "accuracy": 0.53067, + "f1": 0.497748, + "f1_weighted": 0.532838 + }, + { + "accuracy": 0.447932, + "f1": 0.441147, + "f1_weighted": 0.451224 + }, + { + "accuracy": 0.53923, + "f1": 0.520889, + "f1_weighted": 0.547618 + }, + { + "accuracy": 0.503566, + "f1": 0.48756, + "f1_weighted": 0.51339 + }, + { + "accuracy": 0.46933, + "f1": 0.463973, + "f1_weighted": 0.465618 + }, + { + "accuracy": 0.503566, + "f1": 0.489934, + "f1_weighted": 0.508166 + }, + { + "accuracy": 0.486448, + "f1": 0.471096, + "f1_weighted": 0.489108 + }, + { + "accuracy": 0.46933, + "f1": 0.450536, + "f1_weighted": 0.472193 + }, + { + "accuracy": 0.472183, + "f1": 0.457817, + "f1_weighted": 0.485889 + } + ], + "main_score": 0.496434, + "hf_subset": "fur_Latn", + "languages": [ + "fur-Latn" + ] + }, + { + "accuracy": 0.349073, + "f1": 0.340639, + "f1_weighted": 0.352544, + "scores_per_experiment": [ + { + "accuracy": 0.365193, + "f1": 0.350175, + "f1_weighted": 0.364711 + }, + { + "accuracy": 0.356633, + "f1": 0.34842, + "f1_weighted": 0.357948 + }, + { + "accuracy": 0.315264, + "f1": 0.309619, + "f1_weighted": 0.319529 + }, + { + "accuracy": 0.366619, + "f1": 0.357829, + "f1_weighted": 0.374916 + }, + { + "accuracy": 0.346648, + "f1": 0.334045, + "f1_weighted": 0.348929 + }, + { + "accuracy": 0.31669, + "f1": 0.323792, + "f1_weighted": 0.322719 + }, + { + "accuracy": 0.356633, + "f1": 0.34896, + "f1_weighted": 0.362104 + }, + { + "accuracy": 0.378031, + "f1": 0.372809, + "f1_weighted": 0.380036 + }, + { + "accuracy": 0.342368, + "f1": 0.332161, + "f1_weighted": 0.336495 + }, + { + "accuracy": 0.346648, + "f1": 0.32858, + "f1_weighted": 0.358056 + } + ], + "main_score": 0.349073, + "hf_subset": "fuv_Latn", + "languages": [ + "fuv-Latn" + ] + }, + { + "accuracy": 0.29786, + "f1": 0.293738, + "f1_weighted": 0.299701, + "scores_per_experiment": [ + { + "accuracy": 0.305278, + "f1": 0.302026, + "f1_weighted": 0.305085 + }, + { + "accuracy": 0.266762, + "f1": 0.257969, + "f1_weighted": 0.272626 + }, + { + "accuracy": 0.282454, + "f1": 0.27864, + "f1_weighted": 0.28082 + }, + { + "accuracy": 0.315264, + "f1": 0.306062, + "f1_weighted": 0.318093 + }, + { + "accuracy": 0.313837, + "f1": 0.305164, + "f1_weighted": 0.315188 + }, + { + "accuracy": 0.278174, + "f1": 0.277748, + "f1_weighted": 0.277597 + }, + { + "accuracy": 0.338088, + "f1": 0.328791, + "f1_weighted": 0.338448 + }, + { + "accuracy": 0.291013, + "f1": 0.292962, + "f1_weighted": 0.285806 + }, + { + "accuracy": 0.286733, + "f1": 0.28916, + "f1_weighted": 0.290128 + }, + { + "accuracy": 0.300999, + "f1": 0.298855, + "f1_weighted": 0.31322 + } + ], + "main_score": 0.29786, + "hf_subset": "gaz_Latn", + "languages": [ + "gaz-Latn" + ] + }, + { + "accuracy": 0.307846, + "f1": 0.303257, + "f1_weighted": 0.30729, + "scores_per_experiment": [ + { + "accuracy": 0.332382, + "f1": 0.331829, + "f1_weighted": 0.33749 + }, + { + "accuracy": 0.28816, + "f1": 0.283881, + "f1_weighted": 0.289221 + }, + { + "accuracy": 0.282454, + "f1": 0.283286, + "f1_weighted": 0.276672 + }, + { + "accuracy": 0.313837, + "f1": 0.300616, + "f1_weighted": 0.31409 + }, + { + "accuracy": 0.308131, + "f1": 0.301525, + "f1_weighted": 0.30848 + }, + { + "accuracy": 0.313837, + "f1": 0.309046, + "f1_weighted": 0.315138 + }, + { + "accuracy": 0.313837, + "f1": 0.300342, + "f1_weighted": 0.309801 + }, + { + "accuracy": 0.319544, + "f1": 0.316434, + "f1_weighted": 0.318728 + }, + { + "accuracy": 0.306705, + "f1": 0.303859, + "f1_weighted": 0.306608 + }, + { + "accuracy": 0.299572, + "f1": 0.301755, + "f1_weighted": 0.296669 + } + ], + "main_score": 0.307846, + "hf_subset": "gla_Latn", + "languages": [ + "gla-Latn" + ] + }, + { + "accuracy": 0.315407, + "f1": 0.316223, + "f1_weighted": 0.31278, + "scores_per_experiment": [ + { + "accuracy": 0.342368, + "f1": 0.343088, + "f1_weighted": 0.343193 + }, + { + "accuracy": 0.292439, + "f1": 0.294117, + "f1_weighted": 0.293674 + }, + { + "accuracy": 0.286733, + "f1": 0.289067, + "f1_weighted": 0.274612 + }, + { + "accuracy": 0.342368, + "f1": 0.353721, + "f1_weighted": 0.335913 + }, + { + "accuracy": 0.306705, + "f1": 0.312527, + "f1_weighted": 0.308281 + }, + { + "accuracy": 0.28388, + "f1": 0.282497, + "f1_weighted": 0.269783 + }, + { + "accuracy": 0.309558, + "f1": 0.30656, + "f1_weighted": 0.307328 + }, + { + "accuracy": 0.355207, + "f1": 0.351507, + "f1_weighted": 0.354589 + }, + { + "accuracy": 0.319544, + "f1": 0.312367, + "f1_weighted": 0.323987 + }, + { + "accuracy": 0.315264, + "f1": 0.316777, + "f1_weighted": 0.316442 + } + ], + "main_score": 0.315407, + "hf_subset": "gle_Latn", + "languages": [ + "gle-Latn" + ] + }, + { + "accuracy": 0.551641, + "f1": 0.53306, + "f1_weighted": 0.554307, + "scores_per_experiment": [ + { + "accuracy": 0.60485, + "f1": 0.583967, + "f1_weighted": 0.609312 + }, + { + "accuracy": 0.557775, + "f1": 0.536677, + "f1_weighted": 0.56011 + }, + { + "accuracy": 0.540656, + "f1": 0.525969, + "f1_weighted": 0.539788 + }, + { + "accuracy": 0.574893, + "f1": 0.559489, + "f1_weighted": 0.579151 + }, + { + "accuracy": 0.536377, + "f1": 0.514311, + "f1_weighted": 0.541077 + }, + { + "accuracy": 0.544936, + "f1": 0.530273, + "f1_weighted": 0.550896 + }, + { + "accuracy": 0.536377, + "f1": 0.521564, + "f1_weighted": 0.528405 + }, + { + "accuracy": 0.549215, + "f1": 0.527579, + "f1_weighted": 0.557889 + }, + { + "accuracy": 0.543509, + "f1": 0.523124, + "f1_weighted": 0.544337 + }, + { + "accuracy": 0.527817, + "f1": 0.50765, + "f1_weighted": 0.532108 + } + ], + "main_score": 0.551641, + "hf_subset": "glg_Latn", + "languages": [ + "glg-Latn" + ] + }, + { + "accuracy": 0.420399, + "f1": 0.410801, + "f1_weighted": 0.426219, + "scores_per_experiment": [ + { + "accuracy": 0.457917, + "f1": 0.442561, + "f1_weighted": 0.461379 + }, + { + "accuracy": 0.425107, + "f1": 0.406303, + "f1_weighted": 0.431569 + }, + { + "accuracy": 0.406562, + "f1": 0.403109, + "f1_weighted": 0.414766 + }, + { + "accuracy": 0.409415, + "f1": 0.397859, + "f1_weighted": 0.410493 + }, + { + "accuracy": 0.462197, + "f1": 0.445573, + "f1_weighted": 0.463555 + }, + { + "accuracy": 0.39515, + "f1": 0.388312, + "f1_weighted": 0.40236 + }, + { + "accuracy": 0.417974, + "f1": 0.406728, + "f1_weighted": 0.422598 + }, + { + "accuracy": 0.415121, + "f1": 0.416011, + "f1_weighted": 0.428032 + }, + { + "accuracy": 0.450785, + "f1": 0.440415, + "f1_weighted": 0.457871 + }, + { + "accuracy": 0.363766, + "f1": 0.361133, + "f1_weighted": 0.369567 + } + ], + "main_score": 0.420399, + "hf_subset": "grn_Latn", + "languages": [ + "grn-Latn" + ] + }, + { + "accuracy": 0.197432, + "f1": 0.174203, + "f1_weighted": 0.169478, + "scores_per_experiment": [ + { + "accuracy": 0.238231, + "f1": 0.190868, + "f1_weighted": 0.192882 + }, + { + "accuracy": 0.184023, + "f1": 0.161874, + "f1_weighted": 0.155749 + }, + { + "accuracy": 0.199715, + "f1": 0.196778, + "f1_weighted": 0.197261 + }, + { + "accuracy": 0.168331, + "f1": 0.147836, + "f1_weighted": 0.133823 + }, + { + "accuracy": 0.195435, + "f1": 0.184757, + "f1_weighted": 0.174515 + }, + { + "accuracy": 0.154066, + "f1": 0.155732, + "f1_weighted": 0.136983 + }, + { + "accuracy": 0.188302, + "f1": 0.174916, + "f1_weighted": 0.165722 + }, + { + "accuracy": 0.196862, + "f1": 0.167797, + "f1_weighted": 0.167442 + }, + { + "accuracy": 0.25535, + "f1": 0.191814, + "f1_weighted": 0.205737 + }, + { + "accuracy": 0.194009, + "f1": 0.169654, + "f1_weighted": 0.164668 + } + ], + "main_score": 0.197432, + "hf_subset": "guj_Gujr", + "languages": [ + "guj-Gujr" + ] + }, + { + "accuracy": 0.35806, + "f1": 0.352382, + "f1_weighted": 0.362892, + "scores_per_experiment": [ + { + "accuracy": 0.419401, + "f1": 0.397613, + "f1_weighted": 0.423445 + }, + { + "accuracy": 0.31669, + "f1": 0.315211, + "f1_weighted": 0.322564 + }, + { + "accuracy": 0.319544, + "f1": 0.319441, + "f1_weighted": 0.327963 + }, + { + "accuracy": 0.359486, + "f1": 0.348269, + "f1_weighted": 0.367375 + }, + { + "accuracy": 0.332382, + "f1": 0.327835, + "f1_weighted": 0.330156 + }, + { + "accuracy": 0.363766, + "f1": 0.35944, + "f1_weighted": 0.36454 + }, + { + "accuracy": 0.345221, + "f1": 0.350027, + "f1_weighted": 0.353401 + }, + { + "accuracy": 0.380884, + "f1": 0.380612, + "f1_weighted": 0.387276 + }, + { + "accuracy": 0.39515, + "f1": 0.379129, + "f1_weighted": 0.397979 + }, + { + "accuracy": 0.348074, + "f1": 0.346243, + "f1_weighted": 0.354219 + } + ], + "main_score": 0.35806, + "hf_subset": "hat_Latn", + "languages": [ + "hat-Latn" + ] + }, + { + "accuracy": 0.353495, + "f1": 0.346048, + "f1_weighted": 0.35686, + "scores_per_experiment": [ + { + "accuracy": 0.375178, + "f1": 0.37292, + "f1_weighted": 0.383966 + }, + { + "accuracy": 0.313837, + "f1": 0.314073, + "f1_weighted": 0.316661 + }, + { + "accuracy": 0.346648, + "f1": 0.340435, + "f1_weighted": 0.342497 + }, + { + "accuracy": 0.35378, + "f1": 0.343474, + "f1_weighted": 0.35861 + }, + { + "accuracy": 0.328103, + "f1": 0.309395, + "f1_weighted": 0.32939 + }, + { + "accuracy": 0.312411, + "f1": 0.312947, + "f1_weighted": 0.306336 + }, + { + "accuracy": 0.419401, + "f1": 0.399832, + "f1_weighted": 0.42703 + }, + { + "accuracy": 0.35806, + "f1": 0.353758, + "f1_weighted": 0.361938 + }, + { + "accuracy": 0.339515, + "f1": 0.331202, + "f1_weighted": 0.349967 + }, + { + "accuracy": 0.388017, + "f1": 0.382438, + "f1_weighted": 0.392204 + } + ], + "main_score": 0.353495, + "hf_subset": "hau_Latn", + "languages": [ + "hau-Latn" + ] + }, + { + "accuracy": 0.211698, + "f1": 0.177972, + "f1_weighted": 0.179888, + "scores_per_experiment": [ + { + "accuracy": 0.266762, + "f1": 0.198961, + "f1_weighted": 0.199498 + }, + { + "accuracy": 0.226819, + "f1": 0.180489, + "f1_weighted": 0.204103 + }, + { + "accuracy": 0.165478, + "f1": 0.168142, + "f1_weighted": 0.159543 + }, + { + "accuracy": 0.192582, + "f1": 0.157434, + "f1_weighted": 0.151572 + }, + { + "accuracy": 0.235378, + "f1": 0.16663, + "f1_weighted": 0.169069 + }, + { + "accuracy": 0.145506, + "f1": 0.143922, + "f1_weighted": 0.127664 + }, + { + "accuracy": 0.239658, + "f1": 0.183864, + "f1_weighted": 0.202571 + }, + { + "accuracy": 0.25107, + "f1": 0.231153, + "f1_weighted": 0.24494 + }, + { + "accuracy": 0.18117, + "f1": 0.18254, + "f1_weighted": 0.177998 + }, + { + "accuracy": 0.212553, + "f1": 0.166582, + "f1_weighted": 0.161923 + } + ], + "main_score": 0.211698, + "hf_subset": "heb_Hebr", + "languages": [ + "heb-Hebr" + ] + }, + { + "accuracy": 0.191726, + "f1": 0.171553, + "f1_weighted": 0.175511, + "scores_per_experiment": [ + { + "accuracy": 0.222539, + "f1": 0.181845, + "f1_weighted": 0.189085 + }, + { + "accuracy": 0.179743, + "f1": 0.147038, + "f1_weighted": 0.145576 + }, + { + "accuracy": 0.201141, + "f1": 0.180806, + "f1_weighted": 0.183328 + }, + { + "accuracy": 0.174037, + "f1": 0.16062, + "f1_weighted": 0.144321 + }, + { + "accuracy": 0.184023, + "f1": 0.169799, + "f1_weighted": 0.167152 + }, + { + "accuracy": 0.156919, + "f1": 0.148509, + "f1_weighted": 0.150548 + }, + { + "accuracy": 0.198288, + "f1": 0.168715, + "f1_weighted": 0.190263 + }, + { + "accuracy": 0.18117, + "f1": 0.18549, + "f1_weighted": 0.181566 + }, + { + "accuracy": 0.21398, + "f1": 0.189636, + "f1_weighted": 0.206789 + }, + { + "accuracy": 0.205421, + "f1": 0.183072, + "f1_weighted": 0.196485 + } + ], + "main_score": 0.191726, + "hf_subset": "hin_Deva", + "languages": [ + "hin-Deva" + ] + }, + { + "accuracy": 0.199715, + "f1": 0.162425, + "f1_weighted": 0.166297, + "scores_per_experiment": [ + { + "accuracy": 0.179743, + "f1": 0.150173, + "f1_weighted": 0.151569 + }, + { + "accuracy": 0.178317, + "f1": 0.148234, + "f1_weighted": 0.147584 + }, + { + "accuracy": 0.195435, + "f1": 0.186295, + "f1_weighted": 0.190978 + }, + { + "accuracy": 0.17689, + "f1": 0.160104, + "f1_weighted": 0.143844 + }, + { + "accuracy": 0.212553, + "f1": 0.169991, + "f1_weighted": 0.173885 + }, + { + "accuracy": 0.216833, + "f1": 0.17417, + "f1_weighted": 0.200001 + }, + { + "accuracy": 0.21398, + "f1": 0.160437, + "f1_weighted": 0.168041 + }, + { + "accuracy": 0.21826, + "f1": 0.166895, + "f1_weighted": 0.17071 + }, + { + "accuracy": 0.199715, + "f1": 0.155615, + "f1_weighted": 0.165995 + }, + { + "accuracy": 0.205421, + "f1": 0.152333, + "f1_weighted": 0.150365 + } + ], + "main_score": 0.199715, + "hf_subset": "hne_Deva", + "languages": [ + "hne-Deva" + ] + }, + { + "accuracy": 0.407275, + "f1": 0.398273, + "f1_weighted": 0.408971, + "scores_per_experiment": [ + { + "accuracy": 0.420827, + "f1": 0.418765, + "f1_weighted": 0.422553 + }, + { + "accuracy": 0.376605, + "f1": 0.374246, + "f1_weighted": 0.378915 + }, + { + "accuracy": 0.369472, + "f1": 0.362889, + "f1_weighted": 0.376045 + }, + { + "accuracy": 0.435093, + "f1": 0.415254, + "f1_weighted": 0.439714 + }, + { + "accuracy": 0.402282, + "f1": 0.393077, + "f1_weighted": 0.40753 + }, + { + "accuracy": 0.422254, + "f1": 0.423258, + "f1_weighted": 0.424635 + }, + { + "accuracy": 0.416548, + "f1": 0.402985, + "f1_weighted": 0.405713 + }, + { + "accuracy": 0.430813, + "f1": 0.417365, + "f1_weighted": 0.436697 + }, + { + "accuracy": 0.442225, + "f1": 0.421786, + "f1_weighted": 0.438936 + }, + { + "accuracy": 0.356633, + "f1": 0.353108, + "f1_weighted": 0.358974 + } + ], + "main_score": 0.407275, + "hf_subset": "hrv_Latn", + "languages": [ + "hrv-Latn" + ] + }, + { + "accuracy": 0.340086, + "f1": 0.332844, + "f1_weighted": 0.34168, + "scores_per_experiment": [ + { + "accuracy": 0.393723, + "f1": 0.388484, + "f1_weighted": 0.396614 + }, + { + "accuracy": 0.282454, + "f1": 0.28231, + "f1_weighted": 0.279188 + }, + { + "accuracy": 0.310984, + "f1": 0.296961, + "f1_weighted": 0.31333 + }, + { + "accuracy": 0.336662, + "f1": 0.325251, + "f1_weighted": 0.337618 + }, + { + "accuracy": 0.349501, + "f1": 0.336493, + "f1_weighted": 0.35648 + }, + { + "accuracy": 0.343795, + "f1": 0.333612, + "f1_weighted": 0.336536 + }, + { + "accuracy": 0.350927, + "f1": 0.338389, + "f1_weighted": 0.354159 + }, + { + "accuracy": 0.36234, + "f1": 0.365564, + "f1_weighted": 0.371208 + }, + { + "accuracy": 0.352354, + "f1": 0.346205, + "f1_weighted": 0.354225 + }, + { + "accuracy": 0.318117, + "f1": 0.31517, + "f1_weighted": 0.317441 + } + ], + "main_score": 0.340086, + "hf_subset": "hun_Latn", + "languages": [ + "hun-Latn" + ] + }, + { + "accuracy": 0.199001, + "f1": 0.172944, + "f1_weighted": 0.175648, + "scores_per_experiment": [ + { + "accuracy": 0.233951, + "f1": 0.187924, + "f1_weighted": 0.194635 + }, + { + "accuracy": 0.162625, + "f1": 0.148547, + "f1_weighted": 0.150078 + }, + { + "accuracy": 0.208274, + "f1": 0.180046, + "f1_weighted": 0.18214 + }, + { + "accuracy": 0.245364, + "f1": 0.23107, + "f1_weighted": 0.241841 + }, + { + "accuracy": 0.203994, + "f1": 0.172061, + "f1_weighted": 0.182806 + }, + { + "accuracy": 0.145506, + "f1": 0.140367, + "f1_weighted": 0.130391 + }, + { + "accuracy": 0.17689, + "f1": 0.16309, + "f1_weighted": 0.150719 + }, + { + "accuracy": 0.212553, + "f1": 0.179659, + "f1_weighted": 0.190567 + }, + { + "accuracy": 0.202568, + "f1": 0.171298, + "f1_weighted": 0.179561 + }, + { + "accuracy": 0.198288, + "f1": 0.15538, + "f1_weighted": 0.15374 + } + ], + "main_score": 0.199001, + "hf_subset": "hye_Armn", + "languages": [ + "hye-Armn" + ] + }, + { + "accuracy": 0.338944, + "f1": 0.327757, + "f1_weighted": 0.342172, + "scores_per_experiment": [ + { + "accuracy": 0.363766, + "f1": 0.346641, + "f1_weighted": 0.3627 + }, + { + "accuracy": 0.318117, + "f1": 0.312275, + "f1_weighted": 0.325366 + }, + { + "accuracy": 0.342368, + "f1": 0.3263, + "f1_weighted": 0.346525 + }, + { + "accuracy": 0.349501, + "f1": 0.328336, + "f1_weighted": 0.355965 + }, + { + "accuracy": 0.32097, + "f1": 0.303286, + "f1_weighted": 0.319808 + }, + { + "accuracy": 0.303852, + "f1": 0.300772, + "f1_weighted": 0.305124 + }, + { + "accuracy": 0.349501, + "f1": 0.339503, + "f1_weighted": 0.347321 + }, + { + "accuracy": 0.365193, + "f1": 0.358453, + "f1_weighted": 0.372107 + }, + { + "accuracy": 0.332382, + "f1": 0.328549, + "f1_weighted": 0.338067 + }, + { + "accuracy": 0.343795, + "f1": 0.333453, + "f1_weighted": 0.348734 + } + ], + "main_score": 0.338944, + "hf_subset": "ibo_Latn", + "languages": [ + "ibo-Latn" + ] + }, + { + "accuracy": 0.457632, + "f1": 0.440625, + "f1_weighted": 0.463767, + "scores_per_experiment": [ + { + "accuracy": 0.483595, + "f1": 0.472474, + "f1_weighted": 0.485694 + }, + { + "accuracy": 0.477889, + "f1": 0.45303, + "f1_weighted": 0.479913 + }, + { + "accuracy": 0.406562, + "f1": 0.397803, + "f1_weighted": 0.408899 + }, + { + "accuracy": 0.493581, + "f1": 0.479252, + "f1_weighted": 0.497257 + }, + { + "accuracy": 0.415121, + "f1": 0.398197, + "f1_weighted": 0.42898 + }, + { + "accuracy": 0.450785, + "f1": 0.432047, + "f1_weighted": 0.452088 + }, + { + "accuracy": 0.485021, + "f1": 0.463486, + "f1_weighted": 0.493192 + }, + { + "accuracy": 0.455064, + "f1": 0.448398, + "f1_weighted": 0.469605 + }, + { + "accuracy": 0.422254, + "f1": 0.3991, + "f1_weighted": 0.429166 + }, + { + "accuracy": 0.486448, + "f1": 0.462465, + "f1_weighted": 0.492871 + } + ], + "main_score": 0.457632, + "hf_subset": "ilo_Latn", + "languages": [ + "ilo-Latn" + ] + }, + { + "accuracy": 0.453495, + "f1": 0.440923, + "f1_weighted": 0.460564, + "scores_per_experiment": [ + { + "accuracy": 0.483595, + "f1": 0.479462, + "f1_weighted": 0.488927 + }, + { + "accuracy": 0.439372, + "f1": 0.407381, + "f1_weighted": 0.440843 + }, + { + "accuracy": 0.425107, + "f1": 0.424449, + "f1_weighted": 0.437497 + }, + { + "accuracy": 0.487874, + "f1": 0.475338, + "f1_weighted": 0.493697 + }, + { + "accuracy": 0.426534, + "f1": 0.417043, + "f1_weighted": 0.435165 + }, + { + "accuracy": 0.443652, + "f1": 0.432586, + "f1_weighted": 0.441954 + }, + { + "accuracy": 0.472183, + "f1": 0.447182, + "f1_weighted": 0.479504 + }, + { + "accuracy": 0.445078, + "f1": 0.436953, + "f1_weighted": 0.46428 + }, + { + "accuracy": 0.46505, + "f1": 0.447488, + "f1_weighted": 0.46808 + }, + { + "accuracy": 0.446505, + "f1": 0.441344, + "f1_weighted": 0.455697 + } + ], + "main_score": 0.453495, + "hf_subset": "ind_Latn", + "languages": [ + "ind-Latn" + ] + }, + { + "accuracy": 0.35107, + "f1": 0.341506, + "f1_weighted": 0.351487, + "scores_per_experiment": [ + { + "accuracy": 0.389444, + "f1": 0.36898, + "f1_weighted": 0.388553 + }, + { + "accuracy": 0.305278, + "f1": 0.302588, + "f1_weighted": 0.305638 + }, + { + "accuracy": 0.319544, + "f1": 0.326957, + "f1_weighted": 0.323148 + }, + { + "accuracy": 0.356633, + "f1": 0.344914, + "f1_weighted": 0.356402 + }, + { + "accuracy": 0.340942, + "f1": 0.328463, + "f1_weighted": 0.335214 + }, + { + "accuracy": 0.36234, + "f1": 0.347303, + "f1_weighted": 0.368896 + }, + { + "accuracy": 0.370899, + "f1": 0.342981, + "f1_weighted": 0.365655 + }, + { + "accuracy": 0.378031, + "f1": 0.372197, + "f1_weighted": 0.377906 + }, + { + "accuracy": 0.32525, + "f1": 0.321748, + "f1_weighted": 0.328608 + }, + { + "accuracy": 0.36234, + "f1": 0.358929, + "f1_weighted": 0.36485 + } + ], + "main_score": 0.35107, + "hf_subset": "isl_Latn", + "languages": [ + "isl-Latn" + ] + }, + { + "accuracy": 0.57689, + "f1": 0.555699, + "f1_weighted": 0.578376, + "scores_per_experiment": [ + { + "accuracy": 0.570613, + "f1": 0.55967, + "f1_weighted": 0.582201 + }, + { + "accuracy": 0.593438, + "f1": 0.560488, + "f1_weighted": 0.591547 + }, + { + "accuracy": 0.53923, + "f1": 0.530866, + "f1_weighted": 0.538453 + }, + { + "accuracy": 0.630528, + "f1": 0.605032, + "f1_weighted": 0.631781 + }, + { + "accuracy": 0.584879, + "f1": 0.557444, + "f1_weighted": 0.585134 + }, + { + "accuracy": 0.57632, + "f1": 0.564073, + "f1_weighted": 0.576389 + }, + { + "accuracy": 0.573466, + "f1": 0.544459, + "f1_weighted": 0.567256 + }, + { + "accuracy": 0.562054, + "f1": 0.544875, + "f1_weighted": 0.565912 + }, + { + "accuracy": 0.552068, + "f1": 0.528563, + "f1_weighted": 0.556839 + }, + { + "accuracy": 0.586305, + "f1": 0.561526, + "f1_weighted": 0.588247 + } + ], + "main_score": 0.57689, + "hf_subset": "ita_Latn", + "languages": [ + "ita-Latn" + ] + }, + { + "accuracy": 0.350499, + "f1": 0.34232, + "f1_weighted": 0.353466, + "scores_per_experiment": [ + { + "accuracy": 0.406562, + "f1": 0.401364, + "f1_weighted": 0.410455 + }, + { + "accuracy": 0.363766, + "f1": 0.35047, + "f1_weighted": 0.36988 + }, + { + "accuracy": 0.326676, + "f1": 0.325407, + "f1_weighted": 0.327686 + }, + { + "accuracy": 0.340942, + "f1": 0.32476, + "f1_weighted": 0.344752 + }, + { + "accuracy": 0.313837, + "f1": 0.306952, + "f1_weighted": 0.315223 + }, + { + "accuracy": 0.335235, + "f1": 0.318292, + "f1_weighted": 0.327751 + }, + { + "accuracy": 0.369472, + "f1": 0.367674, + "f1_weighted": 0.373753 + }, + { + "accuracy": 0.359486, + "f1": 0.347837, + "f1_weighted": 0.361016 + }, + { + "accuracy": 0.335235, + "f1": 0.335523, + "f1_weighted": 0.343328 + }, + { + "accuracy": 0.35378, + "f1": 0.344921, + "f1_weighted": 0.360819 + } + ], + "main_score": 0.350499, + "hf_subset": "jav_Latn", + "languages": [ + "jav-Latn" + ] + }, + { + "accuracy": 0.221969, + "f1": 0.15978, + "f1_weighted": 0.170072, + "scores_per_experiment": [ + { + "accuracy": 0.259629, + "f1": 0.166346, + "f1_weighted": 0.183762 + }, + { + "accuracy": 0.156919, + "f1": 0.131701, + "f1_weighted": 0.11312 + }, + { + "accuracy": 0.1398, + "f1": 0.134065, + "f1_weighted": 0.126119 + }, + { + "accuracy": 0.278174, + "f1": 0.183466, + "f1_weighted": 0.207753 + }, + { + "accuracy": 0.269615, + "f1": 0.184663, + "f1_weighted": 0.207957 + }, + { + "accuracy": 0.248217, + "f1": 0.169826, + "f1_weighted": 0.194613 + }, + { + "accuracy": 0.24679, + "f1": 0.155271, + "f1_weighted": 0.178066 + }, + { + "accuracy": 0.262482, + "f1": 0.182385, + "f1_weighted": 0.201439 + }, + { + "accuracy": 0.182596, + "f1": 0.15066, + "f1_weighted": 0.153042 + }, + { + "accuracy": 0.175464, + "f1": 0.139413, + "f1_weighted": 0.134845 + } + ], + "main_score": 0.221969, + "hf_subset": "jpn_Jpan", + "languages": [ + "jpn-Jpan" + ] + }, + { + "accuracy": 0.287161, + "f1": 0.282338, + "f1_weighted": 0.28889, + "scores_per_experiment": [ + { + "accuracy": 0.306705, + "f1": 0.296763, + "f1_weighted": 0.311502 + }, + { + "accuracy": 0.259629, + "f1": 0.266038, + "f1_weighted": 0.258788 + }, + { + "accuracy": 0.291013, + "f1": 0.280573, + "f1_weighted": 0.296767 + }, + { + "accuracy": 0.296719, + "f1": 0.291937, + "f1_weighted": 0.299429 + }, + { + "accuracy": 0.291013, + "f1": 0.287246, + "f1_weighted": 0.289556 + }, + { + "accuracy": 0.281027, + "f1": 0.277056, + "f1_weighted": 0.275647 + }, + { + "accuracy": 0.303852, + "f1": 0.285895, + "f1_weighted": 0.303675 + }, + { + "accuracy": 0.273894, + "f1": 0.28033, + "f1_weighted": 0.279666 + }, + { + "accuracy": 0.28388, + "f1": 0.275972, + "f1_weighted": 0.291296 + }, + { + "accuracy": 0.28388, + "f1": 0.281567, + "f1_weighted": 0.282579 + } + ], + "main_score": 0.287161, + "hf_subset": "kab_Latn", + "languages": [ + "kab-Latn" + ] + }, + { + "accuracy": 0.388445, + "f1": 0.376312, + "f1_weighted": 0.39251, + "scores_per_experiment": [ + { + "accuracy": 0.388017, + "f1": 0.381069, + "f1_weighted": 0.394022 + }, + { + "accuracy": 0.359486, + "f1": 0.353778, + "f1_weighted": 0.360058 + }, + { + "accuracy": 0.405136, + "f1": 0.399506, + "f1_weighted": 0.409317 + }, + { + "accuracy": 0.400856, + "f1": 0.388336, + "f1_weighted": 0.40849 + }, + { + "accuracy": 0.393723, + "f1": 0.377644, + "f1_weighted": 0.399973 + }, + { + "accuracy": 0.373752, + "f1": 0.356908, + "f1_weighted": 0.365587 + }, + { + "accuracy": 0.388017, + "f1": 0.366182, + "f1_weighted": 0.393097 + }, + { + "accuracy": 0.39087, + "f1": 0.388829, + "f1_weighted": 0.401429 + }, + { + "accuracy": 0.370899, + "f1": 0.35695, + "f1_weighted": 0.377815 + }, + { + "accuracy": 0.413695, + "f1": 0.393917, + "f1_weighted": 0.415307 + } + ], + "main_score": 0.388445, + "hf_subset": "kac_Latn", + "languages": [ + "kac-Latn" + ] + }, + { + "accuracy": 0.355064, + "f1": 0.348446, + "f1_weighted": 0.356878, + "scores_per_experiment": [ + { + "accuracy": 0.376605, + "f1": 0.359919, + "f1_weighted": 0.375994 + }, + { + "accuracy": 0.318117, + "f1": 0.316554, + "f1_weighted": 0.320549 + }, + { + "accuracy": 0.330956, + "f1": 0.333004, + "f1_weighted": 0.333713 + }, + { + "accuracy": 0.349501, + "f1": 0.343568, + "f1_weighted": 0.354448 + }, + { + "accuracy": 0.39087, + "f1": 0.37673, + "f1_weighted": 0.390475 + }, + { + "accuracy": 0.315264, + "f1": 0.311174, + "f1_weighted": 0.309232 + }, + { + "accuracy": 0.346648, + "f1": 0.339503, + "f1_weighted": 0.347119 + }, + { + "accuracy": 0.396576, + "f1": 0.388846, + "f1_weighted": 0.402043 + }, + { + "accuracy": 0.370899, + "f1": 0.367087, + "f1_weighted": 0.37558 + }, + { + "accuracy": 0.355207, + "f1": 0.348077, + "f1_weighted": 0.359626 + } + ], + "main_score": 0.355064, + "hf_subset": "kam_Latn", + "languages": [ + "kam-Latn" + ] + }, + { + "accuracy": 0.200999, + "f1": 0.174961, + "f1_weighted": 0.175147, + "scores_per_experiment": [ + { + "accuracy": 0.253923, + "f1": 0.194506, + "f1_weighted": 0.205227 + }, + { + "accuracy": 0.182596, + "f1": 0.15178, + "f1_weighted": 0.151344 + }, + { + "accuracy": 0.145506, + "f1": 0.150716, + "f1_weighted": 0.125728 + }, + { + "accuracy": 0.192582, + "f1": 0.175533, + "f1_weighted": 0.163531 + }, + { + "accuracy": 0.17689, + "f1": 0.161671, + "f1_weighted": 0.144553 + }, + { + "accuracy": 0.228245, + "f1": 0.195414, + "f1_weighted": 0.223684 + }, + { + "accuracy": 0.2097, + "f1": 0.158931, + "f1_weighted": 0.171601 + }, + { + "accuracy": 0.231098, + "f1": 0.205081, + "f1_weighted": 0.225947 + }, + { + "accuracy": 0.196862, + "f1": 0.191948, + "f1_weighted": 0.180698 + }, + { + "accuracy": 0.192582, + "f1": 0.164029, + "f1_weighted": 0.159161 + } + ], + "main_score": 0.200999, + "hf_subset": "kan_Knda", + "languages": [ + "kan-Knda" + ] + }, + { + "accuracy": 0.240371, + "f1": 0.214765, + "f1_weighted": 0.217827, + "scores_per_experiment": [ + { + "accuracy": 0.278174, + "f1": 0.256533, + "f1_weighted": 0.250752 + }, + { + "accuracy": 0.186876, + "f1": 0.180348, + "f1_weighted": 0.159918 + }, + { + "accuracy": 0.185449, + "f1": 0.184138, + "f1_weighted": 0.171337 + }, + { + "accuracy": 0.275321, + "f1": 0.240425, + "f1_weighted": 0.258789 + }, + { + "accuracy": 0.174037, + "f1": 0.170935, + "f1_weighted": 0.153102 + }, + { + "accuracy": 0.276748, + "f1": 0.23413, + "f1_weighted": 0.264784 + }, + { + "accuracy": 0.262482, + "f1": 0.204072, + "f1_weighted": 0.222129 + }, + { + "accuracy": 0.28816, + "f1": 0.25765, + "f1_weighted": 0.277288 + }, + { + "accuracy": 0.272468, + "f1": 0.218452, + "f1_weighted": 0.221944 + }, + { + "accuracy": 0.203994, + "f1": 0.200967, + "f1_weighted": 0.198229 + } + ], + "main_score": 0.240371, + "hf_subset": "kas_Deva", + "languages": [ + "kas-Deva" + ] + }, + { + "accuracy": 0.198859, + "f1": 0.174311, + "f1_weighted": 0.171812, + "scores_per_experiment": [ + { + "accuracy": 0.256776, + "f1": 0.186545, + "f1_weighted": 0.190989 + }, + { + "accuracy": 0.156919, + "f1": 0.124, + "f1_weighted": 0.106117 + }, + { + "accuracy": 0.152639, + "f1": 0.143346, + "f1_weighted": 0.13201 + }, + { + "accuracy": 0.239658, + "f1": 0.215941, + "f1_weighted": 0.224685 + }, + { + "accuracy": 0.154066, + "f1": 0.14579, + "f1_weighted": 0.118421 + }, + { + "accuracy": 0.168331, + "f1": 0.159234, + "f1_weighted": 0.153225 + }, + { + "accuracy": 0.208274, + "f1": 0.188595, + "f1_weighted": 0.204376 + }, + { + "accuracy": 0.276748, + "f1": 0.246569, + "f1_weighted": 0.265614 + }, + { + "accuracy": 0.189729, + "f1": 0.180555, + "f1_weighted": 0.174006 + }, + { + "accuracy": 0.185449, + "f1": 0.152533, + "f1_weighted": 0.148673 + } + ], + "main_score": 0.198859, + "hf_subset": "kat_Geor", + "languages": [ + "kat-Geor" + ] + }, + { + "accuracy": 0.253923, + "f1": 0.243505, + "f1_weighted": 0.247985, + "scores_per_experiment": [ + { + "accuracy": 0.298146, + "f1": 0.253168, + "f1_weighted": 0.268887 + }, + { + "accuracy": 0.21398, + "f1": 0.207984, + "f1_weighted": 0.21335 + }, + { + "accuracy": 0.21398, + "f1": 0.218965, + "f1_weighted": 0.21318 + }, + { + "accuracy": 0.312411, + "f1": 0.269943, + "f1_weighted": 0.295826 + }, + { + "accuracy": 0.24679, + "f1": 0.251158, + "f1_weighted": 0.247921 + }, + { + "accuracy": 0.229672, + "f1": 0.226772, + "f1_weighted": 0.218525 + }, + { + "accuracy": 0.239658, + "f1": 0.233957, + "f1_weighted": 0.233102 + }, + { + "accuracy": 0.279601, + "f1": 0.284629, + "f1_weighted": 0.291354 + }, + { + "accuracy": 0.248217, + "f1": 0.242408, + "f1_weighted": 0.241908 + }, + { + "accuracy": 0.256776, + "f1": 0.246066, + "f1_weighted": 0.255794 + } + ], + "main_score": 0.253923, + "hf_subset": "kaz_Cyrl", + "languages": [ + "kaz-Cyrl" + ] + }, + { + "accuracy": 0.347504, + "f1": 0.337564, + "f1_weighted": 0.35164, + "scores_per_experiment": [ + { + "accuracy": 0.373752, + "f1": 0.365306, + "f1_weighted": 0.381371 + }, + { + "accuracy": 0.322397, + "f1": 0.322231, + "f1_weighted": 0.322535 + }, + { + "accuracy": 0.323823, + "f1": 0.314479, + "f1_weighted": 0.32726 + }, + { + "accuracy": 0.360913, + "f1": 0.351423, + "f1_weighted": 0.372263 + }, + { + "accuracy": 0.356633, + "f1": 0.344369, + "f1_weighted": 0.360166 + }, + { + "accuracy": 0.343795, + "f1": 0.335176, + "f1_weighted": 0.35571 + }, + { + "accuracy": 0.368046, + "f1": 0.341933, + "f1_weighted": 0.363837 + }, + { + "accuracy": 0.319544, + "f1": 0.323724, + "f1_weighted": 0.318025 + }, + { + "accuracy": 0.35378, + "f1": 0.338245, + "f1_weighted": 0.354609 + }, + { + "accuracy": 0.352354, + "f1": 0.338753, + "f1_weighted": 0.36063 + } + ], + "main_score": 0.347504, + "hf_subset": "kbp_Latn", + "languages": [ + "kbp-Latn" + ] + }, + { + "accuracy": 0.443795, + "f1": 0.425898, + "f1_weighted": 0.45007, + "scores_per_experiment": [ + { + "accuracy": 0.50214, + "f1": 0.478399, + "f1_weighted": 0.506118 + }, + { + "accuracy": 0.429387, + "f1": 0.420012, + "f1_weighted": 0.432786 + }, + { + "accuracy": 0.429387, + "f1": 0.427578, + "f1_weighted": 0.440428 + }, + { + "accuracy": 0.480742, + "f1": 0.453099, + "f1_weighted": 0.484062 + }, + { + "accuracy": 0.420827, + "f1": 0.406236, + "f1_weighted": 0.434167 + }, + { + "accuracy": 0.426534, + "f1": 0.405063, + "f1_weighted": 0.434857 + }, + { + "accuracy": 0.415121, + "f1": 0.393787, + "f1_weighted": 0.413733 + }, + { + "accuracy": 0.433666, + "f1": 0.412276, + "f1_weighted": 0.443978 + }, + { + "accuracy": 0.439372, + "f1": 0.42826, + "f1_weighted": 0.443177 + }, + { + "accuracy": 0.46077, + "f1": 0.434271, + "f1_weighted": 0.467393 + } + ], + "main_score": 0.443795, + "hf_subset": "kea_Latn", + "languages": [ + "kea-Latn" + ] + }, + { + "accuracy": 0.253067, + "f1": 0.236833, + "f1_weighted": 0.24718, + "scores_per_experiment": [ + { + "accuracy": 0.298146, + "f1": 0.246959, + "f1_weighted": 0.279094 + }, + { + "accuracy": 0.229672, + "f1": 0.216044, + "f1_weighted": 0.225726 + }, + { + "accuracy": 0.232525, + "f1": 0.240778, + "f1_weighted": 0.235893 + }, + { + "accuracy": 0.266762, + "f1": 0.241004, + "f1_weighted": 0.257387 + }, + { + "accuracy": 0.223966, + "f1": 0.219547, + "f1_weighted": 0.213493 + }, + { + "accuracy": 0.249643, + "f1": 0.247188, + "f1_weighted": 0.255503 + }, + { + "accuracy": 0.256776, + "f1": 0.233212, + "f1_weighted": 0.237602 + }, + { + "accuracy": 0.276748, + "f1": 0.265224, + "f1_weighted": 0.277286 + }, + { + "accuracy": 0.236805, + "f1": 0.2236, + "f1_weighted": 0.233532 + }, + { + "accuracy": 0.259629, + "f1": 0.23477, + "f1_weighted": 0.256286 + } + ], + "main_score": 0.253067, + "hf_subset": "khk_Cyrl", + "languages": [ + "khk-Cyrl" + ] + }, + { + "accuracy": 0.279886, + "f1": 0.253599, + "f1_weighted": 0.25699, + "scores_per_experiment": [ + { + "accuracy": 0.296719, + "f1": 0.255776, + "f1_weighted": 0.257181 + }, + { + "accuracy": 0.28388, + "f1": 0.267061, + "f1_weighted": 0.27747 + }, + { + "accuracy": 0.236805, + "f1": 0.212915, + "f1_weighted": 0.218907 + }, + { + "accuracy": 0.285307, + "f1": 0.269737, + "f1_weighted": 0.250274 + }, + { + "accuracy": 0.271041, + "f1": 0.253602, + "f1_weighted": 0.249856 + }, + { + "accuracy": 0.265335, + "f1": 0.209601, + "f1_weighted": 0.226283 + }, + { + "accuracy": 0.295292, + "f1": 0.267308, + "f1_weighted": 0.275421 + }, + { + "accuracy": 0.315264, + "f1": 0.296908, + "f1_weighted": 0.314141 + }, + { + "accuracy": 0.302425, + "f1": 0.277305, + "f1_weighted": 0.277732 + }, + { + "accuracy": 0.24679, + "f1": 0.225781, + "f1_weighted": 0.22264 + } + ], + "main_score": 0.279886, + "hf_subset": "khm_Khmr", + "languages": [ + "khm-Khmr" + ] + }, + { + "accuracy": 0.413267, + "f1": 0.399056, + "f1_weighted": 0.420228, + "scores_per_experiment": [ + { + "accuracy": 0.416548, + "f1": 0.406322, + "f1_weighted": 0.427009 + }, + { + "accuracy": 0.406562, + "f1": 0.399541, + "f1_weighted": 0.415629 + }, + { + "accuracy": 0.419401, + "f1": 0.398183, + "f1_weighted": 0.416699 + }, + { + "accuracy": 0.396576, + "f1": 0.380573, + "f1_weighted": 0.404882 + }, + { + "accuracy": 0.417974, + "f1": 0.397654, + "f1_weighted": 0.428001 + }, + { + "accuracy": 0.413695, + "f1": 0.399147, + "f1_weighted": 0.413403 + }, + { + "accuracy": 0.39515, + "f1": 0.383289, + "f1_weighted": 0.396366 + }, + { + "accuracy": 0.435093, + "f1": 0.421862, + "f1_weighted": 0.444433 + }, + { + "accuracy": 0.393723, + "f1": 0.377422, + "f1_weighted": 0.404003 + }, + { + "accuracy": 0.437946, + "f1": 0.426564, + "f1_weighted": 0.451855 + } + ], + "main_score": 0.413267, + "hf_subset": "kik_Latn", + "languages": [ + "kik-Latn" + ] + }, + { + "accuracy": 0.34893, + "f1": 0.344669, + "f1_weighted": 0.350575, + "scores_per_experiment": [ + { + "accuracy": 0.409415, + "f1": 0.401431, + "f1_weighted": 0.416306 + }, + { + "accuracy": 0.315264, + "f1": 0.313817, + "f1_weighted": 0.306907 + }, + { + "accuracy": 0.310984, + "f1": 0.305531, + "f1_weighted": 0.308721 + }, + { + "accuracy": 0.382311, + "f1": 0.371014, + "f1_weighted": 0.383391 + }, + { + "accuracy": 0.335235, + "f1": 0.334061, + "f1_weighted": 0.335841 + }, + { + "accuracy": 0.310984, + "f1": 0.313516, + "f1_weighted": 0.316299 + }, + { + "accuracy": 0.352354, + "f1": 0.343345, + "f1_weighted": 0.349457 + }, + { + "accuracy": 0.359486, + "f1": 0.362268, + "f1_weighted": 0.365098 + }, + { + "accuracy": 0.372325, + "f1": 0.357296, + "f1_weighted": 0.37531 + }, + { + "accuracy": 0.340942, + "f1": 0.34441, + "f1_weighted": 0.348426 + } + ], + "main_score": 0.34893, + "hf_subset": "kin_Latn", + "languages": [ + "kin-Latn" + ] + }, + { + "accuracy": 0.252211, + "f1": 0.23959, + "f1_weighted": 0.249764, + "scores_per_experiment": [ + { + "accuracy": 0.279601, + "f1": 0.258481, + "f1_weighted": 0.273646 + }, + { + "accuracy": 0.24679, + "f1": 0.233987, + "f1_weighted": 0.250602 + }, + { + "accuracy": 0.242511, + "f1": 0.240092, + "f1_weighted": 0.242796 + }, + { + "accuracy": 0.273894, + "f1": 0.255142, + "f1_weighted": 0.27676 + }, + { + "accuracy": 0.219686, + "f1": 0.226068, + "f1_weighted": 0.218261 + }, + { + "accuracy": 0.253923, + "f1": 0.245472, + "f1_weighted": 0.252764 + }, + { + "accuracy": 0.2097, + "f1": 0.197905, + "f1_weighted": 0.1961 + }, + { + "accuracy": 0.296719, + "f1": 0.280884, + "f1_weighted": 0.300053 + }, + { + "accuracy": 0.243937, + "f1": 0.233432, + "f1_weighted": 0.24161 + }, + { + "accuracy": 0.25535, + "f1": 0.224433, + "f1_weighted": 0.245045 + } + ], + "main_score": 0.252211, + "hf_subset": "kir_Cyrl", + "languages": [ + "kir-Cyrl" + ] + }, + { + "accuracy": 0.329672, + "f1": 0.321617, + "f1_weighted": 0.331714, + "scores_per_experiment": [ + { + "accuracy": 0.329529, + "f1": 0.325446, + "f1_weighted": 0.338402 + }, + { + "accuracy": 0.330956, + "f1": 0.327935, + "f1_weighted": 0.338397 + }, + { + "accuracy": 0.31669, + "f1": 0.3072, + "f1_weighted": 0.317337 + }, + { + "accuracy": 0.343795, + "f1": 0.338585, + "f1_weighted": 0.344127 + }, + { + "accuracy": 0.32097, + "f1": 0.314321, + "f1_weighted": 0.325539 + }, + { + "accuracy": 0.289586, + "f1": 0.277353, + "f1_weighted": 0.292623 + }, + { + "accuracy": 0.350927, + "f1": 0.338154, + "f1_weighted": 0.347699 + }, + { + "accuracy": 0.372325, + "f1": 0.36337, + "f1_weighted": 0.378905 + }, + { + "accuracy": 0.323823, + "f1": 0.312892, + "f1_weighted": 0.327986 + }, + { + "accuracy": 0.318117, + "f1": 0.310917, + "f1_weighted": 0.30613 + } + ], + "main_score": 0.329672, + "hf_subset": "kmb_Latn", + "languages": [ + "kmb-Latn" + ] + }, + { + "accuracy": 0.331241, + "f1": 0.321792, + "f1_weighted": 0.336215, + "scores_per_experiment": [ + { + "accuracy": 0.356633, + "f1": 0.335867, + "f1_weighted": 0.35758 + }, + { + "accuracy": 0.305278, + "f1": 0.295992, + "f1_weighted": 0.3077 + }, + { + "accuracy": 0.305278, + "f1": 0.294188, + "f1_weighted": 0.316298 + }, + { + "accuracy": 0.300999, + "f1": 0.298593, + "f1_weighted": 0.304036 + }, + { + "accuracy": 0.363766, + "f1": 0.34629, + "f1_weighted": 0.367161 + }, + { + "accuracy": 0.333809, + "f1": 0.319392, + "f1_weighted": 0.341086 + }, + { + "accuracy": 0.310984, + "f1": 0.31337, + "f1_weighted": 0.321264 + }, + { + "accuracy": 0.39087, + "f1": 0.369223, + "f1_weighted": 0.388025 + }, + { + "accuracy": 0.349501, + "f1": 0.351697, + "f1_weighted": 0.360524 + }, + { + "accuracy": 0.295292, + "f1": 0.293305, + "f1_weighted": 0.298476 + } + ], + "main_score": 0.331241, + "hf_subset": "kmr_Latn", + "languages": [ + "kmr-Latn" + ] + }, + { + "accuracy": 0.398573, + "f1": 0.38931, + "f1_weighted": 0.404987, + "scores_per_experiment": [ + { + "accuracy": 0.42368, + "f1": 0.418851, + "f1_weighted": 0.42763 + }, + { + "accuracy": 0.383738, + "f1": 0.371067, + "f1_weighted": 0.387044 + }, + { + "accuracy": 0.388017, + "f1": 0.38621, + "f1_weighted": 0.395318 + }, + { + "accuracy": 0.386591, + "f1": 0.37477, + "f1_weighted": 0.390547 + }, + { + "accuracy": 0.412268, + "f1": 0.403026, + "f1_weighted": 0.424288 + }, + { + "accuracy": 0.393723, + "f1": 0.385197, + "f1_weighted": 0.403451 + }, + { + "accuracy": 0.392297, + "f1": 0.380145, + "f1_weighted": 0.396271 + }, + { + "accuracy": 0.406562, + "f1": 0.412131, + "f1_weighted": 0.419921 + }, + { + "accuracy": 0.405136, + "f1": 0.384487, + "f1_weighted": 0.405332 + }, + { + "accuracy": 0.393723, + "f1": 0.377218, + "f1_weighted": 0.400066 + } + ], + "main_score": 0.398573, + "hf_subset": "knc_Latn", + "languages": [ + "knc-Latn" + ] + }, + { + "accuracy": 0.411983, + "f1": 0.399696, + "f1_weighted": 0.417482, + "scores_per_experiment": [ + { + "accuracy": 0.447932, + "f1": 0.440941, + "f1_weighted": 0.459463 + }, + { + "accuracy": 0.382311, + "f1": 0.379379, + "f1_weighted": 0.383556 + }, + { + "accuracy": 0.415121, + "f1": 0.407249, + "f1_weighted": 0.42176 + }, + { + "accuracy": 0.403709, + "f1": 0.38786, + "f1_weighted": 0.407124 + }, + { + "accuracy": 0.383738, + "f1": 0.358353, + "f1_weighted": 0.387205 + }, + { + "accuracy": 0.382311, + "f1": 0.369396, + "f1_weighted": 0.382038 + }, + { + "accuracy": 0.412268, + "f1": 0.395496, + "f1_weighted": 0.42145 + }, + { + "accuracy": 0.439372, + "f1": 0.435725, + "f1_weighted": 0.450073 + }, + { + "accuracy": 0.403709, + "f1": 0.386004, + "f1_weighted": 0.405909 + }, + { + "accuracy": 0.449358, + "f1": 0.436561, + "f1_weighted": 0.456239 + } + ], + "main_score": 0.411983, + "hf_subset": "kon_Latn", + "languages": [ + "kon-Latn" + ] + }, + { + "accuracy": 0.235093, + "f1": 0.199608, + "f1_weighted": 0.207623, + "scores_per_experiment": [ + { + "accuracy": 0.249643, + "f1": 0.210895, + "f1_weighted": 0.21283 + }, + { + "accuracy": 0.223966, + "f1": 0.188449, + "f1_weighted": 0.189766 + }, + { + "accuracy": 0.179743, + "f1": 0.184457, + "f1_weighted": 0.175176 + }, + { + "accuracy": 0.269615, + "f1": 0.249821, + "f1_weighted": 0.252457 + }, + { + "accuracy": 0.215407, + "f1": 0.204031, + "f1_weighted": 0.205136 + }, + { + "accuracy": 0.25535, + "f1": 0.182698, + "f1_weighted": 0.20625 + }, + { + "accuracy": 0.232525, + "f1": 0.190073, + "f1_weighted": 0.202953 + }, + { + "accuracy": 0.269615, + "f1": 0.223566, + "f1_weighted": 0.251937 + }, + { + "accuracy": 0.263909, + "f1": 0.184253, + "f1_weighted": 0.204188 + }, + { + "accuracy": 0.191155, + "f1": 0.177833, + "f1_weighted": 0.175534 + } + ], + "main_score": 0.235093, + "hf_subset": "kor_Hang", + "languages": [ + "kor-Hang" + ] + }, + { + "accuracy": 0.282026, + "f1": 0.259997, + "f1_weighted": 0.265795, + "scores_per_experiment": [ + { + "accuracy": 0.312411, + "f1": 0.307095, + "f1_weighted": 0.309559 + }, + { + "accuracy": 0.276748, + "f1": 0.255424, + "f1_weighted": 0.268952 + }, + { + "accuracy": 0.282454, + "f1": 0.255842, + "f1_weighted": 0.262401 + }, + { + "accuracy": 0.238231, + "f1": 0.234306, + "f1_weighted": 0.231959 + }, + { + "accuracy": 0.300999, + "f1": 0.256322, + "f1_weighted": 0.279326 + }, + { + "accuracy": 0.272468, + "f1": 0.230044, + "f1_weighted": 0.243738 + }, + { + "accuracy": 0.28816, + "f1": 0.239496, + "f1_weighted": 0.238151 + }, + { + "accuracy": 0.31669, + "f1": 0.31194, + "f1_weighted": 0.306771 + }, + { + "accuracy": 0.253923, + "f1": 0.264838, + "f1_weighted": 0.251919 + }, + { + "accuracy": 0.278174, + "f1": 0.24466, + "f1_weighted": 0.265176 + } + ], + "main_score": 0.282026, + "hf_subset": "lao_Laoo", + "languages": [ + "lao-Laoo" + ] + }, + { + "accuracy": 0.471041, + "f1": 0.453903, + "f1_weighted": 0.471447, + "scores_per_experiment": [ + { + "accuracy": 0.499287, + "f1": 0.480546, + "f1_weighted": 0.503348 + }, + { + "accuracy": 0.485021, + "f1": 0.463635, + "f1_weighted": 0.479874 + }, + { + "accuracy": 0.425107, + "f1": 0.413924, + "f1_weighted": 0.424271 + }, + { + "accuracy": 0.509272, + "f1": 0.481954, + "f1_weighted": 0.508972 + }, + { + "accuracy": 0.440799, + "f1": 0.431735, + "f1_weighted": 0.447502 + }, + { + "accuracy": 0.449358, + "f1": 0.437321, + "f1_weighted": 0.446335 + }, + { + "accuracy": 0.456491, + "f1": 0.431817, + "f1_weighted": 0.450281 + }, + { + "accuracy": 0.490728, + "f1": 0.471964, + "f1_weighted": 0.489213 + }, + { + "accuracy": 0.459344, + "f1": 0.449852, + "f1_weighted": 0.464368 + }, + { + "accuracy": 0.495007, + "f1": 0.476279, + "f1_weighted": 0.500306 + } + ], + "main_score": 0.471041, + "hf_subset": "lij_Latn", + "languages": [ + "lij-Latn" + ] + }, + { + "accuracy": 0.440086, + "f1": 0.435388, + "f1_weighted": 0.443738, + "scores_per_experiment": [ + { + "accuracy": 0.480742, + "f1": 0.470816, + "f1_weighted": 0.482431 + }, + { + "accuracy": 0.439372, + "f1": 0.436625, + "f1_weighted": 0.436564 + }, + { + "accuracy": 0.412268, + "f1": 0.40712, + "f1_weighted": 0.416469 + }, + { + "accuracy": 0.462197, + "f1": 0.45111, + "f1_weighted": 0.46615 + }, + { + "accuracy": 0.46505, + "f1": 0.463387, + "f1_weighted": 0.471195 + }, + { + "accuracy": 0.419401, + "f1": 0.41147, + "f1_weighted": 0.426022 + }, + { + "accuracy": 0.410842, + "f1": 0.409496, + "f1_weighted": 0.411618 + }, + { + "accuracy": 0.455064, + "f1": 0.457271, + "f1_weighted": 0.463624 + }, + { + "accuracy": 0.413695, + "f1": 0.416535, + "f1_weighted": 0.416336 + }, + { + "accuracy": 0.442225, + "f1": 0.430053, + "f1_weighted": 0.446967 + } + ], + "main_score": 0.440086, + "hf_subset": "lim_Latn", + "languages": [ + "lim-Latn" + ] + }, + { + "accuracy": 0.407133, + "f1": 0.399404, + "f1_weighted": 0.408196, + "scores_per_experiment": [ + { + "accuracy": 0.439372, + "f1": 0.433212, + "f1_weighted": 0.448044 + }, + { + "accuracy": 0.366619, + "f1": 0.370622, + "f1_weighted": 0.36703 + }, + { + "accuracy": 0.400856, + "f1": 0.386513, + "f1_weighted": 0.399911 + }, + { + "accuracy": 0.420827, + "f1": 0.41371, + "f1_weighted": 0.42771 + }, + { + "accuracy": 0.435093, + "f1": 0.420727, + "f1_weighted": 0.43889 + }, + { + "accuracy": 0.368046, + "f1": 0.364995, + "f1_weighted": 0.362498 + }, + { + "accuracy": 0.39087, + "f1": 0.370517, + "f1_weighted": 0.377588 + }, + { + "accuracy": 0.43224, + "f1": 0.425899, + "f1_weighted": 0.43576 + }, + { + "accuracy": 0.389444, + "f1": 0.387532, + "f1_weighted": 0.390755 + }, + { + "accuracy": 0.42796, + "f1": 0.420309, + "f1_weighted": 0.433771 + } + ], + "main_score": 0.407133, + "hf_subset": "lin_Latn", + "languages": [ + "lin-Latn" + ] + }, + { + "accuracy": 0.361341, + "f1": 0.348696, + "f1_weighted": 0.363867, + "scores_per_experiment": [ + { + "accuracy": 0.425107, + "f1": 0.410474, + "f1_weighted": 0.43146 + }, + { + "accuracy": 0.336662, + "f1": 0.310879, + "f1_weighted": 0.334459 + }, + { + "accuracy": 0.35806, + "f1": 0.341368, + "f1_weighted": 0.362787 + }, + { + "accuracy": 0.368046, + "f1": 0.353826, + "f1_weighted": 0.369792 + }, + { + "accuracy": 0.340942, + "f1": 0.329427, + "f1_weighted": 0.340957 + }, + { + "accuracy": 0.342368, + "f1": 0.337214, + "f1_weighted": 0.347592 + }, + { + "accuracy": 0.346648, + "f1": 0.339144, + "f1_weighted": 0.353385 + }, + { + "accuracy": 0.389444, + "f1": 0.383364, + "f1_weighted": 0.392994 + }, + { + "accuracy": 0.363766, + "f1": 0.34811, + "f1_weighted": 0.361715 + }, + { + "accuracy": 0.342368, + "f1": 0.333154, + "f1_weighted": 0.343525 + } + ], + "main_score": 0.361341, + "hf_subset": "lit_Latn", + "languages": [ + "lit-Latn" + ] + }, + { + "accuracy": 0.470613, + "f1": 0.452922, + "f1_weighted": 0.473049, + "scores_per_experiment": [ + { + "accuracy": 0.523538, + "f1": 0.508253, + "f1_weighted": 0.529791 + }, + { + "accuracy": 0.470756, + "f1": 0.453181, + "f1_weighted": 0.4764 + }, + { + "accuracy": 0.403709, + "f1": 0.396525, + "f1_weighted": 0.400702 + }, + { + "accuracy": 0.490728, + "f1": 0.461323, + "f1_weighted": 0.491734 + }, + { + "accuracy": 0.477889, + "f1": 0.460842, + "f1_weighted": 0.483092 + }, + { + "accuracy": 0.456491, + "f1": 0.440665, + "f1_weighted": 0.453847 + }, + { + "accuracy": 0.46933, + "f1": 0.443922, + "f1_weighted": 0.466619 + }, + { + "accuracy": 0.472183, + "f1": 0.45432, + "f1_weighted": 0.477375 + }, + { + "accuracy": 0.480742, + "f1": 0.454676, + "f1_weighted": 0.480964 + }, + { + "accuracy": 0.46077, + "f1": 0.455513, + "f1_weighted": 0.469966 + } + ], + "main_score": 0.470613, + "hf_subset": "lmo_Latn", + "languages": [ + "lmo-Latn" + ] + }, + { + "accuracy": 0.382454, + "f1": 0.371439, + "f1_weighted": 0.383607, + "scores_per_experiment": [ + { + "accuracy": 0.407989, + "f1": 0.396095, + "f1_weighted": 0.40968 + }, + { + "accuracy": 0.373752, + "f1": 0.353943, + "f1_weighted": 0.370588 + }, + { + "accuracy": 0.365193, + "f1": 0.351719, + "f1_weighted": 0.368538 + }, + { + "accuracy": 0.378031, + "f1": 0.371942, + "f1_weighted": 0.380222 + }, + { + "accuracy": 0.379458, + "f1": 0.372409, + "f1_weighted": 0.377805 + }, + { + "accuracy": 0.388017, + "f1": 0.382039, + "f1_weighted": 0.379202 + }, + { + "accuracy": 0.39087, + "f1": 0.375542, + "f1_weighted": 0.395122 + }, + { + "accuracy": 0.415121, + "f1": 0.394957, + "f1_weighted": 0.420707 + }, + { + "accuracy": 0.366619, + "f1": 0.365562, + "f1_weighted": 0.37079 + }, + { + "accuracy": 0.359486, + "f1": 0.350179, + "f1_weighted": 0.363414 + } + ], + "main_score": 0.382454, + "hf_subset": "ltg_Latn", + "languages": [ + "ltg-Latn" + ] + }, + { + "accuracy": 0.455492, + "f1": 0.448581, + "f1_weighted": 0.457219, + "scores_per_experiment": [ + { + "accuracy": 0.506419, + "f1": 0.483778, + "f1_weighted": 0.507137 + }, + { + "accuracy": 0.42796, + "f1": 0.431225, + "f1_weighted": 0.424777 + }, + { + "accuracy": 0.436519, + "f1": 0.430687, + "f1_weighted": 0.437166 + }, + { + "accuracy": 0.50214, + "f1": 0.493523, + "f1_weighted": 0.508603 + }, + { + "accuracy": 0.446505, + "f1": 0.442663, + "f1_weighted": 0.451087 + }, + { + "accuracy": 0.470756, + "f1": 0.458969, + "f1_weighted": 0.468431 + }, + { + "accuracy": 0.442225, + "f1": 0.443629, + "f1_weighted": 0.447762 + }, + { + "accuracy": 0.42368, + "f1": 0.420762, + "f1_weighted": 0.427965 + }, + { + "accuracy": 0.443652, + "f1": 0.43039, + "f1_weighted": 0.440325 + }, + { + "accuracy": 0.455064, + "f1": 0.450189, + "f1_weighted": 0.458941 + } + ], + "main_score": 0.455492, + "hf_subset": "ltz_Latn", + "languages": [ + "ltz-Latn" + ] + }, + { + "accuracy": 0.399429, + "f1": 0.386687, + "f1_weighted": 0.400896, + "scores_per_experiment": [ + { + "accuracy": 0.409415, + "f1": 0.402387, + "f1_weighted": 0.420731 + }, + { + "accuracy": 0.370899, + "f1": 0.371386, + "f1_weighted": 0.372713 + }, + { + "accuracy": 0.392297, + "f1": 0.380111, + "f1_weighted": 0.393676 + }, + { + "accuracy": 0.392297, + "f1": 0.379696, + "f1_weighted": 0.401199 + }, + { + "accuracy": 0.413695, + "f1": 0.392311, + "f1_weighted": 0.41142 + }, + { + "accuracy": 0.380884, + "f1": 0.372623, + "f1_weighted": 0.371526 + }, + { + "accuracy": 0.373752, + "f1": 0.352036, + "f1_weighted": 0.370385 + }, + { + "accuracy": 0.455064, + "f1": 0.442299, + "f1_weighted": 0.455519 + }, + { + "accuracy": 0.386591, + "f1": 0.370971, + "f1_weighted": 0.387004 + }, + { + "accuracy": 0.419401, + "f1": 0.403049, + "f1_weighted": 0.424785 + } + ], + "main_score": 0.399429, + "hf_subset": "lua_Latn", + "languages": [ + "lua-Latn" + ] + }, + { + "accuracy": 0.318688, + "f1": 0.316555, + "f1_weighted": 0.32085, + "scores_per_experiment": [ + { + "accuracy": 0.313837, + "f1": 0.326113, + "f1_weighted": 0.314117 + }, + { + "accuracy": 0.32525, + "f1": 0.321853, + "f1_weighted": 0.334373 + }, + { + "accuracy": 0.272468, + "f1": 0.26196, + "f1_weighted": 0.273253 + }, + { + "accuracy": 0.322397, + "f1": 0.313383, + "f1_weighted": 0.325862 + }, + { + "accuracy": 0.340942, + "f1": 0.333343, + "f1_weighted": 0.342027 + }, + { + "accuracy": 0.291013, + "f1": 0.282283, + "f1_weighted": 0.289872 + }, + { + "accuracy": 0.330956, + "f1": 0.320693, + "f1_weighted": 0.326378 + }, + { + "accuracy": 0.365193, + "f1": 0.366145, + "f1_weighted": 0.369646 + }, + { + "accuracy": 0.303852, + "f1": 0.310363, + "f1_weighted": 0.307913 + }, + { + "accuracy": 0.32097, + "f1": 0.329412, + "f1_weighted": 0.325057 + } + ], + "main_score": 0.318688, + "hf_subset": "lug_Latn", + "languages": [ + "lug-Latn" + ] + }, + { + "accuracy": 0.351926, + "f1": 0.344561, + "f1_weighted": 0.356208, + "scores_per_experiment": [ + { + "accuracy": 0.368046, + "f1": 0.344559, + "f1_weighted": 0.369543 + }, + { + "accuracy": 0.336662, + "f1": 0.335595, + "f1_weighted": 0.33463 + }, + { + "accuracy": 0.31669, + "f1": 0.297452, + "f1_weighted": 0.319488 + }, + { + "accuracy": 0.342368, + "f1": 0.335639, + "f1_weighted": 0.351873 + }, + { + "accuracy": 0.36234, + "f1": 0.357977, + "f1_weighted": 0.368276 + }, + { + "accuracy": 0.329529, + "f1": 0.323244, + "f1_weighted": 0.332005 + }, + { + "accuracy": 0.373752, + "f1": 0.35678, + "f1_weighted": 0.373079 + }, + { + "accuracy": 0.382311, + "f1": 0.376912, + "f1_weighted": 0.38811 + }, + { + "accuracy": 0.343795, + "f1": 0.344498, + "f1_weighted": 0.347445 + }, + { + "accuracy": 0.363766, + "f1": 0.372956, + "f1_weighted": 0.377637 + } + ], + "main_score": 0.351926, + "hf_subset": "luo_Latn", + "languages": [ + "luo-Latn" + ] + }, + { + "accuracy": 0.479601, + "f1": 0.460189, + "f1_weighted": 0.486566, + "scores_per_experiment": [ + { + "accuracy": 0.509272, + "f1": 0.486294, + "f1_weighted": 0.517445 + }, + { + "accuracy": 0.472183, + "f1": 0.447602, + "f1_weighted": 0.476328 + }, + { + "accuracy": 0.417974, + "f1": 0.410343, + "f1_weighted": 0.424607 + }, + { + "accuracy": 0.510699, + "f1": 0.479826, + "f1_weighted": 0.511933 + }, + { + "accuracy": 0.50214, + "f1": 0.478673, + "f1_weighted": 0.506852 + }, + { + "accuracy": 0.433666, + "f1": 0.421816, + "f1_weighted": 0.445274 + }, + { + "accuracy": 0.516405, + "f1": 0.493097, + "f1_weighted": 0.525323 + }, + { + "accuracy": 0.50214, + "f1": 0.487547, + "f1_weighted": 0.511992 + }, + { + "accuracy": 0.447932, + "f1": 0.425213, + "f1_weighted": 0.458967 + }, + { + "accuracy": 0.483595, + "f1": 0.471481, + "f1_weighted": 0.48694 + } + ], + "main_score": 0.479601, + "hf_subset": "lus_Latn", + "languages": [ + "lus-Latn" + ] + }, + { + "accuracy": 0.367475, + "f1": 0.360833, + "f1_weighted": 0.370019, + "scores_per_experiment": [ + { + "accuracy": 0.403709, + "f1": 0.395854, + "f1_weighted": 0.407006 + }, + { + "accuracy": 0.359486, + "f1": 0.339197, + "f1_weighted": 0.358421 + }, + { + "accuracy": 0.349501, + "f1": 0.350042, + "f1_weighted": 0.353636 + }, + { + "accuracy": 0.385164, + "f1": 0.382902, + "f1_weighted": 0.393086 + }, + { + "accuracy": 0.345221, + "f1": 0.339972, + "f1_weighted": 0.347702 + }, + { + "accuracy": 0.396576, + "f1": 0.387538, + "f1_weighted": 0.405903 + }, + { + "accuracy": 0.343795, + "f1": 0.342322, + "f1_weighted": 0.343709 + }, + { + "accuracy": 0.359486, + "f1": 0.355812, + "f1_weighted": 0.362477 + }, + { + "accuracy": 0.359486, + "f1": 0.350555, + "f1_weighted": 0.358494 + }, + { + "accuracy": 0.372325, + "f1": 0.36414, + "f1_weighted": 0.369759 + } + ], + "main_score": 0.367475, + "hf_subset": "lvs_Latn", + "languages": [ + "lvs-Latn" + ] + }, + { + "accuracy": 0.194151, + "f1": 0.168312, + "f1_weighted": 0.167263, + "scores_per_experiment": [ + { + "accuracy": 0.171184, + "f1": 0.150068, + "f1_weighted": 0.14625 + }, + { + "accuracy": 0.18117, + "f1": 0.15613, + "f1_weighted": 0.14851 + }, + { + "accuracy": 0.156919, + "f1": 0.157926, + "f1_weighted": 0.138048 + }, + { + "accuracy": 0.174037, + "f1": 0.153395, + "f1_weighted": 0.138308 + }, + { + "accuracy": 0.185449, + "f1": 0.169549, + "f1_weighted": 0.166356 + }, + { + "accuracy": 0.243937, + "f1": 0.20745, + "f1_weighted": 0.228774 + }, + { + "accuracy": 0.194009, + "f1": 0.161056, + "f1_weighted": 0.161005 + }, + { + "accuracy": 0.225392, + "f1": 0.183527, + "f1_weighted": 0.181548 + }, + { + "accuracy": 0.196862, + "f1": 0.170762, + "f1_weighted": 0.175477 + }, + { + "accuracy": 0.212553, + "f1": 0.173257, + "f1_weighted": 0.18835 + } + ], + "main_score": 0.194151, + "hf_subset": "mag_Deva", + "languages": [ + "mag-Deva" + ] + }, + { + "accuracy": 0.214693, + "f1": 0.181117, + "f1_weighted": 0.189242, + "scores_per_experiment": [ + { + "accuracy": 0.245364, + "f1": 0.200947, + "f1_weighted": 0.211659 + }, + { + "accuracy": 0.198288, + "f1": 0.160989, + "f1_weighted": 0.159374 + }, + { + "accuracy": 0.226819, + "f1": 0.184531, + "f1_weighted": 0.205728 + }, + { + "accuracy": 0.188302, + "f1": 0.158847, + "f1_weighted": 0.152309 + }, + { + "accuracy": 0.178317, + "f1": 0.178702, + "f1_weighted": 0.166646 + }, + { + "accuracy": 0.191155, + "f1": 0.182788, + "f1_weighted": 0.189381 + }, + { + "accuracy": 0.192582, + "f1": 0.172801, + "f1_weighted": 0.180852 + }, + { + "accuracy": 0.296719, + "f1": 0.231266, + "f1_weighted": 0.265994 + }, + { + "accuracy": 0.226819, + "f1": 0.18375, + "f1_weighted": 0.195106 + }, + { + "accuracy": 0.202568, + "f1": 0.156545, + "f1_weighted": 0.165371 + } + ], + "main_score": 0.214693, + "hf_subset": "mai_Deva", + "languages": [ + "mai-Deva" + ] + }, + { + "accuracy": 0.2097, + "f1": 0.173237, + "f1_weighted": 0.179561, + "scores_per_experiment": [ + { + "accuracy": 0.242511, + "f1": 0.206325, + "f1_weighted": 0.207432 + }, + { + "accuracy": 0.159772, + "f1": 0.105775, + "f1_weighted": 0.096922 + }, + { + "accuracy": 0.164051, + "f1": 0.161499, + "f1_weighted": 0.143856 + }, + { + "accuracy": 0.199715, + "f1": 0.18769, + "f1_weighted": 0.171343 + }, + { + "accuracy": 0.171184, + "f1": 0.16056, + "f1_weighted": 0.139441 + }, + { + "accuracy": 0.259629, + "f1": 0.18861, + "f1_weighted": 0.224742 + }, + { + "accuracy": 0.223966, + "f1": 0.172648, + "f1_weighted": 0.195763 + }, + { + "accuracy": 0.268188, + "f1": 0.197864, + "f1_weighted": 0.235975 + }, + { + "accuracy": 0.21398, + "f1": 0.177438, + "f1_weighted": 0.193499 + }, + { + "accuracy": 0.194009, + "f1": 0.17396, + "f1_weighted": 0.186638 + } + ], + "main_score": 0.2097, + "hf_subset": "mal_Mlym", + "languages": [ + "mal-Mlym" + ] + }, + { + "accuracy": 0.187161, + "f1": 0.165035, + "f1_weighted": 0.1681, + "scores_per_experiment": [ + { + "accuracy": 0.2097, + "f1": 0.173817, + "f1_weighted": 0.178913 + }, + { + "accuracy": 0.192582, + "f1": 0.158246, + "f1_weighted": 0.173397 + }, + { + "accuracy": 0.171184, + "f1": 0.164035, + "f1_weighted": 0.160439 + }, + { + "accuracy": 0.186876, + "f1": 0.160753, + "f1_weighted": 0.161466 + }, + { + "accuracy": 0.138374, + "f1": 0.132923, + "f1_weighted": 0.109833 + }, + { + "accuracy": 0.233951, + "f1": 0.185638, + "f1_weighted": 0.213519 + }, + { + "accuracy": 0.178317, + "f1": 0.168113, + "f1_weighted": 0.165263 + }, + { + "accuracy": 0.195435, + "f1": 0.19233, + "f1_weighted": 0.190456 + }, + { + "accuracy": 0.18117, + "f1": 0.160977, + "f1_weighted": 0.173032 + }, + { + "accuracy": 0.184023, + "f1": 0.153521, + "f1_weighted": 0.154681 + } + ], + "main_score": 0.187161, + "hf_subset": "mar_Deva", + "languages": [ + "mar-Deva" + ] + }, + { + "accuracy": 0.408417, + "f1": 0.398882, + "f1_weighted": 0.413681, + "scores_per_experiment": [ + { + "accuracy": 0.439372, + "f1": 0.430808, + "f1_weighted": 0.447573 + }, + { + "accuracy": 0.39087, + "f1": 0.376447, + "f1_weighted": 0.398261 + }, + { + "accuracy": 0.412268, + "f1": 0.396718, + "f1_weighted": 0.417671 + }, + { + "accuracy": 0.422254, + "f1": 0.410008, + "f1_weighted": 0.430509 + }, + { + "accuracy": 0.372325, + "f1": 0.357995, + "f1_weighted": 0.375029 + }, + { + "accuracy": 0.363766, + "f1": 0.359503, + "f1_weighted": 0.361381 + }, + { + "accuracy": 0.392297, + "f1": 0.383083, + "f1_weighted": 0.394784 + }, + { + "accuracy": 0.419401, + "f1": 0.412927, + "f1_weighted": 0.431463 + }, + { + "accuracy": 0.446505, + "f1": 0.435265, + "f1_weighted": 0.4479 + }, + { + "accuracy": 0.425107, + "f1": 0.426063, + "f1_weighted": 0.432241 + } + ], + "main_score": 0.408417, + "hf_subset": "min_Latn", + "languages": [ + "min-Latn" + ] + }, + { + "accuracy": 0.245649, + "f1": 0.229796, + "f1_weighted": 0.239038, + "scores_per_experiment": [ + { + "accuracy": 0.309558, + "f1": 0.256683, + "f1_weighted": 0.28108 + }, + { + "accuracy": 0.206847, + "f1": 0.18537, + "f1_weighted": 0.195381 + }, + { + "accuracy": 0.269615, + "f1": 0.253451, + "f1_weighted": 0.266581 + }, + { + "accuracy": 0.248217, + "f1": 0.236182, + "f1_weighted": 0.254533 + }, + { + "accuracy": 0.221113, + "f1": 0.205994, + "f1_weighted": 0.218332 + }, + { + "accuracy": 0.25107, + "f1": 0.223545, + "f1_weighted": 0.230629 + }, + { + "accuracy": 0.199715, + "f1": 0.202967, + "f1_weighted": 0.194571 + }, + { + "accuracy": 0.256776, + "f1": 0.257333, + "f1_weighted": 0.260012 + }, + { + "accuracy": 0.258203, + "f1": 0.249063, + "f1_weighted": 0.256504 + }, + { + "accuracy": 0.235378, + "f1": 0.227368, + "f1_weighted": 0.232758 + } + ], + "main_score": 0.245649, + "hf_subset": "mkd_Cyrl", + "languages": [ + "mkd-Cyrl" + ] + }, + { + "accuracy": 0.380314, + "f1": 0.37673, + "f1_weighted": 0.382781, + "scores_per_experiment": [ + { + "accuracy": 0.420827, + "f1": 0.428342, + "f1_weighted": 0.432957 + }, + { + "accuracy": 0.349501, + "f1": 0.348133, + "f1_weighted": 0.348078 + }, + { + "accuracy": 0.355207, + "f1": 0.356341, + "f1_weighted": 0.351973 + }, + { + "accuracy": 0.403709, + "f1": 0.395919, + "f1_weighted": 0.404048 + }, + { + "accuracy": 0.378031, + "f1": 0.373802, + "f1_weighted": 0.378576 + }, + { + "accuracy": 0.375178, + "f1": 0.368218, + "f1_weighted": 0.378927 + }, + { + "accuracy": 0.380884, + "f1": 0.376535, + "f1_weighted": 0.38519 + }, + { + "accuracy": 0.398003, + "f1": 0.396176, + "f1_weighted": 0.408667 + }, + { + "accuracy": 0.380884, + "f1": 0.365452, + "f1_weighted": 0.376431 + }, + { + "accuracy": 0.360913, + "f1": 0.358382, + "f1_weighted": 0.362963 + } + ], + "main_score": 0.380314, + "hf_subset": "mlt_Latn", + "languages": [ + "mlt-Latn" + ] + }, + { + "accuracy": 0.2, + "f1": 0.177455, + "f1_weighted": 0.178523, + "scores_per_experiment": [ + { + "accuracy": 0.215407, + "f1": 0.187426, + "f1_weighted": 0.184228 + }, + { + "accuracy": 0.216833, + "f1": 0.161521, + "f1_weighted": 0.186518 + }, + { + "accuracy": 0.155492, + "f1": 0.153134, + "f1_weighted": 0.139503 + }, + { + "accuracy": 0.215407, + "f1": 0.211308, + "f1_weighted": 0.211974 + }, + { + "accuracy": 0.201141, + "f1": 0.186637, + "f1_weighted": 0.179581 + }, + { + "accuracy": 0.194009, + "f1": 0.177542, + "f1_weighted": 0.177707 + }, + { + "accuracy": 0.166904, + "f1": 0.159312, + "f1_weighted": 0.150058 + }, + { + "accuracy": 0.28816, + "f1": 0.221425, + "f1_weighted": 0.253219 + }, + { + "accuracy": 0.164051, + "f1": 0.160685, + "f1_weighted": 0.154865 + }, + { + "accuracy": 0.182596, + "f1": 0.155556, + "f1_weighted": 0.147574 + } + ], + "main_score": 0.2, + "hf_subset": "mni_Beng", + "languages": [ + "mni-Beng" + ] + }, + { + "accuracy": 0.352924, + "f1": 0.342433, + "f1_weighted": 0.358707, + "scores_per_experiment": [ + { + "accuracy": 0.375178, + "f1": 0.370522, + "f1_weighted": 0.381699 + }, + { + "accuracy": 0.330956, + "f1": 0.325792, + "f1_weighted": 0.334291 + }, + { + "accuracy": 0.336662, + "f1": 0.334606, + "f1_weighted": 0.346127 + }, + { + "accuracy": 0.378031, + "f1": 0.362424, + "f1_weighted": 0.381993 + }, + { + "accuracy": 0.336662, + "f1": 0.330756, + "f1_weighted": 0.340673 + }, + { + "accuracy": 0.372325, + "f1": 0.351592, + "f1_weighted": 0.379583 + }, + { + "accuracy": 0.355207, + "f1": 0.339652, + "f1_weighted": 0.362496 + }, + { + "accuracy": 0.388017, + "f1": 0.369833, + "f1_weighted": 0.395118 + }, + { + "accuracy": 0.336662, + "f1": 0.322312, + "f1_weighted": 0.339376 + }, + { + "accuracy": 0.319544, + "f1": 0.316844, + "f1_weighted": 0.325716 + } + ], + "main_score": 0.352924, + "hf_subset": "mos_Latn", + "languages": [ + "mos-Latn" + ] + }, + { + "accuracy": 0.37689, + "f1": 0.362184, + "f1_weighted": 0.374107, + "scores_per_experiment": [ + { + "accuracy": 0.402282, + "f1": 0.387218, + "f1_weighted": 0.401116 + }, + { + "accuracy": 0.359486, + "f1": 0.338305, + "f1_weighted": 0.355705 + }, + { + "accuracy": 0.399429, + "f1": 0.375389, + "f1_weighted": 0.391828 + }, + { + "accuracy": 0.35806, + "f1": 0.349725, + "f1_weighted": 0.361475 + }, + { + "accuracy": 0.399429, + "f1": 0.381929, + "f1_weighted": 0.390947 + }, + { + "accuracy": 0.323823, + "f1": 0.311969, + "f1_weighted": 0.325851 + }, + { + "accuracy": 0.379458, + "f1": 0.362061, + "f1_weighted": 0.361917 + }, + { + "accuracy": 0.383738, + "f1": 0.373737, + "f1_weighted": 0.384941 + }, + { + "accuracy": 0.363766, + "f1": 0.350123, + "f1_weighted": 0.360837 + }, + { + "accuracy": 0.399429, + "f1": 0.391387, + "f1_weighted": 0.406452 + } + ], + "main_score": 0.37689, + "hf_subset": "mri_Latn", + "languages": [ + "mri-Latn" + ] + }, + { + "accuracy": 0.184451, + "f1": 0.142573, + "f1_weighted": 0.146874, + "scores_per_experiment": [ + { + "accuracy": 0.25535, + "f1": 0.204115, + "f1_weighted": 0.213323 + }, + { + "accuracy": 0.155492, + "f1": 0.111893, + "f1_weighted": 0.100843 + }, + { + "accuracy": 0.128388, + "f1": 0.116015, + "f1_weighted": 0.107981 + }, + { + "accuracy": 0.155492, + "f1": 0.110537, + "f1_weighted": 0.123556 + }, + { + "accuracy": 0.233951, + "f1": 0.173944, + "f1_weighted": 0.179966 + }, + { + "accuracy": 0.164051, + "f1": 0.133862, + "f1_weighted": 0.145068 + }, + { + "accuracy": 0.172611, + "f1": 0.141826, + "f1_weighted": 0.142488 + }, + { + "accuracy": 0.243937, + "f1": 0.141307, + "f1_weighted": 0.157702 + }, + { + "accuracy": 0.14408, + "f1": 0.11857, + "f1_weighted": 0.128363 + }, + { + "accuracy": 0.191155, + "f1": 0.173664, + "f1_weighted": 0.16945 + } + ], + "main_score": 0.184451, + "hf_subset": "mya_Mymr", + "languages": [ + "mya-Mymr" + ] + }, + { + "accuracy": 0.497575, + "f1": 0.484265, + "f1_weighted": 0.500542, + "scores_per_experiment": [ + { + "accuracy": 0.53495, + "f1": 0.516502, + "f1_weighted": 0.538077 + }, + { + "accuracy": 0.50214, + "f1": 0.492887, + "f1_weighted": 0.506533 + }, + { + "accuracy": 0.430813, + "f1": 0.423687, + "f1_weighted": 0.436658 + }, + { + "accuracy": 0.543509, + "f1": 0.521136, + "f1_weighted": 0.542306 + }, + { + "accuracy": 0.514979, + "f1": 0.499982, + "f1_weighted": 0.519564 + }, + { + "accuracy": 0.477889, + "f1": 0.471257, + "f1_weighted": 0.480201 + }, + { + "accuracy": 0.482168, + "f1": 0.469827, + "f1_weighted": 0.484284 + }, + { + "accuracy": 0.499287, + "f1": 0.494259, + "f1_weighted": 0.50698 + }, + { + "accuracy": 0.487874, + "f1": 0.465231, + "f1_weighted": 0.487563 + }, + { + "accuracy": 0.50214, + "f1": 0.487878, + "f1_weighted": 0.503256 + } + ], + "main_score": 0.497575, + "hf_subset": "nld_Latn", + "languages": [ + "nld-Latn" + ] + }, + { + "accuracy": 0.45806, + "f1": 0.446078, + "f1_weighted": 0.461563, + "scores_per_experiment": [ + { + "accuracy": 0.49786, + "f1": 0.488869, + "f1_weighted": 0.507727 + }, + { + "accuracy": 0.452211, + "f1": 0.441829, + "f1_weighted": 0.45531 + }, + { + "accuracy": 0.447932, + "f1": 0.441783, + "f1_weighted": 0.443727 + }, + { + "accuracy": 0.457917, + "f1": 0.445619, + "f1_weighted": 0.468867 + }, + { + "accuracy": 0.436519, + "f1": 0.415961, + "f1_weighted": 0.437344 + }, + { + "accuracy": 0.436519, + "f1": 0.416454, + "f1_weighted": 0.4376 + }, + { + "accuracy": 0.43224, + "f1": 0.412727, + "f1_weighted": 0.424603 + }, + { + "accuracy": 0.503566, + "f1": 0.493411, + "f1_weighted": 0.507656 + }, + { + "accuracy": 0.470756, + "f1": 0.462732, + "f1_weighted": 0.475234 + }, + { + "accuracy": 0.445078, + "f1": 0.441397, + "f1_weighted": 0.457564 + } + ], + "main_score": 0.45806, + "hf_subset": "nno_Latn", + "languages": [ + "nno-Latn" + ] + }, + { + "accuracy": 0.472183, + "f1": 0.457273, + "f1_weighted": 0.47639, + "scores_per_experiment": [ + { + "accuracy": 0.503566, + "f1": 0.498493, + "f1_weighted": 0.5132 + }, + { + "accuracy": 0.467903, + "f1": 0.450064, + "f1_weighted": 0.469764 + }, + { + "accuracy": 0.425107, + "f1": 0.411705, + "f1_weighted": 0.425509 + }, + { + "accuracy": 0.510699, + "f1": 0.489343, + "f1_weighted": 0.517402 + }, + { + "accuracy": 0.457917, + "f1": 0.443873, + "f1_weighted": 0.463626 + }, + { + "accuracy": 0.429387, + "f1": 0.416892, + "f1_weighted": 0.430874 + }, + { + "accuracy": 0.496434, + "f1": 0.474577, + "f1_weighted": 0.491993 + }, + { + "accuracy": 0.487874, + "f1": 0.474559, + "f1_weighted": 0.498261 + }, + { + "accuracy": 0.496434, + "f1": 0.475184, + "f1_weighted": 0.498028 + }, + { + "accuracy": 0.446505, + "f1": 0.438038, + "f1_weighted": 0.455239 + } + ], + "main_score": 0.472183, + "hf_subset": "nob_Latn", + "languages": [ + "nob-Latn" + ] + }, + { + "accuracy": 0.21184, + "f1": 0.187753, + "f1_weighted": 0.188763, + "scores_per_experiment": [ + { + "accuracy": 0.222539, + "f1": 0.193227, + "f1_weighted": 0.202402 + }, + { + "accuracy": 0.225392, + "f1": 0.172582, + "f1_weighted": 0.179709 + }, + { + "accuracy": 0.158345, + "f1": 0.156936, + "f1_weighted": 0.137082 + }, + { + "accuracy": 0.182596, + "f1": 0.166492, + "f1_weighted": 0.162815 + }, + { + "accuracy": 0.221113, + "f1": 0.199088, + "f1_weighted": 0.195092 + }, + { + "accuracy": 0.212553, + "f1": 0.210258, + "f1_weighted": 0.202177 + }, + { + "accuracy": 0.216833, + "f1": 0.186087, + "f1_weighted": 0.185838 + }, + { + "accuracy": 0.256776, + "f1": 0.226093, + "f1_weighted": 0.245957 + }, + { + "accuracy": 0.203994, + "f1": 0.176144, + "f1_weighted": 0.175082 + }, + { + "accuracy": 0.21826, + "f1": 0.19062, + "f1_weighted": 0.201475 + } + ], + "main_score": 0.21184, + "hf_subset": "npi_Deva", + "languages": [ + "npi-Deva" + ] + }, + { + "accuracy": 0.19572, + "f1": 0.156402, + "f1_weighted": 0.164675, + "scores_per_experiment": [ + { + "accuracy": 0.229672, + "f1": 0.168041, + "f1_weighted": 0.189212 + }, + { + "accuracy": 0.175464, + "f1": 0.167335, + "f1_weighted": 0.172631 + }, + { + "accuracy": 0.2097, + "f1": 0.173196, + "f1_weighted": 0.161819 + }, + { + "accuracy": 0.179743, + "f1": 0.148619, + "f1_weighted": 0.154023 + }, + { + "accuracy": 0.208274, + "f1": 0.14497, + "f1_weighted": 0.171998 + }, + { + "accuracy": 0.154066, + "f1": 0.138215, + "f1_weighted": 0.142213 + }, + { + "accuracy": 0.189729, + "f1": 0.131485, + "f1_weighted": 0.144139 + }, + { + "accuracy": 0.249643, + "f1": 0.187495, + "f1_weighted": 0.218111 + }, + { + "accuracy": 0.165478, + "f1": 0.146381, + "f1_weighted": 0.127655 + }, + { + "accuracy": 0.195435, + "f1": 0.158284, + "f1_weighted": 0.164953 + } + ], + "main_score": 0.19572, + "hf_subset": "nqo_Nkoo", + "languages": [ + "nqo-Nkoo" + ] + }, + { + "accuracy": 0.362054, + "f1": 0.350351, + "f1_weighted": 0.367235, + "scores_per_experiment": [ + { + "accuracy": 0.385164, + "f1": 0.370399, + "f1_weighted": 0.392071 + }, + { + "accuracy": 0.318117, + "f1": 0.317056, + "f1_weighted": 0.321717 + }, + { + "accuracy": 0.313837, + "f1": 0.307529, + "f1_weighted": 0.320221 + }, + { + "accuracy": 0.378031, + "f1": 0.355192, + "f1_weighted": 0.384247 + }, + { + "accuracy": 0.375178, + "f1": 0.351824, + "f1_weighted": 0.376582 + }, + { + "accuracy": 0.336662, + "f1": 0.323065, + "f1_weighted": 0.340644 + }, + { + "accuracy": 0.393723, + "f1": 0.382138, + "f1_weighted": 0.396819 + }, + { + "accuracy": 0.396576, + "f1": 0.391608, + "f1_weighted": 0.405394 + }, + { + "accuracy": 0.35378, + "f1": 0.33941, + "f1_weighted": 0.358225 + }, + { + "accuracy": 0.369472, + "f1": 0.365289, + "f1_weighted": 0.376425 + } + ], + "main_score": 0.362054, + "hf_subset": "nso_Latn", + "languages": [ + "nso-Latn" + ] + }, + { + "accuracy": 0.291013, + "f1": 0.286008, + "f1_weighted": 0.293232, + "scores_per_experiment": [ + { + "accuracy": 0.291013, + "f1": 0.288249, + "f1_weighted": 0.294296 + }, + { + "accuracy": 0.245364, + "f1": 0.249308, + "f1_weighted": 0.242772 + }, + { + "accuracy": 0.262482, + "f1": 0.261148, + "f1_weighted": 0.263253 + }, + { + "accuracy": 0.329529, + "f1": 0.31206, + "f1_weighted": 0.331697 + }, + { + "accuracy": 0.295292, + "f1": 0.295721, + "f1_weighted": 0.29867 + }, + { + "accuracy": 0.28816, + "f1": 0.280226, + "f1_weighted": 0.289383 + }, + { + "accuracy": 0.265335, + "f1": 0.264059, + "f1_weighted": 0.264642 + }, + { + "accuracy": 0.326676, + "f1": 0.318089, + "f1_weighted": 0.331697 + }, + { + "accuracy": 0.315264, + "f1": 0.302439, + "f1_weighted": 0.320317 + }, + { + "accuracy": 0.291013, + "f1": 0.288778, + "f1_weighted": 0.295592 + } + ], + "main_score": 0.291013, + "hf_subset": "nus_Latn", + "languages": [ + "nus-Latn" + ] + }, + { + "accuracy": 0.398146, + "f1": 0.381556, + "f1_weighted": 0.405002, + "scores_per_experiment": [ + { + "accuracy": 0.435093, + "f1": 0.415326, + "f1_weighted": 0.446031 + }, + { + "accuracy": 0.389444, + "f1": 0.374091, + "f1_weighted": 0.400779 + }, + { + "accuracy": 0.336662, + "f1": 0.326978, + "f1_weighted": 0.334659 + }, + { + "accuracy": 0.396576, + "f1": 0.373696, + "f1_weighted": 0.398296 + }, + { + "accuracy": 0.407989, + "f1": 0.385197, + "f1_weighted": 0.417383 + }, + { + "accuracy": 0.35378, + "f1": 0.334626, + "f1_weighted": 0.347338 + }, + { + "accuracy": 0.405136, + "f1": 0.38935, + "f1_weighted": 0.42108 + }, + { + "accuracy": 0.446505, + "f1": 0.436943, + "f1_weighted": 0.459642 + }, + { + "accuracy": 0.380884, + "f1": 0.376467, + "f1_weighted": 0.393847 + }, + { + "accuracy": 0.429387, + "f1": 0.402883, + "f1_weighted": 0.430964 + } + ], + "main_score": 0.398146, + "hf_subset": "nya_Latn", + "languages": [ + "nya-Latn" + ] + }, + { + "accuracy": 0.522111, + "f1": 0.509071, + "f1_weighted": 0.526028, + "scores_per_experiment": [ + { + "accuracy": 0.547789, + "f1": 0.536124, + "f1_weighted": 0.554847 + }, + { + "accuracy": 0.489301, + "f1": 0.479379, + "f1_weighted": 0.492571 + }, + { + "accuracy": 0.46933, + "f1": 0.463868, + "f1_weighted": 0.467647 + }, + { + "accuracy": 0.554922, + "f1": 0.539406, + "f1_weighted": 0.559976 + }, + { + "accuracy": 0.543509, + "f1": 0.538682, + "f1_weighted": 0.548447 + }, + { + "accuracy": 0.506419, + "f1": 0.490224, + "f1_weighted": 0.510627 + }, + { + "accuracy": 0.536377, + "f1": 0.513134, + "f1_weighted": 0.537221 + }, + { + "accuracy": 0.529244, + "f1": 0.522078, + "f1_weighted": 0.53673 + }, + { + "accuracy": 0.510699, + "f1": 0.478738, + "f1_weighted": 0.512516 + }, + { + "accuracy": 0.533524, + "f1": 0.529074, + "f1_weighted": 0.539695 + } + ], + "main_score": 0.522111, + "hf_subset": "oci_Latn", + "languages": [ + "oci-Latn" + ] + }, + { + "accuracy": 0.191441, + "f1": 0.176078, + "f1_weighted": 0.17366, + "scores_per_experiment": [ + { + "accuracy": 0.219686, + "f1": 0.201021, + "f1_weighted": 0.202041 + }, + { + "accuracy": 0.174037, + "f1": 0.151815, + "f1_weighted": 0.146905 + }, + { + "accuracy": 0.152639, + "f1": 0.145606, + "f1_weighted": 0.124495 + }, + { + "accuracy": 0.191155, + "f1": 0.180309, + "f1_weighted": 0.17333 + }, + { + "accuracy": 0.162625, + "f1": 0.160256, + "f1_weighted": 0.144305 + }, + { + "accuracy": 0.198288, + "f1": 0.171498, + "f1_weighted": 0.189625 + }, + { + "accuracy": 0.198288, + "f1": 0.181775, + "f1_weighted": 0.179727 + }, + { + "accuracy": 0.223966, + "f1": 0.21506, + "f1_weighted": 0.219791 + }, + { + "accuracy": 0.179743, + "f1": 0.16908, + "f1_weighted": 0.163426 + }, + { + "accuracy": 0.21398, + "f1": 0.184357, + "f1_weighted": 0.192952 + } + ], + "main_score": 0.191441, + "hf_subset": "ory_Orya", + "languages": [ + "ory-Orya" + ] + }, + { + "accuracy": 0.501141, + "f1": 0.487054, + "f1_weighted": 0.507131, + "scores_per_experiment": [ + { + "accuracy": 0.553495, + "f1": 0.537979, + "f1_weighted": 0.561338 + }, + { + "accuracy": 0.507846, + "f1": 0.495256, + "f1_weighted": 0.511127 + }, + { + "accuracy": 0.480742, + "f1": 0.471326, + "f1_weighted": 0.488442 + }, + { + "accuracy": 0.487874, + "f1": 0.475618, + "f1_weighted": 0.492441 + }, + { + "accuracy": 0.477889, + "f1": 0.473994, + "f1_weighted": 0.485192 + }, + { + "accuracy": 0.49786, + "f1": 0.479968, + "f1_weighted": 0.499621 + }, + { + "accuracy": 0.50214, + "f1": 0.479154, + "f1_weighted": 0.507397 + }, + { + "accuracy": 0.512126, + "f1": 0.494667, + "f1_weighted": 0.521309 + }, + { + "accuracy": 0.483595, + "f1": 0.464095, + "f1_weighted": 0.482919 + }, + { + "accuracy": 0.507846, + "f1": 0.498485, + "f1_weighted": 0.521527 + } + ], + "main_score": 0.501141, + "hf_subset": "pag_Latn", + "languages": [ + "pag-Latn" + ] + }, + { + "accuracy": 0.191583, + "f1": 0.163237, + "f1_weighted": 0.16125, + "scores_per_experiment": [ + { + "accuracy": 0.192582, + "f1": 0.171395, + "f1_weighted": 0.171286 + }, + { + "accuracy": 0.165478, + "f1": 0.134277, + "f1_weighted": 0.125392 + }, + { + "accuracy": 0.158345, + "f1": 0.15731, + "f1_weighted": 0.147837 + }, + { + "accuracy": 0.178317, + "f1": 0.155791, + "f1_weighted": 0.142757 + }, + { + "accuracy": 0.203994, + "f1": 0.160367, + "f1_weighted": 0.157245 + }, + { + "accuracy": 0.171184, + "f1": 0.171231, + "f1_weighted": 0.163255 + }, + { + "accuracy": 0.2097, + "f1": 0.158922, + "f1_weighted": 0.163352 + }, + { + "accuracy": 0.208274, + "f1": 0.171279, + "f1_weighted": 0.170596 + }, + { + "accuracy": 0.216833, + "f1": 0.176403, + "f1_weighted": 0.183328 + }, + { + "accuracy": 0.211127, + "f1": 0.175395, + "f1_weighted": 0.187448 + } + ], + "main_score": 0.191583, + "hf_subset": "pan_Guru", + "languages": [ + "pan-Guru" + ] + }, + { + "accuracy": 0.458345, + "f1": 0.446933, + "f1_weighted": 0.463057, + "scores_per_experiment": [ + { + "accuracy": 0.49786, + "f1": 0.486057, + "f1_weighted": 0.503017 + }, + { + "accuracy": 0.436519, + "f1": 0.424708, + "f1_weighted": 0.445886 + }, + { + "accuracy": 0.420827, + "f1": 0.416636, + "f1_weighted": 0.422042 + }, + { + "accuracy": 0.487874, + "f1": 0.466262, + "f1_weighted": 0.49213 + }, + { + "accuracy": 0.46077, + "f1": 0.451148, + "f1_weighted": 0.472585 + }, + { + "accuracy": 0.457917, + "f1": 0.450682, + "f1_weighted": 0.459081 + }, + { + "accuracy": 0.46933, + "f1": 0.451266, + "f1_weighted": 0.469871 + }, + { + "accuracy": 0.482168, + "f1": 0.470487, + "f1_weighted": 0.484141 + }, + { + "accuracy": 0.420827, + "f1": 0.41284, + "f1_weighted": 0.425791 + }, + { + "accuracy": 0.449358, + "f1": 0.439242, + "f1_weighted": 0.456024 + } + ], + "main_score": 0.458345, + "hf_subset": "pap_Latn", + "languages": [ + "pap-Latn" + ] + }, + { + "accuracy": 0.261626, + "f1": 0.199973, + "f1_weighted": 0.212134, + "scores_per_experiment": [ + { + "accuracy": 0.289586, + "f1": 0.241034, + "f1_weighted": 0.229192 + }, + { + "accuracy": 0.266762, + "f1": 0.187078, + "f1_weighted": 0.194152 + }, + { + "accuracy": 0.212553, + "f1": 0.169168, + "f1_weighted": 0.185892 + }, + { + "accuracy": 0.276748, + "f1": 0.190361, + "f1_weighted": 0.202517 + }, + { + "accuracy": 0.296719, + "f1": 0.236979, + "f1_weighted": 0.23397 + }, + { + "accuracy": 0.212553, + "f1": 0.208238, + "f1_weighted": 0.213978 + }, + { + "accuracy": 0.261056, + "f1": 0.182975, + "f1_weighted": 0.197868 + }, + { + "accuracy": 0.223966, + "f1": 0.178763, + "f1_weighted": 0.183847 + }, + { + "accuracy": 0.271041, + "f1": 0.176959, + "f1_weighted": 0.210862 + }, + { + "accuracy": 0.305278, + "f1": 0.228174, + "f1_weighted": 0.269064 + } + ], + "main_score": 0.261626, + "hf_subset": "pbt_Arab", + "languages": [ + "pbt-Arab" + ] + }, + { + "accuracy": 0.213124, + "f1": 0.182502, + "f1_weighted": 0.178634, + "scores_per_experiment": [ + { + "accuracy": 0.265335, + "f1": 0.192022, + "f1_weighted": 0.198635 + }, + { + "accuracy": 0.165478, + "f1": 0.123962, + "f1_weighted": 0.108609 + }, + { + "accuracy": 0.172611, + "f1": 0.180554, + "f1_weighted": 0.171359 + }, + { + "accuracy": 0.279601, + "f1": 0.226222, + "f1_weighted": 0.236013 + }, + { + "accuracy": 0.239658, + "f1": 0.185182, + "f1_weighted": 0.175984 + }, + { + "accuracy": 0.18117, + "f1": 0.186332, + "f1_weighted": 0.170166 + }, + { + "accuracy": 0.185449, + "f1": 0.167222, + "f1_weighted": 0.168362 + }, + { + "accuracy": 0.281027, + "f1": 0.22636, + "f1_weighted": 0.24578 + }, + { + "accuracy": 0.191155, + "f1": 0.184429, + "f1_weighted": 0.17199 + }, + { + "accuracy": 0.169757, + "f1": 0.152731, + "f1_weighted": 0.139447 + } + ], + "main_score": 0.213124, + "hf_subset": "pes_Arab", + "languages": [ + "pes-Arab" + ] + }, + { + "accuracy": 0.355064, + "f1": 0.349306, + "f1_weighted": 0.353068, + "scores_per_experiment": [ + { + "accuracy": 0.363766, + "f1": 0.369301, + "f1_weighted": 0.365935 + }, + { + "accuracy": 0.326676, + "f1": 0.327618, + "f1_weighted": 0.33068 + }, + { + "accuracy": 0.338088, + "f1": 0.337747, + "f1_weighted": 0.336346 + }, + { + "accuracy": 0.39515, + "f1": 0.378212, + "f1_weighted": 0.39293 + }, + { + "accuracy": 0.375178, + "f1": 0.371963, + "f1_weighted": 0.372704 + }, + { + "accuracy": 0.299572, + "f1": 0.298465, + "f1_weighted": 0.295238 + }, + { + "accuracy": 0.378031, + "f1": 0.360059, + "f1_weighted": 0.370032 + }, + { + "accuracy": 0.366619, + "f1": 0.359623, + "f1_weighted": 0.360522 + }, + { + "accuracy": 0.350927, + "f1": 0.338236, + "f1_weighted": 0.351696 + }, + { + "accuracy": 0.356633, + "f1": 0.351837, + "f1_weighted": 0.3546 + } + ], + "main_score": 0.355064, + "hf_subset": "plt_Latn", + "languages": [ + "plt-Latn" + ] + }, + { + "accuracy": 0.381027, + "f1": 0.372066, + "f1_weighted": 0.382116, + "scores_per_experiment": [ + { + "accuracy": 0.409415, + "f1": 0.408619, + "f1_weighted": 0.423773 + }, + { + "accuracy": 0.35378, + "f1": 0.341189, + "f1_weighted": 0.360178 + }, + { + "accuracy": 0.328103, + "f1": 0.325308, + "f1_weighted": 0.329401 + }, + { + "accuracy": 0.416548, + "f1": 0.404736, + "f1_weighted": 0.418566 + }, + { + "accuracy": 0.363766, + "f1": 0.355721, + "f1_weighted": 0.360711 + }, + { + "accuracy": 0.359486, + "f1": 0.35584, + "f1_weighted": 0.354031 + }, + { + "accuracy": 0.368046, + "f1": 0.356545, + "f1_weighted": 0.354539 + }, + { + "accuracy": 0.429387, + "f1": 0.410205, + "f1_weighted": 0.43054 + }, + { + "accuracy": 0.393723, + "f1": 0.382601, + "f1_weighted": 0.392002 + }, + { + "accuracy": 0.388017, + "f1": 0.37989, + "f1_weighted": 0.397417 + } + ], + "main_score": 0.381027, + "hf_subset": "pol_Latn", + "languages": [ + "pol-Latn" + ] + }, + { + "accuracy": 0.589444, + "f1": 0.570938, + "f1_weighted": 0.591332, + "scores_per_experiment": [ + { + "accuracy": 0.616262, + "f1": 0.597359, + "f1_weighted": 0.622005 + }, + { + "accuracy": 0.57632, + "f1": 0.552253, + "f1_weighted": 0.579172 + }, + { + "accuracy": 0.562054, + "f1": 0.552507, + "f1_weighted": 0.56029 + }, + { + "accuracy": 0.613409, + "f1": 0.596702, + "f1_weighted": 0.616229 + }, + { + "accuracy": 0.616262, + "f1": 0.590339, + "f1_weighted": 0.619049 + }, + { + "accuracy": 0.560628, + "f1": 0.543156, + "f1_weighted": 0.565516 + }, + { + "accuracy": 0.586305, + "f1": 0.5693, + "f1_weighted": 0.585545 + }, + { + "accuracy": 0.590585, + "f1": 0.573557, + "f1_weighted": 0.59304 + }, + { + "accuracy": 0.573466, + "f1": 0.550735, + "f1_weighted": 0.573459 + }, + { + "accuracy": 0.599144, + "f1": 0.583472, + "f1_weighted": 0.599018 + } + ], + "main_score": 0.589444, + "hf_subset": "por_Latn", + "languages": [ + "por-Latn" + ] + }, + { + "accuracy": 0.21826, + "f1": 0.174756, + "f1_weighted": 0.176762, + "scores_per_experiment": [ + { + "accuracy": 0.265335, + "f1": 0.191774, + "f1_weighted": 0.204025 + }, + { + "accuracy": 0.165478, + "f1": 0.145566, + "f1_weighted": 0.119323 + }, + { + "accuracy": 0.169757, + "f1": 0.159999, + "f1_weighted": 0.152215 + }, + { + "accuracy": 0.241084, + "f1": 0.158642, + "f1_weighted": 0.155535 + }, + { + "accuracy": 0.211127, + "f1": 0.175499, + "f1_weighted": 0.175457 + }, + { + "accuracy": 0.211127, + "f1": 0.178945, + "f1_weighted": 0.179161 + }, + { + "accuracy": 0.236805, + "f1": 0.193121, + "f1_weighted": 0.210737 + }, + { + "accuracy": 0.282454, + "f1": 0.216172, + "f1_weighted": 0.245775 + }, + { + "accuracy": 0.169757, + "f1": 0.133727, + "f1_weighted": 0.120109 + }, + { + "accuracy": 0.229672, + "f1": 0.194119, + "f1_weighted": 0.20528 + } + ], + "main_score": 0.21826, + "hf_subset": "prs_Arab", + "languages": [ + "prs-Arab" + ] + }, + { + "accuracy": 0.360913, + "f1": 0.352494, + "f1_weighted": 0.366281, + "scores_per_experiment": [ + { + "accuracy": 0.396576, + "f1": 0.39251, + "f1_weighted": 0.404222 + }, + { + "accuracy": 0.308131, + "f1": 0.30357, + "f1_weighted": 0.32041 + }, + { + "accuracy": 0.365193, + "f1": 0.343598, + "f1_weighted": 0.362951 + }, + { + "accuracy": 0.368046, + "f1": 0.366999, + "f1_weighted": 0.378836 + }, + { + "accuracy": 0.369472, + "f1": 0.360889, + "f1_weighted": 0.378001 + }, + { + "accuracy": 0.35378, + "f1": 0.349678, + "f1_weighted": 0.353116 + }, + { + "accuracy": 0.35806, + "f1": 0.346492, + "f1_weighted": 0.367771 + }, + { + "accuracy": 0.369472, + "f1": 0.366322, + "f1_weighted": 0.373311 + }, + { + "accuracy": 0.386591, + "f1": 0.368666, + "f1_weighted": 0.391396 + }, + { + "accuracy": 0.333809, + "f1": 0.326215, + "f1_weighted": 0.3328 + } + ], + "main_score": 0.360913, + "hf_subset": "quy_Latn", + "languages": [ + "quy-Latn" + ] + }, + { + "accuracy": 0.598003, + "f1": 0.584779, + "f1_weighted": 0.601016, + "scores_per_experiment": [ + { + "accuracy": 0.596291, + "f1": 0.584576, + "f1_weighted": 0.599451 + }, + { + "accuracy": 0.56776, + "f1": 0.560701, + "f1_weighted": 0.569934 + }, + { + "accuracy": 0.564907, + "f1": 0.563279, + "f1_weighted": 0.566054 + }, + { + "accuracy": 0.626248, + "f1": 0.593444, + "f1_weighted": 0.625346 + }, + { + "accuracy": 0.624822, + "f1": 0.612473, + "f1_weighted": 0.631131 + }, + { + "accuracy": 0.600571, + "f1": 0.593047, + "f1_weighted": 0.604378 + }, + { + "accuracy": 0.590585, + "f1": 0.577771, + "f1_weighted": 0.589397 + }, + { + "accuracy": 0.611983, + "f1": 0.593415, + "f1_weighted": 0.614975 + }, + { + "accuracy": 0.60485, + "f1": 0.587706, + "f1_weighted": 0.606105 + }, + { + "accuracy": 0.592011, + "f1": 0.581376, + "f1_weighted": 0.603389 + } + ], + "main_score": 0.598003, + "hf_subset": "ron_Latn", + "languages": [ + "ron-Latn" + ] + }, + { + "accuracy": 0.335521, + "f1": 0.327597, + "f1_weighted": 0.336293, + "scores_per_experiment": [ + { + "accuracy": 0.379458, + "f1": 0.363225, + "f1_weighted": 0.379846 + }, + { + "accuracy": 0.322397, + "f1": 0.321007, + "f1_weighted": 0.321281 + }, + { + "accuracy": 0.308131, + "f1": 0.308615, + "f1_weighted": 0.298749 + }, + { + "accuracy": 0.352354, + "f1": 0.348792, + "f1_weighted": 0.355197 + }, + { + "accuracy": 0.346648, + "f1": 0.331524, + "f1_weighted": 0.35533 + }, + { + "accuracy": 0.312411, + "f1": 0.300696, + "f1_weighted": 0.314602 + }, + { + "accuracy": 0.330956, + "f1": 0.31988, + "f1_weighted": 0.325722 + }, + { + "accuracy": 0.330956, + "f1": 0.333188, + "f1_weighted": 0.332136 + }, + { + "accuracy": 0.338088, + "f1": 0.327623, + "f1_weighted": 0.339858 + }, + { + "accuracy": 0.333809, + "f1": 0.321419, + "f1_weighted": 0.340206 + } + ], + "main_score": 0.335521, + "hf_subset": "run_Latn", + "languages": [ + "run-Latn" + ] + }, + { + "accuracy": 0.260485, + "f1": 0.250044, + "f1_weighted": 0.25222, + "scores_per_experiment": [ + { + "accuracy": 0.309558, + "f1": 0.296023, + "f1_weighted": 0.301643 + }, + { + "accuracy": 0.21826, + "f1": 0.202218, + "f1_weighted": 0.209978 + }, + { + "accuracy": 0.291013, + "f1": 0.284201, + "f1_weighted": 0.283225 + }, + { + "accuracy": 0.266762, + "f1": 0.259702, + "f1_weighted": 0.266967 + }, + { + "accuracy": 0.223966, + "f1": 0.216924, + "f1_weighted": 0.207084 + }, + { + "accuracy": 0.228245, + "f1": 0.224733, + "f1_weighted": 0.220125 + }, + { + "accuracy": 0.233951, + "f1": 0.217654, + "f1_weighted": 0.206628 + }, + { + "accuracy": 0.271041, + "f1": 0.266208, + "f1_weighted": 0.271525 + }, + { + "accuracy": 0.28816, + "f1": 0.271668, + "f1_weighted": 0.281923 + }, + { + "accuracy": 0.273894, + "f1": 0.261108, + "f1_weighted": 0.273104 + } + ], + "main_score": 0.260485, + "hf_subset": "rus_Cyrl", + "languages": [ + "rus-Cyrl" + ] + }, + { + "accuracy": 0.411127, + "f1": 0.39098, + "f1_weighted": 0.414663, + "scores_per_experiment": [ + { + "accuracy": 0.439372, + "f1": 0.423113, + "f1_weighted": 0.44136 + }, + { + "accuracy": 0.415121, + "f1": 0.395771, + "f1_weighted": 0.417424 + }, + { + "accuracy": 0.370899, + "f1": 0.360666, + "f1_weighted": 0.380235 + }, + { + "accuracy": 0.439372, + "f1": 0.420283, + "f1_weighted": 0.442202 + }, + { + "accuracy": 0.369472, + "f1": 0.350559, + "f1_weighted": 0.374542 + }, + { + "accuracy": 0.399429, + "f1": 0.378428, + "f1_weighted": 0.405424 + }, + { + "accuracy": 0.378031, + "f1": 0.356048, + "f1_weighted": 0.382766 + }, + { + "accuracy": 0.447932, + "f1": 0.422599, + "f1_weighted": 0.445651 + }, + { + "accuracy": 0.443652, + "f1": 0.412442, + "f1_weighted": 0.443008 + }, + { + "accuracy": 0.407989, + "f1": 0.389896, + "f1_weighted": 0.414019 + } + ], + "main_score": 0.411127, + "hf_subset": "sag_Latn", + "languages": [ + "sag-Latn" + ] + }, + { + "accuracy": 0.200856, + "f1": 0.180477, + "f1_weighted": 0.182658, + "scores_per_experiment": [ + { + "accuracy": 0.215407, + "f1": 0.189132, + "f1_weighted": 0.190326 + }, + { + "accuracy": 0.172611, + "f1": 0.164926, + "f1_weighted": 0.177088 + }, + { + "accuracy": 0.168331, + "f1": 0.173567, + "f1_weighted": 0.167873 + }, + { + "accuracy": 0.18117, + "f1": 0.155223, + "f1_weighted": 0.145136 + }, + { + "accuracy": 0.221113, + "f1": 0.203849, + "f1_weighted": 0.191426 + }, + { + "accuracy": 0.206847, + "f1": 0.201402, + "f1_weighted": 0.207666 + }, + { + "accuracy": 0.223966, + "f1": 0.190081, + "f1_weighted": 0.197444 + }, + { + "accuracy": 0.223966, + "f1": 0.18351, + "f1_weighted": 0.191603 + }, + { + "accuracy": 0.182596, + "f1": 0.171389, + "f1_weighted": 0.179617 + }, + { + "accuracy": 0.212553, + "f1": 0.17169, + "f1_weighted": 0.178398 + } + ], + "main_score": 0.200856, + "hf_subset": "san_Deva", + "languages": [ + "san-Deva" + ] + }, + { + "accuracy": 0.164765, + "f1": 0.127931, + "f1_weighted": 0.129253, + "scores_per_experiment": [ + { + "accuracy": 0.164051, + "f1": 0.118157, + "f1_weighted": 0.119333 + }, + { + "accuracy": 0.14408, + "f1": 0.122737, + "f1_weighted": 0.110883 + }, + { + "accuracy": 0.125535, + "f1": 0.108725, + "f1_weighted": 0.0984 + }, + { + "accuracy": 0.142653, + "f1": 0.115552, + "f1_weighted": 0.10701 + }, + { + "accuracy": 0.199715, + "f1": 0.176037, + "f1_weighted": 0.164853 + }, + { + "accuracy": 0.162625, + "f1": 0.137881, + "f1_weighted": 0.138689 + }, + { + "accuracy": 0.174037, + "f1": 0.128511, + "f1_weighted": 0.151417 + }, + { + "accuracy": 0.205421, + "f1": 0.126718, + "f1_weighted": 0.13342 + }, + { + "accuracy": 0.159772, + "f1": 0.11591, + "f1_weighted": 0.136502 + }, + { + "accuracy": 0.169757, + "f1": 0.129077, + "f1_weighted": 0.13202 + } + ], + "main_score": 0.164765, + "hf_subset": "sat_Olck", + "languages": [ + "sat-Olck" + ] + }, + { + "accuracy": 0.458345, + "f1": 0.440989, + "f1_weighted": 0.462637, + "scores_per_experiment": [ + { + "accuracy": 0.480742, + "f1": 0.463054, + "f1_weighted": 0.486774 + }, + { + "accuracy": 0.475036, + "f1": 0.454037, + "f1_weighted": 0.479691 + }, + { + "accuracy": 0.386591, + "f1": 0.37638, + "f1_weighted": 0.386687 + }, + { + "accuracy": 0.504993, + "f1": 0.476982, + "f1_weighted": 0.503297 + }, + { + "accuracy": 0.473609, + "f1": 0.448251, + "f1_weighted": 0.475505 + }, + { + "accuracy": 0.437946, + "f1": 0.425724, + "f1_weighted": 0.442841 + }, + { + "accuracy": 0.442225, + "f1": 0.424126, + "f1_weighted": 0.445846 + }, + { + "accuracy": 0.473609, + "f1": 0.453837, + "f1_weighted": 0.486307 + }, + { + "accuracy": 0.459344, + "f1": 0.445604, + "f1_weighted": 0.460578 + }, + { + "accuracy": 0.449358, + "f1": 0.441897, + "f1_weighted": 0.458845 + } + ], + "main_score": 0.458345, + "hf_subset": "scn_Latn", + "languages": [ + "scn-Latn" + ] + }, + { + "accuracy": 0.288873, + "f1": 0.269837, + "f1_weighted": 0.280254, + "scores_per_experiment": [ + { + "accuracy": 0.293866, + "f1": 0.308929, + "f1_weighted": 0.305607 + }, + { + "accuracy": 0.216833, + "f1": 0.205853, + "f1_weighted": 0.200668 + }, + { + "accuracy": 0.212553, + "f1": 0.204669, + "f1_weighted": 0.201599 + }, + { + "accuracy": 0.262482, + "f1": 0.270202, + "f1_weighted": 0.273868 + }, + { + "accuracy": 0.302425, + "f1": 0.263009, + "f1_weighted": 0.283343 + }, + { + "accuracy": 0.259629, + "f1": 0.213911, + "f1_weighted": 0.230892 + }, + { + "accuracy": 0.306705, + "f1": 0.26031, + "f1_weighted": 0.283598 + }, + { + "accuracy": 0.399429, + "f1": 0.381106, + "f1_weighted": 0.399327 + }, + { + "accuracy": 0.312411, + "f1": 0.295044, + "f1_weighted": 0.308842 + }, + { + "accuracy": 0.322397, + "f1": 0.295335, + "f1_weighted": 0.314796 + } + ], + "main_score": 0.288873, + "hf_subset": "shn_Mymr", + "languages": [ + "shn-Mymr" + ] + }, + { + "accuracy": 0.218117, + "f1": 0.178792, + "f1_weighted": 0.19057, + "scores_per_experiment": [ + { + "accuracy": 0.21398, + "f1": 0.182065, + "f1_weighted": 0.185961 + }, + { + "accuracy": 0.18117, + "f1": 0.136454, + "f1_weighted": 0.139971 + }, + { + "accuracy": 0.219686, + "f1": 0.201508, + "f1_weighted": 0.194863 + }, + { + "accuracy": 0.215407, + "f1": 0.192376, + "f1_weighted": 0.204295 + }, + { + "accuracy": 0.131241, + "f1": 0.108082, + "f1_weighted": 0.10072 + }, + { + "accuracy": 0.259629, + "f1": 0.191424, + "f1_weighted": 0.227326 + }, + { + "accuracy": 0.196862, + "f1": 0.154717, + "f1_weighted": 0.161808 + }, + { + "accuracy": 0.291013, + "f1": 0.235914, + "f1_weighted": 0.268006 + }, + { + "accuracy": 0.24679, + "f1": 0.180849, + "f1_weighted": 0.19962 + }, + { + "accuracy": 0.225392, + "f1": 0.204528, + "f1_weighted": 0.223128 + } + ], + "main_score": 0.218117, + "hf_subset": "sin_Sinh", + "languages": [ + "sin-Sinh" + ] + }, + { + "accuracy": 0.371755, + "f1": 0.363676, + "f1_weighted": 0.377705, + "scores_per_experiment": [ + { + "accuracy": 0.452211, + "f1": 0.442309, + "f1_weighted": 0.467364 + }, + { + "accuracy": 0.322397, + "f1": 0.314082, + "f1_weighted": 0.334704 + }, + { + "accuracy": 0.326676, + "f1": 0.32039, + "f1_weighted": 0.329068 + }, + { + "accuracy": 0.417974, + "f1": 0.410146, + "f1_weighted": 0.42538 + }, + { + "accuracy": 0.346648, + "f1": 0.348323, + "f1_weighted": 0.351968 + }, + { + "accuracy": 0.363766, + "f1": 0.35159, + "f1_weighted": 0.362776 + }, + { + "accuracy": 0.355207, + "f1": 0.346261, + "f1_weighted": 0.355353 + }, + { + "accuracy": 0.42368, + "f1": 0.401567, + "f1_weighted": 0.426229 + }, + { + "accuracy": 0.368046, + "f1": 0.35781, + "f1_weighted": 0.375288 + }, + { + "accuracy": 0.340942, + "f1": 0.344277, + "f1_weighted": 0.34892 + } + ], + "main_score": 0.371755, + "hf_subset": "slk_Latn", + "languages": [ + "slk-Latn" + ] + }, + { + "accuracy": 0.372896, + "f1": 0.36592, + "f1_weighted": 0.37587, + "scores_per_experiment": [ + { + "accuracy": 0.389444, + "f1": 0.392597, + "f1_weighted": 0.393933 + }, + { + "accuracy": 0.328103, + "f1": 0.317893, + "f1_weighted": 0.332794 + }, + { + "accuracy": 0.339515, + "f1": 0.334062, + "f1_weighted": 0.342615 + }, + { + "accuracy": 0.405136, + "f1": 0.397486, + "f1_weighted": 0.411035 + }, + { + "accuracy": 0.356633, + "f1": 0.348028, + "f1_weighted": 0.362748 + }, + { + "accuracy": 0.372325, + "f1": 0.371922, + "f1_weighted": 0.368008 + }, + { + "accuracy": 0.355207, + "f1": 0.347681, + "f1_weighted": 0.354275 + }, + { + "accuracy": 0.449358, + "f1": 0.43589, + "f1_weighted": 0.452252 + }, + { + "accuracy": 0.378031, + "f1": 0.368892, + "f1_weighted": 0.380759 + }, + { + "accuracy": 0.355207, + "f1": 0.344753, + "f1_weighted": 0.360281 + } + ], + "main_score": 0.372896, + "hf_subset": "slv_Latn", + "languages": [ + "slv-Latn" + ] + }, + { + "accuracy": 0.399001, + "f1": 0.385874, + "f1_weighted": 0.401974, + "scores_per_experiment": [ + { + "accuracy": 0.457917, + "f1": 0.425931, + "f1_weighted": 0.459008 + }, + { + "accuracy": 0.392297, + "f1": 0.380097, + "f1_weighted": 0.400063 + }, + { + "accuracy": 0.355207, + "f1": 0.351859, + "f1_weighted": 0.361746 + }, + { + "accuracy": 0.435093, + "f1": 0.416042, + "f1_weighted": 0.438399 + }, + { + "accuracy": 0.385164, + "f1": 0.368751, + "f1_weighted": 0.380066 + }, + { + "accuracy": 0.36234, + "f1": 0.356961, + "f1_weighted": 0.369535 + }, + { + "accuracy": 0.416548, + "f1": 0.397358, + "f1_weighted": 0.413438 + }, + { + "accuracy": 0.402282, + "f1": 0.397218, + "f1_weighted": 0.406359 + }, + { + "accuracy": 0.39087, + "f1": 0.374621, + "f1_weighted": 0.390699 + }, + { + "accuracy": 0.392297, + "f1": 0.389901, + "f1_weighted": 0.40043 + } + ], + "main_score": 0.399001, + "hf_subset": "smo_Latn", + "languages": [ + "smo-Latn" + ] + }, + { + "accuracy": 0.370328, + "f1": 0.361862, + "f1_weighted": 0.372917, + "scores_per_experiment": [ + { + "accuracy": 0.416548, + "f1": 0.407571, + "f1_weighted": 0.427764 + }, + { + "accuracy": 0.333809, + "f1": 0.328768, + "f1_weighted": 0.331571 + }, + { + "accuracy": 0.346648, + "f1": 0.339289, + "f1_weighted": 0.350059 + }, + { + "accuracy": 0.378031, + "f1": 0.36685, + "f1_weighted": 0.383701 + }, + { + "accuracy": 0.393723, + "f1": 0.37967, + "f1_weighted": 0.399547 + }, + { + "accuracy": 0.295292, + "f1": 0.290709, + "f1_weighted": 0.297804 + }, + { + "accuracy": 0.39515, + "f1": 0.384102, + "f1_weighted": 0.382977 + }, + { + "accuracy": 0.370899, + "f1": 0.367603, + "f1_weighted": 0.373086 + }, + { + "accuracy": 0.398003, + "f1": 0.382815, + "f1_weighted": 0.396827 + }, + { + "accuracy": 0.375178, + "f1": 0.371242, + "f1_weighted": 0.385835 + } + ], + "main_score": 0.370328, + "hf_subset": "sna_Latn", + "languages": [ + "sna-Latn" + ] + }, + { + "accuracy": 0.201141, + "f1": 0.160787, + "f1_weighted": 0.15595, + "scores_per_experiment": [ + { + "accuracy": 0.263909, + "f1": 0.184979, + "f1_weighted": 0.194826 + }, + { + "accuracy": 0.162625, + "f1": 0.117989, + "f1_weighted": 0.103153 + }, + { + "accuracy": 0.142653, + "f1": 0.143501, + "f1_weighted": 0.126913 + }, + { + "accuracy": 0.235378, + "f1": 0.147785, + "f1_weighted": 0.149685 + }, + { + "accuracy": 0.238231, + "f1": 0.180261, + "f1_weighted": 0.184986 + }, + { + "accuracy": 0.169757, + "f1": 0.158415, + "f1_weighted": 0.148192 + }, + { + "accuracy": 0.195435, + "f1": 0.179361, + "f1_weighted": 0.171849 + }, + { + "accuracy": 0.268188, + "f1": 0.201574, + "f1_weighted": 0.215266 + }, + { + "accuracy": 0.166904, + "f1": 0.145081, + "f1_weighted": 0.123744 + }, + { + "accuracy": 0.168331, + "f1": 0.14892, + "f1_weighted": 0.14088 + } + ], + "main_score": 0.201141, + "hf_subset": "snd_Arab", + "languages": [ + "snd-Arab" + ] + }, + { + "accuracy": 0.324251, + "f1": 0.317198, + "f1_weighted": 0.328751, + "scores_per_experiment": [ + { + "accuracy": 0.300999, + "f1": 0.304614, + "f1_weighted": 0.307544 + }, + { + "accuracy": 0.309558, + "f1": 0.304637, + "f1_weighted": 0.312378 + }, + { + "accuracy": 0.32525, + "f1": 0.312644, + "f1_weighted": 0.329569 + }, + { + "accuracy": 0.310984, + "f1": 0.307811, + "f1_weighted": 0.315066 + }, + { + "accuracy": 0.356633, + "f1": 0.346482, + "f1_weighted": 0.359096 + }, + { + "accuracy": 0.309558, + "f1": 0.291753, + "f1_weighted": 0.311867 + }, + { + "accuracy": 0.349501, + "f1": 0.334687, + "f1_weighted": 0.345832 + }, + { + "accuracy": 0.329529, + "f1": 0.327225, + "f1_weighted": 0.33562 + }, + { + "accuracy": 0.312411, + "f1": 0.308503, + "f1_weighted": 0.326966 + }, + { + "accuracy": 0.338088, + "f1": 0.333624, + "f1_weighted": 0.343573 + } + ], + "main_score": 0.324251, + "hf_subset": "som_Latn", + "languages": [ + "som-Latn" + ] + }, + { + "accuracy": 0.34893, + "f1": 0.339179, + "f1_weighted": 0.356829, + "scores_per_experiment": [ + { + "accuracy": 0.375178, + "f1": 0.369683, + "f1_weighted": 0.386516 + }, + { + "accuracy": 0.322397, + "f1": 0.321905, + "f1_weighted": 0.325611 + }, + { + "accuracy": 0.336662, + "f1": 0.328593, + "f1_weighted": 0.34526 + }, + { + "accuracy": 0.369472, + "f1": 0.357332, + "f1_weighted": 0.383014 + }, + { + "accuracy": 0.319544, + "f1": 0.317947, + "f1_weighted": 0.331645 + }, + { + "accuracy": 0.326676, + "f1": 0.319721, + "f1_weighted": 0.332508 + }, + { + "accuracy": 0.375178, + "f1": 0.349954, + "f1_weighted": 0.371213 + }, + { + "accuracy": 0.359486, + "f1": 0.340072, + "f1_weighted": 0.365038 + }, + { + "accuracy": 0.348074, + "f1": 0.342001, + "f1_weighted": 0.354044 + }, + { + "accuracy": 0.356633, + "f1": 0.344579, + "f1_weighted": 0.373445 + } + ], + "main_score": 0.34893, + "hf_subset": "sot_Latn", + "languages": [ + "sot-Latn" + ] + }, + { + "accuracy": 0.589016, + "f1": 0.570911, + "f1_weighted": 0.590126, + "scores_per_experiment": [ + { + "accuracy": 0.629101, + "f1": 0.605657, + "f1_weighted": 0.628758 + }, + { + "accuracy": 0.592011, + "f1": 0.579406, + "f1_weighted": 0.589327 + }, + { + "accuracy": 0.57204, + "f1": 0.561858, + "f1_weighted": 0.573405 + }, + { + "accuracy": 0.603424, + "f1": 0.580435, + "f1_weighted": 0.603877 + }, + { + "accuracy": 0.592011, + "f1": 0.571794, + "f1_weighted": 0.594213 + }, + { + "accuracy": 0.557775, + "f1": 0.541638, + "f1_weighted": 0.562276 + }, + { + "accuracy": 0.57204, + "f1": 0.559119, + "f1_weighted": 0.566748 + }, + { + "accuracy": 0.566334, + "f1": 0.549639, + "f1_weighted": 0.57081 + }, + { + "accuracy": 0.613409, + "f1": 0.588186, + "f1_weighted": 0.613633 + }, + { + "accuracy": 0.592011, + "f1": 0.571378, + "f1_weighted": 0.598213 + } + ], + "main_score": 0.589016, + "hf_subset": "spa_Latn", + "languages": [ + "spa-Latn" + ] + }, + { + "accuracy": 0.461626, + "f1": 0.451234, + "f1_weighted": 0.464619, + "scores_per_experiment": [ + { + "accuracy": 0.490728, + "f1": 0.478423, + "f1_weighted": 0.497614 + }, + { + "accuracy": 0.412268, + "f1": 0.408862, + "f1_weighted": 0.41647 + }, + { + "accuracy": 0.433666, + "f1": 0.425779, + "f1_weighted": 0.429546 + }, + { + "accuracy": 0.476462, + "f1": 0.467162, + "f1_weighted": 0.481646 + }, + { + "accuracy": 0.482168, + "f1": 0.466935, + "f1_weighted": 0.489666 + }, + { + "accuracy": 0.445078, + "f1": 0.437185, + "f1_weighted": 0.44754 + }, + { + "accuracy": 0.485021, + "f1": 0.471106, + "f1_weighted": 0.487524 + }, + { + "accuracy": 0.46505, + "f1": 0.456935, + "f1_weighted": 0.467788 + }, + { + "accuracy": 0.443652, + "f1": 0.422913, + "f1_weighted": 0.438822 + }, + { + "accuracy": 0.482168, + "f1": 0.477041, + "f1_weighted": 0.489569 + } + ], + "main_score": 0.461626, + "hf_subset": "srd_Latn", + "languages": [ + "srd-Latn" + ] + }, + { + "accuracy": 0.251213, + "f1": 0.232553, + "f1_weighted": 0.241338, + "scores_per_experiment": [ + { + "accuracy": 0.291013, + "f1": 0.258475, + "f1_weighted": 0.285218 + }, + { + "accuracy": 0.231098, + "f1": 0.2232, + "f1_weighted": 0.22892 + }, + { + "accuracy": 0.219686, + "f1": 0.220117, + "f1_weighted": 0.214832 + }, + { + "accuracy": 0.28816, + "f1": 0.254787, + "f1_weighted": 0.277841 + }, + { + "accuracy": 0.24679, + "f1": 0.22767, + "f1_weighted": 0.226845 + }, + { + "accuracy": 0.221113, + "f1": 0.213276, + "f1_weighted": 0.226014 + }, + { + "accuracy": 0.253923, + "f1": 0.221231, + "f1_weighted": 0.216186 + }, + { + "accuracy": 0.259629, + "f1": 0.237127, + "f1_weighted": 0.245669 + }, + { + "accuracy": 0.252496, + "f1": 0.253101, + "f1_weighted": 0.251971 + }, + { + "accuracy": 0.248217, + "f1": 0.216543, + "f1_weighted": 0.239884 + } + ], + "main_score": 0.251213, + "hf_subset": "srp_Cyrl", + "languages": [ + "srp-Cyrl" + ] + }, + { + "accuracy": 0.347218, + "f1": 0.340415, + "f1_weighted": 0.349434, + "scores_per_experiment": [ + { + "accuracy": 0.352354, + "f1": 0.348149, + "f1_weighted": 0.358978 + }, + { + "accuracy": 0.319544, + "f1": 0.324105, + "f1_weighted": 0.319598 + }, + { + "accuracy": 0.298146, + "f1": 0.289141, + "f1_weighted": 0.295762 + }, + { + "accuracy": 0.336662, + "f1": 0.330349, + "f1_weighted": 0.342869 + }, + { + "accuracy": 0.380884, + "f1": 0.366743, + "f1_weighted": 0.38412 + }, + { + "accuracy": 0.32525, + "f1": 0.319947, + "f1_weighted": 0.332077 + }, + { + "accuracy": 0.35378, + "f1": 0.335024, + "f1_weighted": 0.347727 + }, + { + "accuracy": 0.360913, + "f1": 0.346883, + "f1_weighted": 0.360567 + }, + { + "accuracy": 0.376605, + "f1": 0.376455, + "f1_weighted": 0.375915 + }, + { + "accuracy": 0.368046, + "f1": 0.367351, + "f1_weighted": 0.376731 + } + ], + "main_score": 0.347218, + "hf_subset": "ssw_Latn", + "languages": [ + "ssw-Latn" + ] + }, + { + "accuracy": 0.393723, + "f1": 0.383391, + "f1_weighted": 0.399623, + "scores_per_experiment": [ + { + "accuracy": 0.42796, + "f1": 0.419625, + "f1_weighted": 0.43667 + }, + { + "accuracy": 0.39515, + "f1": 0.375862, + "f1_weighted": 0.395888 + }, + { + "accuracy": 0.382311, + "f1": 0.371868, + "f1_weighted": 0.387754 + }, + { + "accuracy": 0.410842, + "f1": 0.397218, + "f1_weighted": 0.420572 + }, + { + "accuracy": 0.380884, + "f1": 0.369078, + "f1_weighted": 0.382381 + }, + { + "accuracy": 0.340942, + "f1": 0.342473, + "f1_weighted": 0.347449 + }, + { + "accuracy": 0.407989, + "f1": 0.392763, + "f1_weighted": 0.411117 + }, + { + "accuracy": 0.409415, + "f1": 0.40358, + "f1_weighted": 0.420295 + }, + { + "accuracy": 0.399429, + "f1": 0.387626, + "f1_weighted": 0.401403 + }, + { + "accuracy": 0.382311, + "f1": 0.373813, + "f1_weighted": 0.392699 + } + ], + "main_score": 0.393723, + "hf_subset": "sun_Latn", + "languages": [ + "sun-Latn" + ] + }, + { + "accuracy": 0.485307, + "f1": 0.469525, + "f1_weighted": 0.489201, + "scores_per_experiment": [ + { + "accuracy": 0.542083, + "f1": 0.528996, + "f1_weighted": 0.548883 + }, + { + "accuracy": 0.462197, + "f1": 0.448317, + "f1_weighted": 0.472586 + }, + { + "accuracy": 0.457917, + "f1": 0.451436, + "f1_weighted": 0.453786 + }, + { + "accuracy": 0.50214, + "f1": 0.471671, + "f1_weighted": 0.499871 + }, + { + "accuracy": 0.506419, + "f1": 0.497449, + "f1_weighted": 0.512751 + }, + { + "accuracy": 0.472183, + "f1": 0.454912, + "f1_weighted": 0.478191 + }, + { + "accuracy": 0.46933, + "f1": 0.452912, + "f1_weighted": 0.469588 + }, + { + "accuracy": 0.479315, + "f1": 0.466997, + "f1_weighted": 0.488605 + }, + { + "accuracy": 0.46077, + "f1": 0.439624, + "f1_weighted": 0.45696 + }, + { + "accuracy": 0.500713, + "f1": 0.482933, + "f1_weighted": 0.510792 + } + ], + "main_score": 0.485307, + "hf_subset": "swe_Latn", + "languages": [ + "swe-Latn" + ] + }, + { + "accuracy": 0.343224, + "f1": 0.337401, + "f1_weighted": 0.346233, + "scores_per_experiment": [ + { + "accuracy": 0.378031, + "f1": 0.365229, + "f1_weighted": 0.382236 + }, + { + "accuracy": 0.32097, + "f1": 0.322358, + "f1_weighted": 0.319432 + }, + { + "accuracy": 0.295292, + "f1": 0.297702, + "f1_weighted": 0.297555 + }, + { + "accuracy": 0.318117, + "f1": 0.304882, + "f1_weighted": 0.313438 + }, + { + "accuracy": 0.32525, + "f1": 0.315666, + "f1_weighted": 0.327169 + }, + { + "accuracy": 0.356633, + "f1": 0.349825, + "f1_weighted": 0.363917 + }, + { + "accuracy": 0.366619, + "f1": 0.353606, + "f1_weighted": 0.369856 + }, + { + "accuracy": 0.346648, + "f1": 0.354264, + "f1_weighted": 0.353438 + }, + { + "accuracy": 0.35806, + "f1": 0.346094, + "f1_weighted": 0.365082 + }, + { + "accuracy": 0.366619, + "f1": 0.364382, + "f1_weighted": 0.370204 + } + ], + "main_score": 0.343224, + "hf_subset": "swh_Latn", + "languages": [ + "swh-Latn" + ] + }, + { + "accuracy": 0.344365, + "f1": 0.340417, + "f1_weighted": 0.348601, + "scores_per_experiment": [ + { + "accuracy": 0.35378, + "f1": 0.359794, + "f1_weighted": 0.369542 + }, + { + "accuracy": 0.342368, + "f1": 0.33853, + "f1_weighted": 0.349435 + }, + { + "accuracy": 0.336662, + "f1": 0.328978, + "f1_weighted": 0.340929 + }, + { + "accuracy": 0.359486, + "f1": 0.363058, + "f1_weighted": 0.363922 + }, + { + "accuracy": 0.339515, + "f1": 0.335396, + "f1_weighted": 0.347933 + }, + { + "accuracy": 0.339515, + "f1": 0.330544, + "f1_weighted": 0.336622 + }, + { + "accuracy": 0.340942, + "f1": 0.332401, + "f1_weighted": 0.343469 + }, + { + "accuracy": 0.346648, + "f1": 0.346303, + "f1_weighted": 0.350804 + }, + { + "accuracy": 0.366619, + "f1": 0.345086, + "f1_weighted": 0.367809 + }, + { + "accuracy": 0.318117, + "f1": 0.324078, + "f1_weighted": 0.315543 + } + ], + "main_score": 0.344365, + "hf_subset": "szl_Latn", + "languages": [ + "szl-Latn" + ] + }, + { + "accuracy": 0.20485, + "f1": 0.177349, + "f1_weighted": 0.181784, + "scores_per_experiment": [ + { + "accuracy": 0.238231, + "f1": 0.194994, + "f1_weighted": 0.20028 + }, + { + "accuracy": 0.165478, + "f1": 0.142579, + "f1_weighted": 0.154564 + }, + { + "accuracy": 0.145506, + "f1": 0.143419, + "f1_weighted": 0.126962 + }, + { + "accuracy": 0.225392, + "f1": 0.205063, + "f1_weighted": 0.223152 + }, + { + "accuracy": 0.146933, + "f1": 0.141872, + "f1_weighted": 0.128316 + }, + { + "accuracy": 0.226819, + "f1": 0.178429, + "f1_weighted": 0.182122 + }, + { + "accuracy": 0.205421, + "f1": 0.170255, + "f1_weighted": 0.181559 + }, + { + "accuracy": 0.24679, + "f1": 0.213011, + "f1_weighted": 0.222096 + }, + { + "accuracy": 0.236805, + "f1": 0.199479, + "f1_weighted": 0.223388 + }, + { + "accuracy": 0.211127, + "f1": 0.184393, + "f1_weighted": 0.175396 + } + ], + "main_score": 0.20485, + "hf_subset": "tam_Taml", + "languages": [ + "tam-Taml" + ] + }, + { + "accuracy": 0.210842, + "f1": 0.192701, + "f1_weighted": 0.19185, + "scores_per_experiment": [ + { + "accuracy": 0.268188, + "f1": 0.224915, + "f1_weighted": 0.223622 + }, + { + "accuracy": 0.186876, + "f1": 0.174507, + "f1_weighted": 0.174799 + }, + { + "accuracy": 0.171184, + "f1": 0.176379, + "f1_weighted": 0.166963 + }, + { + "accuracy": 0.191155, + "f1": 0.190965, + "f1_weighted": 0.171936 + }, + { + "accuracy": 0.236805, + "f1": 0.206645, + "f1_weighted": 0.209388 + }, + { + "accuracy": 0.18117, + "f1": 0.172181, + "f1_weighted": 0.16482 + }, + { + "accuracy": 0.235378, + "f1": 0.204108, + "f1_weighted": 0.223933 + }, + { + "accuracy": 0.242511, + "f1": 0.227865, + "f1_weighted": 0.221907 + }, + { + "accuracy": 0.229672, + "f1": 0.207175, + "f1_weighted": 0.226012 + }, + { + "accuracy": 0.165478, + "f1": 0.142271, + "f1_weighted": 0.135122 + } + ], + "main_score": 0.210842, + "hf_subset": "taq_Tfng", + "languages": [ + "taq-Tfng" + ] + }, + { + "accuracy": 0.256205, + "f1": 0.241151, + "f1_weighted": 0.2509, + "scores_per_experiment": [ + { + "accuracy": 0.278174, + "f1": 0.242573, + "f1_weighted": 0.266916 + }, + { + "accuracy": 0.201141, + "f1": 0.200382, + "f1_weighted": 0.191123 + }, + { + "accuracy": 0.233951, + "f1": 0.227807, + "f1_weighted": 0.242278 + }, + { + "accuracy": 0.278174, + "f1": 0.255817, + "f1_weighted": 0.270502 + }, + { + "accuracy": 0.259629, + "f1": 0.254836, + "f1_weighted": 0.269088 + }, + { + "accuracy": 0.258203, + "f1": 0.243459, + "f1_weighted": 0.253003 + }, + { + "accuracy": 0.272468, + "f1": 0.264822, + "f1_weighted": 0.264843 + }, + { + "accuracy": 0.273894, + "f1": 0.251715, + "f1_weighted": 0.268322 + }, + { + "accuracy": 0.226819, + "f1": 0.217307, + "f1_weighted": 0.215179 + }, + { + "accuracy": 0.279601, + "f1": 0.252796, + "f1_weighted": 0.267745 + } + ], + "main_score": 0.256205, + "hf_subset": "tat_Cyrl", + "languages": [ + "tat-Cyrl" + ] + }, + { + "accuracy": 0.207275, + "f1": 0.180638, + "f1_weighted": 0.182529, + "scores_per_experiment": [ + { + "accuracy": 0.265335, + "f1": 0.197377, + "f1_weighted": 0.20544 + }, + { + "accuracy": 0.195435, + "f1": 0.175426, + "f1_weighted": 0.18139 + }, + { + "accuracy": 0.182596, + "f1": 0.18202, + "f1_weighted": 0.167442 + }, + { + "accuracy": 0.198288, + "f1": 0.197934, + "f1_weighted": 0.181745 + }, + { + "accuracy": 0.2097, + "f1": 0.18255, + "f1_weighted": 0.178958 + }, + { + "accuracy": 0.168331, + "f1": 0.15113, + "f1_weighted": 0.152261 + }, + { + "accuracy": 0.219686, + "f1": 0.165471, + "f1_weighted": 0.179461 + }, + { + "accuracy": 0.259629, + "f1": 0.211218, + "f1_weighted": 0.236001 + }, + { + "accuracy": 0.185449, + "f1": 0.170729, + "f1_weighted": 0.161212 + }, + { + "accuracy": 0.188302, + "f1": 0.172529, + "f1_weighted": 0.18138 + } + ], + "main_score": 0.207275, + "hf_subset": "tel_Telu", + "languages": [ + "tel-Telu" + ] + }, + { + "accuracy": 0.244223, + "f1": 0.231333, + "f1_weighted": 0.236044, + "scores_per_experiment": [ + { + "accuracy": 0.300999, + "f1": 0.279943, + "f1_weighted": 0.291087 + }, + { + "accuracy": 0.201141, + "f1": 0.200174, + "f1_weighted": 0.199453 + }, + { + "accuracy": 0.2097, + "f1": 0.217077, + "f1_weighted": 0.212903 + }, + { + "accuracy": 0.300999, + "f1": 0.273069, + "f1_weighted": 0.285778 + }, + { + "accuracy": 0.172611, + "f1": 0.165233, + "f1_weighted": 0.149005 + }, + { + "accuracy": 0.223966, + "f1": 0.221972, + "f1_weighted": 0.219988 + }, + { + "accuracy": 0.258203, + "f1": 0.231507, + "f1_weighted": 0.250589 + }, + { + "accuracy": 0.259629, + "f1": 0.25085, + "f1_weighted": 0.263467 + }, + { + "accuracy": 0.279601, + "f1": 0.254271, + "f1_weighted": 0.267937 + }, + { + "accuracy": 0.235378, + "f1": 0.21923, + "f1_weighted": 0.220234 + } + ], + "main_score": 0.244223, + "hf_subset": "tgk_Cyrl", + "languages": [ + "tgk-Cyrl" + ] + }, + { + "accuracy": 0.456776, + "f1": 0.440906, + "f1_weighted": 0.462962, + "scores_per_experiment": [ + { + "accuracy": 0.46505, + "f1": 0.457805, + "f1_weighted": 0.474538 + }, + { + "accuracy": 0.420827, + "f1": 0.387033, + "f1_weighted": 0.42152 + }, + { + "accuracy": 0.405136, + "f1": 0.394107, + "f1_weighted": 0.408686 + }, + { + "accuracy": 0.46077, + "f1": 0.432474, + "f1_weighted": 0.464329 + }, + { + "accuracy": 0.459344, + "f1": 0.448323, + "f1_weighted": 0.463314 + }, + { + "accuracy": 0.410842, + "f1": 0.400169, + "f1_weighted": 0.420098 + }, + { + "accuracy": 0.50214, + "f1": 0.478135, + "f1_weighted": 0.503565 + }, + { + "accuracy": 0.495007, + "f1": 0.483744, + "f1_weighted": 0.508621 + }, + { + "accuracy": 0.477889, + "f1": 0.462912, + "f1_weighted": 0.481352 + }, + { + "accuracy": 0.470756, + "f1": 0.464358, + "f1_weighted": 0.483594 + } + ], + "main_score": 0.456776, + "hf_subset": "tgl_Latn", + "languages": [ + "tgl-Latn" + ] + }, + { + "accuracy": 0.23495, + "f1": 0.199325, + "f1_weighted": 0.210451, + "scores_per_experiment": [ + { + "accuracy": 0.226819, + "f1": 0.226415, + "f1_weighted": 0.221006 + }, + { + "accuracy": 0.245364, + "f1": 0.188848, + "f1_weighted": 0.219301 + }, + { + "accuracy": 0.212553, + "f1": 0.189797, + "f1_weighted": 0.194194 + }, + { + "accuracy": 0.252496, + "f1": 0.232971, + "f1_weighted": 0.234227 + }, + { + "accuracy": 0.233951, + "f1": 0.177198, + "f1_weighted": 0.18196 + }, + { + "accuracy": 0.222539, + "f1": 0.163007, + "f1_weighted": 0.183954 + }, + { + "accuracy": 0.223966, + "f1": 0.181601, + "f1_weighted": 0.18872 + }, + { + "accuracy": 0.298146, + "f1": 0.246243, + "f1_weighted": 0.275245 + }, + { + "accuracy": 0.184023, + "f1": 0.178617, + "f1_weighted": 0.170414 + }, + { + "accuracy": 0.249643, + "f1": 0.20855, + "f1_weighted": 0.23549 + } + ], + "main_score": 0.23495, + "hf_subset": "tha_Thai", + "languages": [ + "tha-Thai" + ] + }, + { + "accuracy": 0.209558, + "f1": 0.16851, + "f1_weighted": 0.172061, + "scores_per_experiment": [ + { + "accuracy": 0.252496, + "f1": 0.170808, + "f1_weighted": 0.183258 + }, + { + "accuracy": 0.155492, + "f1": 0.136611, + "f1_weighted": 0.128045 + }, + { + "accuracy": 0.14408, + "f1": 0.121945, + "f1_weighted": 0.104669 + }, + { + "accuracy": 0.25107, + "f1": 0.179365, + "f1_weighted": 0.19487 + }, + { + "accuracy": 0.228245, + "f1": 0.207203, + "f1_weighted": 0.20297 + }, + { + "accuracy": 0.182596, + "f1": 0.173758, + "f1_weighted": 0.175713 + }, + { + "accuracy": 0.222539, + "f1": 0.164103, + "f1_weighted": 0.169626 + }, + { + "accuracy": 0.263909, + "f1": 0.22053, + "f1_weighted": 0.231385 + }, + { + "accuracy": 0.225392, + "f1": 0.166235, + "f1_weighted": 0.197898 + }, + { + "accuracy": 0.169757, + "f1": 0.144542, + "f1_weighted": 0.132171 + } + ], + "main_score": 0.209558, + "hf_subset": "tir_Ethi", + "languages": [ + "tir-Ethi" + ] + }, + { + "accuracy": 0.5398, + "f1": 0.525328, + "f1_weighted": 0.544856, + "scores_per_experiment": [ + { + "accuracy": 0.574893, + "f1": 0.562107, + "f1_weighted": 0.581598 + }, + { + "accuracy": 0.514979, + "f1": 0.485214, + "f1_weighted": 0.515314 + }, + { + "accuracy": 0.500713, + "f1": 0.495125, + "f1_weighted": 0.505639 + }, + { + "accuracy": 0.569187, + "f1": 0.562379, + "f1_weighted": 0.580896 + }, + { + "accuracy": 0.50214, + "f1": 0.491066, + "f1_weighted": 0.507537 + }, + { + "accuracy": 0.517832, + "f1": 0.509793, + "f1_weighted": 0.527575 + }, + { + "accuracy": 0.590585, + "f1": 0.566398, + "f1_weighted": 0.590965 + }, + { + "accuracy": 0.53923, + "f1": 0.531392, + "f1_weighted": 0.549282 + }, + { + "accuracy": 0.512126, + "f1": 0.485798, + "f1_weighted": 0.512386 + }, + { + "accuracy": 0.57632, + "f1": 0.56401, + "f1_weighted": 0.577374 + } + ], + "main_score": 0.5398, + "hf_subset": "tpi_Latn", + "languages": [ + "tpi-Latn" + ] + }, + { + "accuracy": 0.36234, + "f1": 0.352296, + "f1_weighted": 0.366489, + "scores_per_experiment": [ + { + "accuracy": 0.392297, + "f1": 0.387459, + "f1_weighted": 0.401725 + }, + { + "accuracy": 0.318117, + "f1": 0.315209, + "f1_weighted": 0.316286 + }, + { + "accuracy": 0.369472, + "f1": 0.351364, + "f1_weighted": 0.371247 + }, + { + "accuracy": 0.340942, + "f1": 0.325759, + "f1_weighted": 0.344845 + }, + { + "accuracy": 0.363766, + "f1": 0.35501, + "f1_weighted": 0.365697 + }, + { + "accuracy": 0.375178, + "f1": 0.358311, + "f1_weighted": 0.376291 + }, + { + "accuracy": 0.348074, + "f1": 0.344062, + "f1_weighted": 0.35382 + }, + { + "accuracy": 0.380884, + "f1": 0.372544, + "f1_weighted": 0.383391 + }, + { + "accuracy": 0.366619, + "f1": 0.353759, + "f1_weighted": 0.373965 + }, + { + "accuracy": 0.368046, + "f1": 0.359483, + "f1_weighted": 0.377622 + } + ], + "main_score": 0.36234, + "hf_subset": "tsn_Latn", + "languages": [ + "tsn-Latn" + ] + }, + { + "accuracy": 0.363623, + "f1": 0.357394, + "f1_weighted": 0.368198, + "scores_per_experiment": [ + { + "accuracy": 0.435093, + "f1": 0.419869, + "f1_weighted": 0.439599 + }, + { + "accuracy": 0.333809, + "f1": 0.331909, + "f1_weighted": 0.335887 + }, + { + "accuracy": 0.336662, + "f1": 0.332134, + "f1_weighted": 0.337701 + }, + { + "accuracy": 0.333809, + "f1": 0.328212, + "f1_weighted": 0.34266 + }, + { + "accuracy": 0.409415, + "f1": 0.396689, + "f1_weighted": 0.414651 + }, + { + "accuracy": 0.328103, + "f1": 0.322595, + "f1_weighted": 0.337241 + }, + { + "accuracy": 0.368046, + "f1": 0.353743, + "f1_weighted": 0.360359 + }, + { + "accuracy": 0.359486, + "f1": 0.367035, + "f1_weighted": 0.367129 + }, + { + "accuracy": 0.382311, + "f1": 0.371787, + "f1_weighted": 0.39084 + }, + { + "accuracy": 0.349501, + "f1": 0.349973, + "f1_weighted": 0.355909 + } + ], + "main_score": 0.363623, + "hf_subset": "tso_Latn", + "languages": [ + "tso-Latn" + ] + }, + { + "accuracy": 0.352068, + "f1": 0.341972, + "f1_weighted": 0.355558, + "scores_per_experiment": [ + { + "accuracy": 0.378031, + "f1": 0.367066, + "f1_weighted": 0.38527 + }, + { + "accuracy": 0.330956, + "f1": 0.320217, + "f1_weighted": 0.328816 + }, + { + "accuracy": 0.312411, + "f1": 0.309019, + "f1_weighted": 0.319044 + }, + { + "accuracy": 0.373752, + "f1": 0.362175, + "f1_weighted": 0.378385 + }, + { + "accuracy": 0.35378, + "f1": 0.346364, + "f1_weighted": 0.357492 + }, + { + "accuracy": 0.342368, + "f1": 0.327361, + "f1_weighted": 0.338581 + }, + { + "accuracy": 0.368046, + "f1": 0.3409, + "f1_weighted": 0.360159 + }, + { + "accuracy": 0.352354, + "f1": 0.353205, + "f1_weighted": 0.359457 + }, + { + "accuracy": 0.356633, + "f1": 0.342387, + "f1_weighted": 0.367 + }, + { + "accuracy": 0.352354, + "f1": 0.351028, + "f1_weighted": 0.361376 + } + ], + "main_score": 0.352068, + "hf_subset": "tuk_Latn", + "languages": [ + "tuk-Latn" + ] + }, + { + "accuracy": 0.411412, + "f1": 0.395923, + "f1_weighted": 0.417886, + "scores_per_experiment": [ + { + "accuracy": 0.437946, + "f1": 0.413003, + "f1_weighted": 0.445235 + }, + { + "accuracy": 0.386591, + "f1": 0.387893, + "f1_weighted": 0.395535 + }, + { + "accuracy": 0.383738, + "f1": 0.365983, + "f1_weighted": 0.389038 + }, + { + "accuracy": 0.378031, + "f1": 0.355523, + "f1_weighted": 0.388335 + }, + { + "accuracy": 0.442225, + "f1": 0.409689, + "f1_weighted": 0.438983 + }, + { + "accuracy": 0.388017, + "f1": 0.377489, + "f1_weighted": 0.392086 + }, + { + "accuracy": 0.425107, + "f1": 0.404946, + "f1_weighted": 0.417729 + }, + { + "accuracy": 0.429387, + "f1": 0.424705, + "f1_weighted": 0.440422 + }, + { + "accuracy": 0.433666, + "f1": 0.425086, + "f1_weighted": 0.443802 + }, + { + "accuracy": 0.409415, + "f1": 0.394917, + "f1_weighted": 0.427691 + } + ], + "main_score": 0.411412, + "hf_subset": "tum_Latn", + "languages": [ + "tum-Latn" + ] + }, + { + "accuracy": 0.353923, + "f1": 0.346404, + "f1_weighted": 0.356468, + "scores_per_experiment": [ + { + "accuracy": 0.379458, + "f1": 0.377223, + "f1_weighted": 0.386582 + }, + { + "accuracy": 0.295292, + "f1": 0.29556, + "f1_weighted": 0.296851 + }, + { + "accuracy": 0.328103, + "f1": 0.323419, + "f1_weighted": 0.334482 + }, + { + "accuracy": 0.392297, + "f1": 0.376712, + "f1_weighted": 0.395706 + }, + { + "accuracy": 0.328103, + "f1": 0.319881, + "f1_weighted": 0.319648 + }, + { + "accuracy": 0.350927, + "f1": 0.340514, + "f1_weighted": 0.350843 + }, + { + "accuracy": 0.380884, + "f1": 0.372687, + "f1_weighted": 0.382461 + }, + { + "accuracy": 0.373752, + "f1": 0.364894, + "f1_weighted": 0.381538 + }, + { + "accuracy": 0.375178, + "f1": 0.362185, + "f1_weighted": 0.373369 + }, + { + "accuracy": 0.335235, + "f1": 0.330968, + "f1_weighted": 0.343199 + } + ], + "main_score": 0.353923, + "hf_subset": "tur_Latn", + "languages": [ + "tur-Latn" + ] + }, + { + "accuracy": 0.419116, + "f1": 0.408351, + "f1_weighted": 0.421802, + "scores_per_experiment": [ + { + "accuracy": 0.436519, + "f1": 0.410413, + "f1_weighted": 0.437846 + }, + { + "accuracy": 0.383738, + "f1": 0.371445, + "f1_weighted": 0.39205 + }, + { + "accuracy": 0.422254, + "f1": 0.410979, + "f1_weighted": 0.416966 + }, + { + "accuracy": 0.42796, + "f1": 0.416722, + "f1_weighted": 0.435606 + }, + { + "accuracy": 0.389444, + "f1": 0.389458, + "f1_weighted": 0.387153 + }, + { + "accuracy": 0.39087, + "f1": 0.383076, + "f1_weighted": 0.393262 + }, + { + "accuracy": 0.437946, + "f1": 0.423179, + "f1_weighted": 0.435604 + }, + { + "accuracy": 0.439372, + "f1": 0.432001, + "f1_weighted": 0.441209 + }, + { + "accuracy": 0.445078, + "f1": 0.429575, + "f1_weighted": 0.448682 + }, + { + "accuracy": 0.417974, + "f1": 0.416666, + "f1_weighted": 0.429641 + } + ], + "main_score": 0.419116, + "hf_subset": "twi_Latn", + "languages": [ + "twi-Latn" + ] + }, + { + "accuracy": 0.190157, + "f1": 0.165728, + "f1_weighted": 0.161371, + "scores_per_experiment": [ + { + "accuracy": 0.211127, + "f1": 0.168938, + "f1_weighted": 0.171925 + }, + { + "accuracy": 0.194009, + "f1": 0.163845, + "f1_weighted": 0.15958 + }, + { + "accuracy": 0.149786, + "f1": 0.128689, + "f1_weighted": 0.103438 + }, + { + "accuracy": 0.175464, + "f1": 0.155127, + "f1_weighted": 0.138242 + }, + { + "accuracy": 0.185449, + "f1": 0.170901, + "f1_weighted": 0.183309 + }, + { + "accuracy": 0.206847, + "f1": 0.186119, + "f1_weighted": 0.199699 + }, + { + "accuracy": 0.194009, + "f1": 0.171392, + "f1_weighted": 0.163618 + }, + { + "accuracy": 0.211127, + "f1": 0.195701, + "f1_weighted": 0.1828 + }, + { + "accuracy": 0.172611, + "f1": 0.154943, + "f1_weighted": 0.150185 + }, + { + "accuracy": 0.201141, + "f1": 0.161627, + "f1_weighted": 0.160912 + } + ], + "main_score": 0.190157, + "hf_subset": "tzm_Tfng", + "languages": [ + "tzm-Tfng" + ] + }, + { + "accuracy": 0.196148, + "f1": 0.166247, + "f1_weighted": 0.167366, + "scores_per_experiment": [ + { + "accuracy": 0.261056, + "f1": 0.193325, + "f1_weighted": 0.19883 + }, + { + "accuracy": 0.146933, + "f1": 0.130428, + "f1_weighted": 0.111571 + }, + { + "accuracy": 0.14408, + "f1": 0.145648, + "f1_weighted": 0.135146 + }, + { + "accuracy": 0.261056, + "f1": 0.194904, + "f1_weighted": 0.215236 + }, + { + "accuracy": 0.179743, + "f1": 0.174814, + "f1_weighted": 0.151131 + }, + { + "accuracy": 0.174037, + "f1": 0.169474, + "f1_weighted": 0.1784 + }, + { + "accuracy": 0.203994, + "f1": 0.176756, + "f1_weighted": 0.180261 + }, + { + "accuracy": 0.24679, + "f1": 0.18867, + "f1_weighted": 0.217969 + }, + { + "accuracy": 0.18117, + "f1": 0.151668, + "f1_weighted": 0.153211 + }, + { + "accuracy": 0.162625, + "f1": 0.136788, + "f1_weighted": 0.131904 + } + ], + "main_score": 0.196148, + "hf_subset": "uig_Arab", + "languages": [ + "uig-Arab" + ] + }, + { + "accuracy": 0.259914, + "f1": 0.242399, + "f1_weighted": 0.254337, + "scores_per_experiment": [ + { + "accuracy": 0.28388, + "f1": 0.225278, + "f1_weighted": 0.252282 + }, + { + "accuracy": 0.206847, + "f1": 0.206458, + "f1_weighted": 0.202358 + }, + { + "accuracy": 0.263909, + "f1": 0.258656, + "f1_weighted": 0.266766 + }, + { + "accuracy": 0.259629, + "f1": 0.24935, + "f1_weighted": 0.259643 + }, + { + "accuracy": 0.231098, + "f1": 0.224711, + "f1_weighted": 0.21965 + }, + { + "accuracy": 0.222539, + "f1": 0.209476, + "f1_weighted": 0.225283 + }, + { + "accuracy": 0.258203, + "f1": 0.231602, + "f1_weighted": 0.249335 + }, + { + "accuracy": 0.295292, + "f1": 0.268359, + "f1_weighted": 0.293379 + }, + { + "accuracy": 0.308131, + "f1": 0.296547, + "f1_weighted": 0.304386 + }, + { + "accuracy": 0.269615, + "f1": 0.253556, + "f1_weighted": 0.270284 + } + ], + "main_score": 0.259914, + "hf_subset": "ukr_Cyrl", + "languages": [ + "ukr-Cyrl" + ] + }, + { + "accuracy": 0.334094, + "f1": 0.327104, + "f1_weighted": 0.33929, + "scores_per_experiment": [ + { + "accuracy": 0.36234, + "f1": 0.358982, + "f1_weighted": 0.366819 + }, + { + "accuracy": 0.309558, + "f1": 0.308984, + "f1_weighted": 0.309798 + }, + { + "accuracy": 0.315264, + "f1": 0.314748, + "f1_weighted": 0.324708 + }, + { + "accuracy": 0.338088, + "f1": 0.327874, + "f1_weighted": 0.350048 + }, + { + "accuracy": 0.346648, + "f1": 0.335298, + "f1_weighted": 0.35336 + }, + { + "accuracy": 0.309558, + "f1": 0.30576, + "f1_weighted": 0.313185 + }, + { + "accuracy": 0.330956, + "f1": 0.319521, + "f1_weighted": 0.332279 + }, + { + "accuracy": 0.365193, + "f1": 0.356086, + "f1_weighted": 0.36628 + }, + { + "accuracy": 0.300999, + "f1": 0.29993, + "f1_weighted": 0.310734 + }, + { + "accuracy": 0.36234, + "f1": 0.343858, + "f1_weighted": 0.365685 + } + ], + "main_score": 0.334094, + "hf_subset": "umb_Latn", + "languages": [ + "umb-Latn" + ] + }, + { + "accuracy": 0.202853, + "f1": 0.164628, + "f1_weighted": 0.160412, + "scores_per_experiment": [ + { + "accuracy": 0.25535, + "f1": 0.190525, + "f1_weighted": 0.194102 + }, + { + "accuracy": 0.152639, + "f1": 0.131887, + "f1_weighted": 0.113873 + }, + { + "accuracy": 0.156919, + "f1": 0.13166, + "f1_weighted": 0.106592 + }, + { + "accuracy": 0.243937, + "f1": 0.177793, + "f1_weighted": 0.185085 + }, + { + "accuracy": 0.223966, + "f1": 0.166077, + "f1_weighted": 0.163252 + }, + { + "accuracy": 0.175464, + "f1": 0.161491, + "f1_weighted": 0.166762 + }, + { + "accuracy": 0.222539, + "f1": 0.175282, + "f1_weighted": 0.181661 + }, + { + "accuracy": 0.172611, + "f1": 0.155624, + "f1_weighted": 0.131466 + }, + { + "accuracy": 0.202568, + "f1": 0.173321, + "f1_weighted": 0.172443 + }, + { + "accuracy": 0.222539, + "f1": 0.182615, + "f1_weighted": 0.188886 + } + ], + "main_score": 0.202853, + "hf_subset": "urd_Arab", + "languages": [ + "urd-Arab" + ] + }, + { + "accuracy": 0.333951, + "f1": 0.330391, + "f1_weighted": 0.335316, + "scores_per_experiment": [ + { + "accuracy": 0.370899, + "f1": 0.367474, + "f1_weighted": 0.378324 + }, + { + "accuracy": 0.323823, + "f1": 0.325621, + "f1_weighted": 0.319258 + }, + { + "accuracy": 0.322397, + "f1": 0.320895, + "f1_weighted": 0.32438 + }, + { + "accuracy": 0.360913, + "f1": 0.349548, + "f1_weighted": 0.370157 + }, + { + "accuracy": 0.31669, + "f1": 0.316732, + "f1_weighted": 0.313013 + }, + { + "accuracy": 0.328103, + "f1": 0.326152, + "f1_weighted": 0.331807 + }, + { + "accuracy": 0.359486, + "f1": 0.348688, + "f1_weighted": 0.358501 + }, + { + "accuracy": 0.310984, + "f1": 0.311464, + "f1_weighted": 0.31819 + }, + { + "accuracy": 0.333809, + "f1": 0.328712, + "f1_weighted": 0.331483 + }, + { + "accuracy": 0.312411, + "f1": 0.308619, + "f1_weighted": 0.308044 + } + ], + "main_score": 0.333951, + "hf_subset": "uzn_Latn", + "languages": [ + "uzn-Latn" + ] + }, + { + "accuracy": 0.4903, + "f1": 0.472991, + "f1_weighted": 0.493507, + "scores_per_experiment": [ + { + "accuracy": 0.500713, + "f1": 0.486203, + "f1_weighted": 0.508273 + }, + { + "accuracy": 0.510699, + "f1": 0.48907, + "f1_weighted": 0.509861 + }, + { + "accuracy": 0.437946, + "f1": 0.425196, + "f1_weighted": 0.443172 + }, + { + "accuracy": 0.524964, + "f1": 0.503218, + "f1_weighted": 0.530542 + }, + { + "accuracy": 0.492154, + "f1": 0.47419, + "f1_weighted": 0.499568 + }, + { + "accuracy": 0.466476, + "f1": 0.456859, + "f1_weighted": 0.462427 + }, + { + "accuracy": 0.496434, + "f1": 0.472193, + "f1_weighted": 0.498394 + }, + { + "accuracy": 0.509272, + "f1": 0.488071, + "f1_weighted": 0.507282 + }, + { + "accuracy": 0.46077, + "f1": 0.444206, + "f1_weighted": 0.468516 + }, + { + "accuracy": 0.503566, + "f1": 0.490705, + "f1_weighted": 0.507034 + } + ], + "main_score": 0.4903, + "hf_subset": "vec_Latn", + "languages": [ + "vec-Latn" + ] + }, + { + "accuracy": 0.337375, + "f1": 0.327062, + "f1_weighted": 0.335327, + "scores_per_experiment": [ + { + "accuracy": 0.348074, + "f1": 0.350418, + "f1_weighted": 0.359267 + }, + { + "accuracy": 0.293866, + "f1": 0.293467, + "f1_weighted": 0.285437 + }, + { + "accuracy": 0.303852, + "f1": 0.294313, + "f1_weighted": 0.304965 + }, + { + "accuracy": 0.346648, + "f1": 0.331869, + "f1_weighted": 0.346531 + }, + { + "accuracy": 0.338088, + "f1": 0.327756, + "f1_weighted": 0.33843 + }, + { + "accuracy": 0.335235, + "f1": 0.294329, + "f1_weighted": 0.31446 + }, + { + "accuracy": 0.355207, + "f1": 0.329457, + "f1_weighted": 0.336588 + }, + { + "accuracy": 0.35806, + "f1": 0.361466, + "f1_weighted": 0.362686 + }, + { + "accuracy": 0.356633, + "f1": 0.35037, + "f1_weighted": 0.358625 + }, + { + "accuracy": 0.338088, + "f1": 0.337168, + "f1_weighted": 0.34628 + } + ], + "main_score": 0.337375, + "hf_subset": "vie_Latn", + "languages": [ + "vie-Latn" + ] + }, + { + "accuracy": 0.526676, + "f1": 0.509543, + "f1_weighted": 0.531785, + "scores_per_experiment": [ + { + "accuracy": 0.562054, + "f1": 0.544914, + "f1_weighted": 0.567838 + }, + { + "accuracy": 0.53495, + "f1": 0.498072, + "f1_weighted": 0.533551 + }, + { + "accuracy": 0.492154, + "f1": 0.491551, + "f1_weighted": 0.497912 + }, + { + "accuracy": 0.533524, + "f1": 0.519527, + "f1_weighted": 0.536719 + }, + { + "accuracy": 0.490728, + "f1": 0.475988, + "f1_weighted": 0.49821 + }, + { + "accuracy": 0.550642, + "f1": 0.530999, + "f1_weighted": 0.550105 + }, + { + "accuracy": 0.510699, + "f1": 0.484781, + "f1_weighted": 0.512896 + }, + { + "accuracy": 0.556348, + "f1": 0.550158, + "f1_weighted": 0.563985 + }, + { + "accuracy": 0.487874, + "f1": 0.468578, + "f1_weighted": 0.495725 + }, + { + "accuracy": 0.547789, + "f1": 0.530861, + "f1_weighted": 0.560911 + } + ], + "main_score": 0.526676, + "hf_subset": "war_Latn", + "languages": [ + "war-Latn" + ] + }, + { + "accuracy": 0.370471, + "f1": 0.358552, + "f1_weighted": 0.373948, + "scores_per_experiment": [ + { + "accuracy": 0.392297, + "f1": 0.384572, + "f1_weighted": 0.399271 + }, + { + "accuracy": 0.342368, + "f1": 0.343077, + "f1_weighted": 0.338559 + }, + { + "accuracy": 0.382311, + "f1": 0.374503, + "f1_weighted": 0.390485 + }, + { + "accuracy": 0.352354, + "f1": 0.338183, + "f1_weighted": 0.359419 + }, + { + "accuracy": 0.376605, + "f1": 0.357796, + "f1_weighted": 0.377376 + }, + { + "accuracy": 0.35378, + "f1": 0.349195, + "f1_weighted": 0.356991 + }, + { + "accuracy": 0.35806, + "f1": 0.342336, + "f1_weighted": 0.353597 + }, + { + "accuracy": 0.402282, + "f1": 0.390103, + "f1_weighted": 0.410014 + }, + { + "accuracy": 0.380884, + "f1": 0.363676, + "f1_weighted": 0.386132 + }, + { + "accuracy": 0.363766, + "f1": 0.342082, + "f1_weighted": 0.36764 + } + ], + "main_score": 0.370471, + "hf_subset": "wol_Latn", + "languages": [ + "wol-Latn" + ] + }, + { + "accuracy": 0.357347, + "f1": 0.349429, + "f1_weighted": 0.364519, + "scores_per_experiment": [ + { + "accuracy": 0.388017, + "f1": 0.375881, + "f1_weighted": 0.389629 + }, + { + "accuracy": 0.329529, + "f1": 0.324957, + "f1_weighted": 0.338768 + }, + { + "accuracy": 0.346648, + "f1": 0.338987, + "f1_weighted": 0.355105 + }, + { + "accuracy": 0.31669, + "f1": 0.310243, + "f1_weighted": 0.321022 + }, + { + "accuracy": 0.378031, + "f1": 0.367375, + "f1_weighted": 0.37703 + }, + { + "accuracy": 0.338088, + "f1": 0.324968, + "f1_weighted": 0.346414 + }, + { + "accuracy": 0.368046, + "f1": 0.355644, + "f1_weighted": 0.374752 + }, + { + "accuracy": 0.343795, + "f1": 0.352636, + "f1_weighted": 0.357225 + }, + { + "accuracy": 0.399429, + "f1": 0.384663, + "f1_weighted": 0.407961 + }, + { + "accuracy": 0.365193, + "f1": 0.358934, + "f1_weighted": 0.377288 + } + ], + "main_score": 0.357347, + "hf_subset": "xho_Latn", + "languages": [ + "xho-Latn" + ] + }, + { + "accuracy": 0.205849, + "f1": 0.179256, + "f1_weighted": 0.185741, + "scores_per_experiment": [ + { + "accuracy": 0.236805, + "f1": 0.191096, + "f1_weighted": 0.199735 + }, + { + "accuracy": 0.206847, + "f1": 0.169945, + "f1_weighted": 0.184998 + }, + { + "accuracy": 0.156919, + "f1": 0.149111, + "f1_weighted": 0.139509 + }, + { + "accuracy": 0.223966, + "f1": 0.207545, + "f1_weighted": 0.214256 + }, + { + "accuracy": 0.17689, + "f1": 0.161168, + "f1_weighted": 0.160262 + }, + { + "accuracy": 0.196862, + "f1": 0.187518, + "f1_weighted": 0.18298 + }, + { + "accuracy": 0.185449, + "f1": 0.164564, + "f1_weighted": 0.165998 + }, + { + "accuracy": 0.276748, + "f1": 0.214552, + "f1_weighted": 0.248112 + }, + { + "accuracy": 0.198288, + "f1": 0.187282, + "f1_weighted": 0.199877 + }, + { + "accuracy": 0.199715, + "f1": 0.159777, + "f1_weighted": 0.161687 + } + ], + "main_score": 0.205849, + "hf_subset": "ydd_Hebr", + "languages": [ + "ydd-Hebr" + ] + }, + { + "accuracy": 0.299857, + "f1": 0.295877, + "f1_weighted": 0.299938, + "scores_per_experiment": [ + { + "accuracy": 0.315264, + "f1": 0.311166, + "f1_weighted": 0.313266 + }, + { + "accuracy": 0.275321, + "f1": 0.275947, + "f1_weighted": 0.27396 + }, + { + "accuracy": 0.266762, + "f1": 0.274373, + "f1_weighted": 0.27388 + }, + { + "accuracy": 0.330956, + "f1": 0.321277, + "f1_weighted": 0.341134 + }, + { + "accuracy": 0.286733, + "f1": 0.26998, + "f1_weighted": 0.277145 + }, + { + "accuracy": 0.272468, + "f1": 0.273716, + "f1_weighted": 0.27961 + }, + { + "accuracy": 0.318117, + "f1": 0.306174, + "f1_weighted": 0.311332 + }, + { + "accuracy": 0.329529, + "f1": 0.318608, + "f1_weighted": 0.323634 + }, + { + "accuracy": 0.285307, + "f1": 0.284961, + "f1_weighted": 0.28737 + }, + { + "accuracy": 0.318117, + "f1": 0.322569, + "f1_weighted": 0.318045 + } + ], + "main_score": 0.299857, + "hf_subset": "yor_Latn", + "languages": [ + "yor-Latn" + ] + }, + { + "accuracy": 0.192011, + "f1": 0.16917, + "f1_weighted": 0.156661, + "scores_per_experiment": [ + { + "accuracy": 0.263909, + "f1": 0.190902, + "f1_weighted": 0.190852 + }, + { + "accuracy": 0.189729, + "f1": 0.165654, + "f1_weighted": 0.142708 + }, + { + "accuracy": 0.174037, + "f1": 0.165054, + "f1_weighted": 0.145013 + }, + { + "accuracy": 0.216833, + "f1": 0.200972, + "f1_weighted": 0.190305 + }, + { + "accuracy": 0.164051, + "f1": 0.148372, + "f1_weighted": 0.110631 + }, + { + "accuracy": 0.142653, + "f1": 0.138859, + "f1_weighted": 0.124209 + }, + { + "accuracy": 0.161198, + "f1": 0.131142, + "f1_weighted": 0.116829 + }, + { + "accuracy": 0.241084, + "f1": 0.182548, + "f1_weighted": 0.187854 + }, + { + "accuracy": 0.196862, + "f1": 0.197771, + "f1_weighted": 0.198926 + }, + { + "accuracy": 0.169757, + "f1": 0.170422, + "f1_weighted": 0.159283 + } + ], + "main_score": 0.192011, + "hf_subset": "yue_Hant", + "languages": [ + "yue-Hant" + ] + }, + { + "accuracy": 0.195863, + "f1": 0.147318, + "f1_weighted": 0.149898, + "scores_per_experiment": [ + { + "accuracy": 0.192582, + "f1": 0.144753, + "f1_weighted": 0.154842 + }, + { + "accuracy": 0.164051, + "f1": 0.126316, + "f1_weighted": 0.106817 + }, + { + "accuracy": 0.165478, + "f1": 0.130332, + "f1_weighted": 0.134736 + }, + { + "accuracy": 0.249643, + "f1": 0.215256, + "f1_weighted": 0.236775 + }, + { + "accuracy": 0.228245, + "f1": 0.166463, + "f1_weighted": 0.164332 + }, + { + "accuracy": 0.185449, + "f1": 0.124053, + "f1_weighted": 0.131077 + }, + { + "accuracy": 0.236805, + "f1": 0.158122, + "f1_weighted": 0.173323 + }, + { + "accuracy": 0.159772, + "f1": 0.09846, + "f1_weighted": 0.088562 + }, + { + "accuracy": 0.191155, + "f1": 0.177702, + "f1_weighted": 0.178994 + }, + { + "accuracy": 0.185449, + "f1": 0.131719, + "f1_weighted": 0.129524 + } + ], + "main_score": 0.195863, + "hf_subset": "zho_Hant", + "languages": [ + "zho-Hant" + ] + }, + { + "accuracy": 0.427675, + "f1": 0.41117, + "f1_weighted": 0.432472, + "scores_per_experiment": [ + { + "accuracy": 0.459344, + "f1": 0.443675, + "f1_weighted": 0.46247 + }, + { + "accuracy": 0.389444, + "f1": 0.373695, + "f1_weighted": 0.394233 + }, + { + "accuracy": 0.426534, + "f1": 0.415378, + "f1_weighted": 0.431492 + }, + { + "accuracy": 0.489301, + "f1": 0.462848, + "f1_weighted": 0.490283 + }, + { + "accuracy": 0.406562, + "f1": 0.390241, + "f1_weighted": 0.408558 + }, + { + "accuracy": 0.382311, + "f1": 0.37706, + "f1_weighted": 0.38966 + }, + { + "accuracy": 0.429387, + "f1": 0.409499, + "f1_weighted": 0.434539 + }, + { + "accuracy": 0.442225, + "f1": 0.420435, + "f1_weighted": 0.445466 + }, + { + "accuracy": 0.443652, + "f1": 0.421604, + "f1_weighted": 0.450662 + }, + { + "accuracy": 0.407989, + "f1": 0.397266, + "f1_weighted": 0.417358 + } + ], + "main_score": 0.427675, + "hf_subset": "zsm_Latn", + "languages": [ + "zsm-Latn" + ] + }, + { + "accuracy": 0.320399, + "f1": 0.316423, + "f1_weighted": 0.323856, + "scores_per_experiment": [ + { + "accuracy": 0.352354, + "f1": 0.349815, + "f1_weighted": 0.359877 + }, + { + "accuracy": 0.293866, + "f1": 0.294795, + "f1_weighted": 0.294059 + }, + { + "accuracy": 0.308131, + "f1": 0.298649, + "f1_weighted": 0.309307 + }, + { + "accuracy": 0.305278, + "f1": 0.303256, + "f1_weighted": 0.312577 + }, + { + "accuracy": 0.383738, + "f1": 0.368545, + "f1_weighted": 0.387132 + }, + { + "accuracy": 0.305278, + "f1": 0.291921, + "f1_weighted": 0.310539 + }, + { + "accuracy": 0.326676, + "f1": 0.314249, + "f1_weighted": 0.317337 + }, + { + "accuracy": 0.330956, + "f1": 0.342524, + "f1_weighted": 0.331649 + }, + { + "accuracy": 0.309558, + "f1": 0.315343, + "f1_weighted": 0.314211 + }, + { + "accuracy": 0.28816, + "f1": 0.285138, + "f1_weighted": 0.301869 + } + ], + "main_score": 0.320399, + "hf_subset": "zul_Latn", + "languages": [ + "zul-Latn" + ] + } + ], + "validation": [ + { + "accuracy": 0.360606, + "f1": 0.349056, + "f1_weighted": 0.367131, + "scores_per_experiment": [ + { + "accuracy": 0.424242, + "f1": 0.420282, + "f1_weighted": 0.432788 + }, + { + "accuracy": 0.363636, + "f1": 0.341043, + "f1_weighted": 0.364733 + }, + { + "accuracy": 0.282828, + "f1": 0.304575, + "f1_weighted": 0.293728 + }, + { + "accuracy": 0.30303, + "f1": 0.298783, + "f1_weighted": 0.317015 + }, + { + "accuracy": 0.313131, + "f1": 0.290947, + "f1_weighted": 0.311939 + }, + { + "accuracy": 0.393939, + "f1": 0.391133, + "f1_weighted": 0.405644 + }, + { + "accuracy": 0.353535, + "f1": 0.338767, + "f1_weighted": 0.367808 + }, + { + "accuracy": 0.383838, + "f1": 0.386124, + "f1_weighted": 0.40112 + }, + { + "accuracy": 0.323232, + "f1": 0.27896, + "f1_weighted": 0.303426 + }, + { + "accuracy": 0.464646, + "f1": 0.439944, + "f1_weighted": 0.473113 + } + ], + "main_score": 0.360606, + "hf_subset": "ace_Latn", + "languages": [ + "ace-Latn" + ] + }, + { + "accuracy": 0.19899, + "f1": 0.145935, + "f1_weighted": 0.146786, + "scores_per_experiment": [ + { + "accuracy": 0.262626, + "f1": 0.196565, + "f1_weighted": 0.1931 + }, + { + "accuracy": 0.222222, + "f1": 0.174741, + "f1_weighted": 0.178073 + }, + { + "accuracy": 0.090909, + "f1": 0.060013, + "f1_weighted": 0.047052 + }, + { + "accuracy": 0.30303, + "f1": 0.214948, + "f1_weighted": 0.225053 + }, + { + "accuracy": 0.212121, + "f1": 0.134641, + "f1_weighted": 0.141368 + }, + { + "accuracy": 0.191919, + "f1": 0.154967, + "f1_weighted": 0.16803 + }, + { + "accuracy": 0.232323, + "f1": 0.159888, + "f1_weighted": 0.165473 + }, + { + "accuracy": 0.141414, + "f1": 0.112671, + "f1_weighted": 0.106157 + }, + { + "accuracy": 0.121212, + "f1": 0.097273, + "f1_weighted": 0.100492 + }, + { + "accuracy": 0.212121, + "f1": 0.153647, + "f1_weighted": 0.143057 + } + ], + "main_score": 0.19899, + "hf_subset": "acm_Arab", + "languages": [ + "acm-Arab" + ] + }, + { + "accuracy": 0.184848, + "f1": 0.132163, + "f1_weighted": 0.131762, + "scores_per_experiment": [ + { + "accuracy": 0.242424, + "f1": 0.170353, + "f1_weighted": 0.169702 + }, + { + "accuracy": 0.232323, + "f1": 0.139529, + "f1_weighted": 0.145912 + }, + { + "accuracy": 0.090909, + "f1": 0.057809, + "f1_weighted": 0.045642 + }, + { + "accuracy": 0.252525, + "f1": 0.178384, + "f1_weighted": 0.171452 + }, + { + "accuracy": 0.161616, + "f1": 0.095114, + "f1_weighted": 0.114415 + }, + { + "accuracy": 0.131313, + "f1": 0.105835, + "f1_weighted": 0.105552 + }, + { + "accuracy": 0.262626, + "f1": 0.193202, + "f1_weighted": 0.191558 + }, + { + "accuracy": 0.121212, + "f1": 0.101474, + "f1_weighted": 0.095519 + }, + { + "accuracy": 0.151515, + "f1": 0.133763, + "f1_weighted": 0.13836 + }, + { + "accuracy": 0.20202, + "f1": 0.146164, + "f1_weighted": 0.13951 + } + ], + "main_score": 0.184848, + "hf_subset": "acq_Arab", + "languages": [ + "acq-Arab" + ] + }, + { + "accuracy": 0.209091, + "f1": 0.161057, + "f1_weighted": 0.156386, + "scores_per_experiment": [ + { + "accuracy": 0.282828, + "f1": 0.207797, + "f1_weighted": 0.202609 + }, + { + "accuracy": 0.222222, + "f1": 0.1325, + "f1_weighted": 0.137932 + }, + { + "accuracy": 0.111111, + "f1": 0.087488, + "f1_weighted": 0.065551 + }, + { + "accuracy": 0.292929, + "f1": 0.229267, + "f1_weighted": 0.234043 + }, + { + "accuracy": 0.181818, + "f1": 0.143892, + "f1_weighted": 0.138124 + }, + { + "accuracy": 0.171717, + "f1": 0.142328, + "f1_weighted": 0.147407 + }, + { + "accuracy": 0.252525, + "f1": 0.169986, + "f1_weighted": 0.171265 + }, + { + "accuracy": 0.20202, + "f1": 0.173076, + "f1_weighted": 0.161245 + }, + { + "accuracy": 0.151515, + "f1": 0.166896, + "f1_weighted": 0.150307 + }, + { + "accuracy": 0.222222, + "f1": 0.157336, + "f1_weighted": 0.155375 + } + ], + "main_score": 0.209091, + "hf_subset": "aeb_Arab", + "languages": [ + "aeb-Arab" + ] + }, + { + "accuracy": 0.343434, + "f1": 0.33082, + "f1_weighted": 0.344536, + "scores_per_experiment": [ + { + "accuracy": 0.363636, + "f1": 0.351354, + "f1_weighted": 0.385494 + }, + { + "accuracy": 0.292929, + "f1": 0.287181, + "f1_weighted": 0.287519 + }, + { + "accuracy": 0.353535, + "f1": 0.349371, + "f1_weighted": 0.330789 + }, + { + "accuracy": 0.40404, + "f1": 0.376154, + "f1_weighted": 0.388884 + }, + { + "accuracy": 0.30303, + "f1": 0.282252, + "f1_weighted": 0.304521 + }, + { + "accuracy": 0.363636, + "f1": 0.353506, + "f1_weighted": 0.371207 + }, + { + "accuracy": 0.282828, + "f1": 0.27943, + "f1_weighted": 0.27683 + }, + { + "accuracy": 0.40404, + "f1": 0.38924, + "f1_weighted": 0.422101 + }, + { + "accuracy": 0.30303, + "f1": 0.288374, + "f1_weighted": 0.304519 + }, + { + "accuracy": 0.363636, + "f1": 0.351338, + "f1_weighted": 0.373495 + } + ], + "main_score": 0.343434, + "hf_subset": "afr_Latn", + "languages": [ + "afr-Latn" + ] + }, + { + "accuracy": 0.183838, + "f1": 0.126525, + "f1_weighted": 0.13057, + "scores_per_experiment": [ + { + "accuracy": 0.222222, + "f1": 0.115589, + "f1_weighted": 0.132158 + }, + { + "accuracy": 0.232323, + "f1": 0.130953, + "f1_weighted": 0.160538 + }, + { + "accuracy": 0.121212, + "f1": 0.094293, + "f1_weighted": 0.072943 + }, + { + "accuracy": 0.262626, + "f1": 0.179226, + "f1_weighted": 0.167111 + }, + { + "accuracy": 0.151515, + "f1": 0.099773, + "f1_weighted": 0.103575 + }, + { + "accuracy": 0.171717, + "f1": 0.180253, + "f1_weighted": 0.168119 + }, + { + "accuracy": 0.181818, + "f1": 0.122573, + "f1_weighted": 0.136604 + }, + { + "accuracy": 0.131313, + "f1": 0.083863, + "f1_weighted": 0.084451 + }, + { + "accuracy": 0.191919, + "f1": 0.127372, + "f1_weighted": 0.141996 + }, + { + "accuracy": 0.171717, + "f1": 0.131352, + "f1_weighted": 0.138203 + } + ], + "main_score": 0.183838, + "hf_subset": "ajp_Arab", + "languages": [ + "ajp-Arab" + ] + }, + { + "accuracy": 0.424242, + "f1": 0.409038, + "f1_weighted": 0.425117, + "scores_per_experiment": [ + { + "accuracy": 0.434343, + "f1": 0.43121, + "f1_weighted": 0.434078 + }, + { + "accuracy": 0.393939, + "f1": 0.386275, + "f1_weighted": 0.381699 + }, + { + "accuracy": 0.393939, + "f1": 0.396257, + "f1_weighted": 0.404411 + }, + { + "accuracy": 0.505051, + "f1": 0.494694, + "f1_weighted": 0.506736 + }, + { + "accuracy": 0.383838, + "f1": 0.377089, + "f1_weighted": 0.388165 + }, + { + "accuracy": 0.383838, + "f1": 0.38677, + "f1_weighted": 0.387877 + }, + { + "accuracy": 0.444444, + "f1": 0.422111, + "f1_weighted": 0.452853 + }, + { + "accuracy": 0.474747, + "f1": 0.411836, + "f1_weighted": 0.462312 + }, + { + "accuracy": 0.424242, + "f1": 0.380105, + "f1_weighted": 0.414462 + }, + { + "accuracy": 0.40404, + "f1": 0.404028, + "f1_weighted": 0.418574 + } + ], + "main_score": 0.424242, + "hf_subset": "aka_Latn", + "languages": [ + "aka-Latn" + ] + }, + { + "accuracy": 0.359596, + "f1": 0.339737, + "f1_weighted": 0.362917, + "scores_per_experiment": [ + { + "accuracy": 0.383838, + "f1": 0.390695, + "f1_weighted": 0.387666 + }, + { + "accuracy": 0.393939, + "f1": 0.358155, + "f1_weighted": 0.403676 + }, + { + "accuracy": 0.313131, + "f1": 0.295572, + "f1_weighted": 0.307176 + }, + { + "accuracy": 0.383838, + "f1": 0.371348, + "f1_weighted": 0.395323 + }, + { + "accuracy": 0.323232, + "f1": 0.310593, + "f1_weighted": 0.323972 + }, + { + "accuracy": 0.333333, + "f1": 0.320499, + "f1_weighted": 0.340017 + }, + { + "accuracy": 0.373737, + "f1": 0.343316, + "f1_weighted": 0.375523 + }, + { + "accuracy": 0.373737, + "f1": 0.361316, + "f1_weighted": 0.37432 + }, + { + "accuracy": 0.373737, + "f1": 0.331701, + "f1_weighted": 0.364346 + }, + { + "accuracy": 0.343434, + "f1": 0.314177, + "f1_weighted": 0.357151 + } + ], + "main_score": 0.359596, + "hf_subset": "als_Latn", + "languages": [ + "als-Latn" + ] + }, + { + "accuracy": 0.20404, + "f1": 0.138978, + "f1_weighted": 0.149409, + "scores_per_experiment": [ + { + "accuracy": 0.282828, + "f1": 0.195443, + "f1_weighted": 0.207058 + }, + { + "accuracy": 0.141414, + "f1": 0.106722, + "f1_weighted": 0.104592 + }, + { + "accuracy": 0.131313, + "f1": 0.10205, + "f1_weighted": 0.088601 + }, + { + "accuracy": 0.282828, + "f1": 0.224546, + "f1_weighted": 0.215507 + }, + { + "accuracy": 0.141414, + "f1": 0.108538, + "f1_weighted": 0.107133 + }, + { + "accuracy": 0.292929, + "f1": 0.120357, + "f1_weighted": 0.169542 + }, + { + "accuracy": 0.171717, + "f1": 0.118042, + "f1_weighted": 0.133334 + }, + { + "accuracy": 0.292929, + "f1": 0.167243, + "f1_weighted": 0.226413 + }, + { + "accuracy": 0.161616, + "f1": 0.116124, + "f1_weighted": 0.104425 + }, + { + "accuracy": 0.141414, + "f1": 0.130714, + "f1_weighted": 0.137482 + } + ], + "main_score": 0.20404, + "hf_subset": "amh_Ethi", + "languages": [ + "amh-Ethi" + ] + }, + { + "accuracy": 0.180808, + "f1": 0.133642, + "f1_weighted": 0.131164, + "scores_per_experiment": [ + { + "accuracy": 0.252525, + "f1": 0.15769, + "f1_weighted": 0.167617 + }, + { + "accuracy": 0.20202, + "f1": 0.106782, + "f1_weighted": 0.122539 + }, + { + "accuracy": 0.111111, + "f1": 0.095158, + "f1_weighted": 0.072665 + }, + { + "accuracy": 0.272727, + "f1": 0.210108, + "f1_weighted": 0.195511 + }, + { + "accuracy": 0.181818, + "f1": 0.136988, + "f1_weighted": 0.12938 + }, + { + "accuracy": 0.131313, + "f1": 0.116699, + "f1_weighted": 0.128283 + }, + { + "accuracy": 0.141414, + "f1": 0.117357, + "f1_weighted": 0.104362 + }, + { + "accuracy": 0.131313, + "f1": 0.107221, + "f1_weighted": 0.094034 + }, + { + "accuracy": 0.191919, + "f1": 0.153313, + "f1_weighted": 0.163739 + }, + { + "accuracy": 0.191919, + "f1": 0.135103, + "f1_weighted": 0.133509 + } + ], + "main_score": 0.180808, + "hf_subset": "apc_Arab", + "languages": [ + "apc-Arab" + ] + }, + { + "accuracy": 0.258586, + "f1": 0.244531, + "f1_weighted": 0.258454, + "scores_per_experiment": [ + { + "accuracy": 0.30303, + "f1": 0.30602, + "f1_weighted": 0.315339 + }, + { + "accuracy": 0.262626, + "f1": 0.244816, + "f1_weighted": 0.250548 + }, + { + "accuracy": 0.212121, + "f1": 0.218772, + "f1_weighted": 0.211477 + }, + { + "accuracy": 0.262626, + "f1": 0.240394, + "f1_weighted": 0.254602 + }, + { + "accuracy": 0.252525, + "f1": 0.234279, + "f1_weighted": 0.254052 + }, + { + "accuracy": 0.222222, + "f1": 0.219814, + "f1_weighted": 0.231006 + }, + { + "accuracy": 0.191919, + "f1": 0.140429, + "f1_weighted": 0.181222 + }, + { + "accuracy": 0.313131, + "f1": 0.297654, + "f1_weighted": 0.317995 + }, + { + "accuracy": 0.252525, + "f1": 0.234796, + "f1_weighted": 0.25223 + }, + { + "accuracy": 0.313131, + "f1": 0.308339, + "f1_weighted": 0.316073 + } + ], + "main_score": 0.258586, + "hf_subset": "arb_Latn", + "languages": [ + "arb-Latn" + ] + }, + { + "accuracy": 0.207071, + "f1": 0.147533, + "f1_weighted": 0.153782, + "scores_per_experiment": [ + { + "accuracy": 0.262626, + "f1": 0.180789, + "f1_weighted": 0.195933 + }, + { + "accuracy": 0.232323, + "f1": 0.139529, + "f1_weighted": 0.145912 + }, + { + "accuracy": 0.090909, + "f1": 0.06064, + "f1_weighted": 0.047165 + }, + { + "accuracy": 0.30303, + "f1": 0.228304, + "f1_weighted": 0.238985 + }, + { + "accuracy": 0.151515, + "f1": 0.08675, + "f1_weighted": 0.101159 + }, + { + "accuracy": 0.161616, + "f1": 0.127102, + "f1_weighted": 0.129588 + }, + { + "accuracy": 0.252525, + "f1": 0.168082, + "f1_weighted": 0.171758 + }, + { + "accuracy": 0.161616, + "f1": 0.132151, + "f1_weighted": 0.124328 + }, + { + "accuracy": 0.191919, + "f1": 0.172108, + "f1_weighted": 0.199498 + }, + { + "accuracy": 0.262626, + "f1": 0.17987, + "f1_weighted": 0.183494 + } + ], + "main_score": 0.207071, + "hf_subset": "ars_Arab", + "languages": [ + "ars-Arab" + ] + }, + { + "accuracy": 0.2, + "f1": 0.152373, + "f1_weighted": 0.155246, + "scores_per_experiment": [ + { + "accuracy": 0.252525, + "f1": 0.180905, + "f1_weighted": 0.182343 + }, + { + "accuracy": 0.141414, + "f1": 0.093112, + "f1_weighted": 0.077781 + }, + { + "accuracy": 0.090909, + "f1": 0.076979, + "f1_weighted": 0.058728 + }, + { + "accuracy": 0.313131, + "f1": 0.258355, + "f1_weighted": 0.273303 + }, + { + "accuracy": 0.181818, + "f1": 0.15942, + "f1_weighted": 0.157132 + }, + { + "accuracy": 0.181818, + "f1": 0.152585, + "f1_weighted": 0.175664 + }, + { + "accuracy": 0.191919, + "f1": 0.11629, + "f1_weighted": 0.128984 + }, + { + "accuracy": 0.242424, + "f1": 0.180131, + "f1_weighted": 0.204429 + }, + { + "accuracy": 0.191919, + "f1": 0.153114, + "f1_weighted": 0.149391 + }, + { + "accuracy": 0.212121, + "f1": 0.152837, + "f1_weighted": 0.144707 + } + ], + "main_score": 0.2, + "hf_subset": "ary_Arab", + "languages": [ + "ary-Arab" + ] + }, + { + "accuracy": 0.20101, + "f1": 0.155975, + "f1_weighted": 0.155845, + "scores_per_experiment": [ + { + "accuracy": 0.252525, + "f1": 0.175467, + "f1_weighted": 0.182281 + }, + { + "accuracy": 0.242424, + "f1": 0.163506, + "f1_weighted": 0.163843 + }, + { + "accuracy": 0.111111, + "f1": 0.085966, + "f1_weighted": 0.064658 + }, + { + "accuracy": 0.272727, + "f1": 0.188569, + "f1_weighted": 0.176503 + }, + { + "accuracy": 0.191919, + "f1": 0.1531, + "f1_weighted": 0.17575 + }, + { + "accuracy": 0.141414, + "f1": 0.134943, + "f1_weighted": 0.123771 + }, + { + "accuracy": 0.161616, + "f1": 0.120195, + "f1_weighted": 0.125224 + }, + { + "accuracy": 0.222222, + "f1": 0.21235, + "f1_weighted": 0.197737 + }, + { + "accuracy": 0.171717, + "f1": 0.134581, + "f1_weighted": 0.155533 + }, + { + "accuracy": 0.242424, + "f1": 0.191071, + "f1_weighted": 0.193147 + } + ], + "main_score": 0.20101, + "hf_subset": "arz_Arab", + "languages": [ + "arz-Arab" + ] + }, + { + "accuracy": 0.160606, + "f1": 0.123407, + "f1_weighted": 0.137976, + "scores_per_experiment": [ + { + "accuracy": 0.161616, + "f1": 0.122939, + "f1_weighted": 0.137573 + }, + { + "accuracy": 0.171717, + "f1": 0.155921, + "f1_weighted": 0.175026 + }, + { + "accuracy": 0.10101, + "f1": 0.09427, + "f1_weighted": 0.072209 + }, + { + "accuracy": 0.171717, + "f1": 0.135861, + "f1_weighted": 0.124522 + }, + { + "accuracy": 0.131313, + "f1": 0.120101, + "f1_weighted": 0.102515 + }, + { + "accuracy": 0.171717, + "f1": 0.103732, + "f1_weighted": 0.152544 + }, + { + "accuracy": 0.181818, + "f1": 0.150512, + "f1_weighted": 0.16587 + }, + { + "accuracy": 0.242424, + "f1": 0.13837, + "f1_weighted": 0.202556 + }, + { + "accuracy": 0.151515, + "f1": 0.120246, + "f1_weighted": 0.130758 + }, + { + "accuracy": 0.121212, + "f1": 0.092118, + "f1_weighted": 0.116187 + } + ], + "main_score": 0.160606, + "hf_subset": "asm_Beng", + "languages": [ + "asm-Beng" + ] + }, + { + "accuracy": 0.446465, + "f1": 0.429753, + "f1_weighted": 0.449405, + "scores_per_experiment": [ + { + "accuracy": 0.474747, + "f1": 0.458249, + "f1_weighted": 0.471808 + }, + { + "accuracy": 0.464646, + "f1": 0.444276, + "f1_weighted": 0.477567 + }, + { + "accuracy": 0.414141, + "f1": 0.412513, + "f1_weighted": 0.416917 + }, + { + "accuracy": 0.494949, + "f1": 0.488299, + "f1_weighted": 0.499788 + }, + { + "accuracy": 0.444444, + "f1": 0.398972, + "f1_weighted": 0.4342 + }, + { + "accuracy": 0.373737, + "f1": 0.36434, + "f1_weighted": 0.382586 + }, + { + "accuracy": 0.484848, + "f1": 0.485393, + "f1_weighted": 0.485365 + }, + { + "accuracy": 0.444444, + "f1": 0.424155, + "f1_weighted": 0.448039 + }, + { + "accuracy": 0.444444, + "f1": 0.410756, + "f1_weighted": 0.443099 + }, + { + "accuracy": 0.424242, + "f1": 0.410578, + "f1_weighted": 0.434683 + } + ], + "main_score": 0.446465, + "hf_subset": "ast_Latn", + "languages": [ + "ast-Latn" + ] + }, + { + "accuracy": 0.188889, + "f1": 0.151123, + "f1_weighted": 0.150111, + "scores_per_experiment": [ + { + "accuracy": 0.212121, + "f1": 0.174242, + "f1_weighted": 0.174686 + }, + { + "accuracy": 0.222222, + "f1": 0.153091, + "f1_weighted": 0.158036 + }, + { + "accuracy": 0.151515, + "f1": 0.14361, + "f1_weighted": 0.125485 + }, + { + "accuracy": 0.20202, + "f1": 0.174999, + "f1_weighted": 0.155094 + }, + { + "accuracy": 0.171717, + "f1": 0.143148, + "f1_weighted": 0.136364 + }, + { + "accuracy": 0.151515, + "f1": 0.115595, + "f1_weighted": 0.117501 + }, + { + "accuracy": 0.232323, + "f1": 0.140882, + "f1_weighted": 0.153594 + }, + { + "accuracy": 0.232323, + "f1": 0.210414, + "f1_weighted": 0.226066 + }, + { + "accuracy": 0.141414, + "f1": 0.122001, + "f1_weighted": 0.1156 + }, + { + "accuracy": 0.171717, + "f1": 0.133249, + "f1_weighted": 0.138688 + } + ], + "main_score": 0.188889, + "hf_subset": "awa_Deva", + "languages": [ + "awa-Deva" + ] + }, + { + "accuracy": 0.30202, + "f1": 0.280787, + "f1_weighted": 0.306793, + "scores_per_experiment": [ + { + "accuracy": 0.363636, + "f1": 0.348808, + "f1_weighted": 0.383673 + }, + { + "accuracy": 0.313131, + "f1": 0.308471, + "f1_weighted": 0.322219 + }, + { + "accuracy": 0.262626, + "f1": 0.243536, + "f1_weighted": 0.254229 + }, + { + "accuracy": 0.292929, + "f1": 0.297504, + "f1_weighted": 0.292418 + }, + { + "accuracy": 0.333333, + "f1": 0.30571, + "f1_weighted": 0.341823 + }, + { + "accuracy": 0.343434, + "f1": 0.312553, + "f1_weighted": 0.344795 + }, + { + "accuracy": 0.181818, + "f1": 0.154977, + "f1_weighted": 0.174164 + }, + { + "accuracy": 0.292929, + "f1": 0.29674, + "f1_weighted": 0.31173 + }, + { + "accuracy": 0.292929, + "f1": 0.236791, + "f1_weighted": 0.291485 + }, + { + "accuracy": 0.343434, + "f1": 0.302785, + "f1_weighted": 0.351389 + } + ], + "main_score": 0.30202, + "hf_subset": "ayr_Latn", + "languages": [ + "ayr-Latn" + ] + }, + { + "accuracy": 0.226263, + "f1": 0.138839, + "f1_weighted": 0.161922, + "scores_per_experiment": [ + { + "accuracy": 0.232323, + "f1": 0.153486, + "f1_weighted": 0.195647 + }, + { + "accuracy": 0.272727, + "f1": 0.163764, + "f1_weighted": 0.185487 + }, + { + "accuracy": 0.10101, + "f1": 0.07388, + "f1_weighted": 0.058707 + }, + { + "accuracy": 0.242424, + "f1": 0.147898, + "f1_weighted": 0.145268 + }, + { + "accuracy": 0.191919, + "f1": 0.144424, + "f1_weighted": 0.168978 + }, + { + "accuracy": 0.212121, + "f1": 0.152984, + "f1_weighted": 0.18293 + }, + { + "accuracy": 0.20202, + "f1": 0.123115, + "f1_weighted": 0.128814 + }, + { + "accuracy": 0.313131, + "f1": 0.167989, + "f1_weighted": 0.236891 + }, + { + "accuracy": 0.282828, + "f1": 0.132414, + "f1_weighted": 0.179368 + }, + { + "accuracy": 0.212121, + "f1": 0.128431, + "f1_weighted": 0.137131 + } + ], + "main_score": 0.226263, + "hf_subset": "azb_Arab", + "languages": [ + "azb-Arab" + ] + }, + { + "accuracy": 0.261616, + "f1": 0.248367, + "f1_weighted": 0.261578, + "scores_per_experiment": [ + { + "accuracy": 0.313131, + "f1": 0.270622, + "f1_weighted": 0.297234 + }, + { + "accuracy": 0.232323, + "f1": 0.212272, + "f1_weighted": 0.233346 + }, + { + "accuracy": 0.242424, + "f1": 0.233088, + "f1_weighted": 0.239014 + }, + { + "accuracy": 0.30303, + "f1": 0.282396, + "f1_weighted": 0.305279 + }, + { + "accuracy": 0.161616, + "f1": 0.148913, + "f1_weighted": 0.151541 + }, + { + "accuracy": 0.191919, + "f1": 0.192332, + "f1_weighted": 0.197636 + }, + { + "accuracy": 0.252525, + "f1": 0.257509, + "f1_weighted": 0.243007 + }, + { + "accuracy": 0.323232, + "f1": 0.315048, + "f1_weighted": 0.349834 + }, + { + "accuracy": 0.222222, + "f1": 0.214691, + "f1_weighted": 0.223469 + }, + { + "accuracy": 0.373737, + "f1": 0.356804, + "f1_weighted": 0.375421 + } + ], + "main_score": 0.261616, + "hf_subset": "azj_Latn", + "languages": [ + "azj-Latn" + ] + }, + { + "accuracy": 0.207071, + "f1": 0.189059, + "f1_weighted": 0.197237, + "scores_per_experiment": [ + { + "accuracy": 0.252525, + "f1": 0.191557, + "f1_weighted": 0.217139 + }, + { + "accuracy": 0.222222, + "f1": 0.236405, + "f1_weighted": 0.225467 + }, + { + "accuracy": 0.151515, + "f1": 0.152782, + "f1_weighted": 0.147249 + }, + { + "accuracy": 0.262626, + "f1": 0.253483, + "f1_weighted": 0.256118 + }, + { + "accuracy": 0.131313, + "f1": 0.105747, + "f1_weighted": 0.110035 + }, + { + "accuracy": 0.161616, + "f1": 0.148054, + "f1_weighted": 0.155853 + }, + { + "accuracy": 0.171717, + "f1": 0.161367, + "f1_weighted": 0.162933 + }, + { + "accuracy": 0.30303, + "f1": 0.249192, + "f1_weighted": 0.284339 + }, + { + "accuracy": 0.181818, + "f1": 0.180152, + "f1_weighted": 0.165785 + }, + { + "accuracy": 0.232323, + "f1": 0.211853, + "f1_weighted": 0.247448 + } + ], + "main_score": 0.207071, + "hf_subset": "bak_Cyrl", + "languages": [ + "bak-Cyrl" + ] + }, + { + "accuracy": 0.319192, + "f1": 0.299427, + "f1_weighted": 0.318455, + "scores_per_experiment": [ + { + "accuracy": 0.343434, + "f1": 0.331745, + "f1_weighted": 0.337981 + }, + { + "accuracy": 0.343434, + "f1": 0.329296, + "f1_weighted": 0.351689 + }, + { + "accuracy": 0.323232, + "f1": 0.303794, + "f1_weighted": 0.323934 + }, + { + "accuracy": 0.373737, + "f1": 0.332527, + "f1_weighted": 0.370463 + }, + { + "accuracy": 0.323232, + "f1": 0.296441, + "f1_weighted": 0.328542 + }, + { + "accuracy": 0.252525, + "f1": 0.23626, + "f1_weighted": 0.261761 + }, + { + "accuracy": 0.333333, + "f1": 0.302596, + "f1_weighted": 0.322541 + }, + { + "accuracy": 0.252525, + "f1": 0.237093, + "f1_weighted": 0.254461 + }, + { + "accuracy": 0.343434, + "f1": 0.332192, + "f1_weighted": 0.33156 + }, + { + "accuracy": 0.30303, + "f1": 0.292322, + "f1_weighted": 0.301621 + } + ], + "main_score": 0.319192, + "hf_subset": "bam_Latn", + "languages": [ + "bam-Latn" + ] + }, + { + "accuracy": 0.344444, + "f1": 0.336, + "f1_weighted": 0.349726, + "scores_per_experiment": [ + { + "accuracy": 0.414141, + "f1": 0.433384, + "f1_weighted": 0.441119 + }, + { + "accuracy": 0.343434, + "f1": 0.33291, + "f1_weighted": 0.347284 + }, + { + "accuracy": 0.242424, + "f1": 0.254714, + "f1_weighted": 0.250757 + }, + { + "accuracy": 0.383838, + "f1": 0.381605, + "f1_weighted": 0.397278 + }, + { + "accuracy": 0.292929, + "f1": 0.267599, + "f1_weighted": 0.299776 + }, + { + "accuracy": 0.353535, + "f1": 0.333499, + "f1_weighted": 0.352256 + }, + { + "accuracy": 0.323232, + "f1": 0.321383, + "f1_weighted": 0.315727 + }, + { + "accuracy": 0.393939, + "f1": 0.383587, + "f1_weighted": 0.405381 + }, + { + "accuracy": 0.323232, + "f1": 0.301335, + "f1_weighted": 0.30841 + }, + { + "accuracy": 0.373737, + "f1": 0.34998, + "f1_weighted": 0.379267 + } + ], + "main_score": 0.344444, + "hf_subset": "ban_Latn", + "languages": [ + "ban-Latn" + ] + }, + { + "accuracy": 0.193939, + "f1": 0.174617, + "f1_weighted": 0.180748, + "scores_per_experiment": [ + { + "accuracy": 0.262626, + "f1": 0.212275, + "f1_weighted": 0.206909 + }, + { + "accuracy": 0.191919, + "f1": 0.153086, + "f1_weighted": 0.173226 + }, + { + "accuracy": 0.080808, + "f1": 0.081795, + "f1_weighted": 0.072438 + }, + { + "accuracy": 0.222222, + "f1": 0.218471, + "f1_weighted": 0.223036 + }, + { + "accuracy": 0.131313, + "f1": 0.112765, + "f1_weighted": 0.129163 + }, + { + "accuracy": 0.20202, + "f1": 0.194447, + "f1_weighted": 0.192944 + }, + { + "accuracy": 0.20202, + "f1": 0.187875, + "f1_weighted": 0.19265 + }, + { + "accuracy": 0.161616, + "f1": 0.147665, + "f1_weighted": 0.162274 + }, + { + "accuracy": 0.232323, + "f1": 0.213659, + "f1_weighted": 0.215735 + }, + { + "accuracy": 0.252525, + "f1": 0.224131, + "f1_weighted": 0.239108 + } + ], + "main_score": 0.193939, + "hf_subset": "bel_Cyrl", + "languages": [ + "bel-Cyrl" + ] + }, + { + "accuracy": 0.337374, + "f1": 0.32441, + "f1_weighted": 0.339197, + "scores_per_experiment": [ + { + "accuracy": 0.343434, + "f1": 0.317414, + "f1_weighted": 0.333861 + }, + { + "accuracy": 0.262626, + "f1": 0.265359, + "f1_weighted": 0.251958 + }, + { + "accuracy": 0.323232, + "f1": 0.310543, + "f1_weighted": 0.321573 + }, + { + "accuracy": 0.313131, + "f1": 0.313198, + "f1_weighted": 0.31303 + }, + { + "accuracy": 0.323232, + "f1": 0.319283, + "f1_weighted": 0.32602 + }, + { + "accuracy": 0.343434, + "f1": 0.32148, + "f1_weighted": 0.369635 + }, + { + "accuracy": 0.393939, + "f1": 0.375877, + "f1_weighted": 0.392976 + }, + { + "accuracy": 0.393939, + "f1": 0.367717, + "f1_weighted": 0.38619 + }, + { + "accuracy": 0.313131, + "f1": 0.30517, + "f1_weighted": 0.316785 + }, + { + "accuracy": 0.363636, + "f1": 0.348057, + "f1_weighted": 0.37994 + } + ], + "main_score": 0.337374, + "hf_subset": "bem_Latn", + "languages": [ + "bem-Latn" + ] + }, + { + "accuracy": 0.162626, + "f1": 0.137348, + "f1_weighted": 0.148858, + "scores_per_experiment": [ + { + "accuracy": 0.181818, + "f1": 0.161976, + "f1_weighted": 0.183266 + }, + { + "accuracy": 0.10101, + "f1": 0.094406, + "f1_weighted": 0.097957 + }, + { + "accuracy": 0.151515, + "f1": 0.116549, + "f1_weighted": 0.120052 + }, + { + "accuracy": 0.181818, + "f1": 0.154248, + "f1_weighted": 0.144024 + }, + { + "accuracy": 0.20202, + "f1": 0.163859, + "f1_weighted": 0.200921 + }, + { + "accuracy": 0.191919, + "f1": 0.112837, + "f1_weighted": 0.150975 + }, + { + "accuracy": 0.191919, + "f1": 0.20556, + "f1_weighted": 0.196844 + }, + { + "accuracy": 0.121212, + "f1": 0.101139, + "f1_weighted": 0.12217 + }, + { + "accuracy": 0.131313, + "f1": 0.126358, + "f1_weighted": 0.127997 + }, + { + "accuracy": 0.171717, + "f1": 0.136544, + "f1_weighted": 0.144374 + } + ], + "main_score": 0.162626, + "hf_subset": "ben_Beng", + "languages": [ + "ben-Beng" + ] + }, + { + "accuracy": 0.226263, + "f1": 0.182008, + "f1_weighted": 0.200022, + "scores_per_experiment": [ + { + "accuracy": 0.191919, + "f1": 0.119974, + "f1_weighted": 0.137575 + }, + { + "accuracy": 0.242424, + "f1": 0.205974, + "f1_weighted": 0.218935 + }, + { + "accuracy": 0.121212, + "f1": 0.133349, + "f1_weighted": 0.124606 + }, + { + "accuracy": 0.343434, + "f1": 0.300338, + "f1_weighted": 0.332503 + }, + { + "accuracy": 0.171717, + "f1": 0.151392, + "f1_weighted": 0.146134 + }, + { + "accuracy": 0.20202, + "f1": 0.184706, + "f1_weighted": 0.189764 + }, + { + "accuracy": 0.191919, + "f1": 0.149429, + "f1_weighted": 0.152203 + }, + { + "accuracy": 0.282828, + "f1": 0.173168, + "f1_weighted": 0.230769 + }, + { + "accuracy": 0.30303, + "f1": 0.218367, + "f1_weighted": 0.27013 + }, + { + "accuracy": 0.212121, + "f1": 0.183382, + "f1_weighted": 0.197595 + } + ], + "main_score": 0.226263, + "hf_subset": "bho_Deva", + "languages": [ + "bho-Deva" + ] + }, + { + "accuracy": 0.351515, + "f1": 0.341912, + "f1_weighted": 0.353373, + "scores_per_experiment": [ + { + "accuracy": 0.414141, + "f1": 0.415221, + "f1_weighted": 0.423561 + }, + { + "accuracy": 0.272727, + "f1": 0.251261, + "f1_weighted": 0.271726 + }, + { + "accuracy": 0.343434, + "f1": 0.34382, + "f1_weighted": 0.336508 + }, + { + "accuracy": 0.414141, + "f1": 0.40123, + "f1_weighted": 0.40717 + }, + { + "accuracy": 0.282828, + "f1": 0.278658, + "f1_weighted": 0.285199 + }, + { + "accuracy": 0.343434, + "f1": 0.347905, + "f1_weighted": 0.353391 + }, + { + "accuracy": 0.363636, + "f1": 0.349393, + "f1_weighted": 0.363606 + }, + { + "accuracy": 0.454545, + "f1": 0.44751, + "f1_weighted": 0.456309 + }, + { + "accuracy": 0.343434, + "f1": 0.309903, + "f1_weighted": 0.326911 + }, + { + "accuracy": 0.282828, + "f1": 0.274219, + "f1_weighted": 0.309345 + } + ], + "main_score": 0.351515, + "hf_subset": "bjn_Latn", + "languages": [ + "bjn-Latn" + ] + }, + { + "accuracy": 0.184848, + "f1": 0.134355, + "f1_weighted": 0.144818, + "scores_per_experiment": [ + { + "accuracy": 0.20202, + "f1": 0.155694, + "f1_weighted": 0.154055 + }, + { + "accuracy": 0.171717, + "f1": 0.102124, + "f1_weighted": 0.113622 + }, + { + "accuracy": 0.131313, + "f1": 0.092686, + "f1_weighted": 0.104773 + }, + { + "accuracy": 0.20202, + "f1": 0.17154, + "f1_weighted": 0.169123 + }, + { + "accuracy": 0.151515, + "f1": 0.104581, + "f1_weighted": 0.120135 + }, + { + "accuracy": 0.161616, + "f1": 0.120247, + "f1_weighted": 0.128248 + }, + { + "accuracy": 0.20202, + "f1": 0.140218, + "f1_weighted": 0.146455 + }, + { + "accuracy": 0.212121, + "f1": 0.179748, + "f1_weighted": 0.187496 + }, + { + "accuracy": 0.222222, + "f1": 0.13173, + "f1_weighted": 0.159902 + }, + { + "accuracy": 0.191919, + "f1": 0.144988, + "f1_weighted": 0.16437 + } + ], + "main_score": 0.184848, + "hf_subset": "bod_Tibt", + "languages": [ + "bod-Tibt" + ] + }, + { + "accuracy": 0.351515, + "f1": 0.327388, + "f1_weighted": 0.35046, + "scores_per_experiment": [ + { + "accuracy": 0.40404, + "f1": 0.401106, + "f1_weighted": 0.410385 + }, + { + "accuracy": 0.414141, + "f1": 0.399106, + "f1_weighted": 0.428236 + }, + { + "accuracy": 0.292929, + "f1": 0.278235, + "f1_weighted": 0.279421 + }, + { + "accuracy": 0.353535, + "f1": 0.322991, + "f1_weighted": 0.348837 + }, + { + "accuracy": 0.313131, + "f1": 0.306174, + "f1_weighted": 0.318904 + }, + { + "accuracy": 0.333333, + "f1": 0.281733, + "f1_weighted": 0.335334 + }, + { + "accuracy": 0.323232, + "f1": 0.278346, + "f1_weighted": 0.30677 + }, + { + "accuracy": 0.414141, + "f1": 0.371663, + "f1_weighted": 0.408807 + }, + { + "accuracy": 0.323232, + "f1": 0.292703, + "f1_weighted": 0.320584 + }, + { + "accuracy": 0.343434, + "f1": 0.341826, + "f1_weighted": 0.347323 + } + ], + "main_score": 0.351515, + "hf_subset": "bos_Latn", + "languages": [ + "bos-Latn" + ] + }, + { + "accuracy": 0.342424, + "f1": 0.318032, + "f1_weighted": 0.346183, + "scores_per_experiment": [ + { + "accuracy": 0.444444, + "f1": 0.429436, + "f1_weighted": 0.45383 + }, + { + "accuracy": 0.353535, + "f1": 0.324416, + "f1_weighted": 0.361186 + }, + { + "accuracy": 0.282828, + "f1": 0.288283, + "f1_weighted": 0.300147 + }, + { + "accuracy": 0.383838, + "f1": 0.35116, + "f1_weighted": 0.380734 + }, + { + "accuracy": 0.292929, + "f1": 0.249302, + "f1_weighted": 0.286445 + }, + { + "accuracy": 0.292929, + "f1": 0.274208, + "f1_weighted": 0.304916 + }, + { + "accuracy": 0.30303, + "f1": 0.28159, + "f1_weighted": 0.291318 + }, + { + "accuracy": 0.343434, + "f1": 0.32406, + "f1_weighted": 0.350773 + }, + { + "accuracy": 0.353535, + "f1": 0.317081, + "f1_weighted": 0.354655 + }, + { + "accuracy": 0.373737, + "f1": 0.340779, + "f1_weighted": 0.37783 + } + ], + "main_score": 0.342424, + "hf_subset": "bug_Latn", + "languages": [ + "bug-Latn" + ] + }, + { + "accuracy": 0.211111, + "f1": 0.191141, + "f1_weighted": 0.198812, + "scores_per_experiment": [ + { + "accuracy": 0.272727, + "f1": 0.241014, + "f1_weighted": 0.253754 + }, + { + "accuracy": 0.242424, + "f1": 0.209221, + "f1_weighted": 0.212138 + }, + { + "accuracy": 0.20202, + "f1": 0.195978, + "f1_weighted": 0.194312 + }, + { + "accuracy": 0.242424, + "f1": 0.215814, + "f1_weighted": 0.23168 + }, + { + "accuracy": 0.171717, + "f1": 0.160731, + "f1_weighted": 0.166854 + }, + { + "accuracy": 0.171717, + "f1": 0.125062, + "f1_weighted": 0.174157 + }, + { + "accuracy": 0.212121, + "f1": 0.182505, + "f1_weighted": 0.163644 + }, + { + "accuracy": 0.222222, + "f1": 0.196017, + "f1_weighted": 0.225412 + }, + { + "accuracy": 0.20202, + "f1": 0.210863, + "f1_weighted": 0.202282 + }, + { + "accuracy": 0.171717, + "f1": 0.174199, + "f1_weighted": 0.163886 + } + ], + "main_score": 0.211111, + "hf_subset": "bul_Cyrl", + "languages": [ + "bul-Cyrl" + ] + }, + { + "accuracy": 0.470707, + "f1": 0.457208, + "f1_weighted": 0.477264, + "scores_per_experiment": [ + { + "accuracy": 0.545455, + "f1": 0.534431, + "f1_weighted": 0.560982 + }, + { + "accuracy": 0.454545, + "f1": 0.45878, + "f1_weighted": 0.471078 + }, + { + "accuracy": 0.383838, + "f1": 0.372814, + "f1_weighted": 0.390303 + }, + { + "accuracy": 0.494949, + "f1": 0.474709, + "f1_weighted": 0.497185 + }, + { + "accuracy": 0.464646, + "f1": 0.454286, + "f1_weighted": 0.472015 + }, + { + "accuracy": 0.464646, + "f1": 0.450154, + "f1_weighted": 0.470471 + }, + { + "accuracy": 0.494949, + "f1": 0.495337, + "f1_weighted": 0.498328 + }, + { + "accuracy": 0.515152, + "f1": 0.50879, + "f1_weighted": 0.527889 + }, + { + "accuracy": 0.373737, + "f1": 0.335732, + "f1_weighted": 0.367088 + }, + { + "accuracy": 0.515152, + "f1": 0.487051, + "f1_weighted": 0.517296 + } + ], + "main_score": 0.470707, + "hf_subset": "cat_Latn", + "languages": [ + "cat-Latn" + ] + }, + { + "accuracy": 0.50101, + "f1": 0.480643, + "f1_weighted": 0.505632, + "scores_per_experiment": [ + { + "accuracy": 0.505051, + "f1": 0.48019, + "f1_weighted": 0.505069 + }, + { + "accuracy": 0.494949, + "f1": 0.473207, + "f1_weighted": 0.504974 + }, + { + "accuracy": 0.464646, + "f1": 0.471507, + "f1_weighted": 0.472559 + }, + { + "accuracy": 0.555556, + "f1": 0.513987, + "f1_weighted": 0.545823 + }, + { + "accuracy": 0.414141, + "f1": 0.394546, + "f1_weighted": 0.420218 + }, + { + "accuracy": 0.545455, + "f1": 0.524098, + "f1_weighted": 0.548348 + }, + { + "accuracy": 0.565657, + "f1": 0.54344, + "f1_weighted": 0.56969 + }, + { + "accuracy": 0.525253, + "f1": 0.528573, + "f1_weighted": 0.545559 + }, + { + "accuracy": 0.474747, + "f1": 0.423992, + "f1_weighted": 0.470187 + }, + { + "accuracy": 0.464646, + "f1": 0.45289, + "f1_weighted": 0.473893 + } + ], + "main_score": 0.50101, + "hf_subset": "ceb_Latn", + "languages": [ + "ceb-Latn" + ] + }, + { + "accuracy": 0.336364, + "f1": 0.330593, + "f1_weighted": 0.338071, + "scores_per_experiment": [ + { + "accuracy": 0.383838, + "f1": 0.395981, + "f1_weighted": 0.389561 + }, + { + "accuracy": 0.272727, + "f1": 0.256298, + "f1_weighted": 0.284101 + }, + { + "accuracy": 0.282828, + "f1": 0.263647, + "f1_weighted": 0.280353 + }, + { + "accuracy": 0.363636, + "f1": 0.359164, + "f1_weighted": 0.368999 + }, + { + "accuracy": 0.383838, + "f1": 0.369052, + "f1_weighted": 0.386358 + }, + { + "accuracy": 0.313131, + "f1": 0.31817, + "f1_weighted": 0.31584 + }, + { + "accuracy": 0.343434, + "f1": 0.332746, + "f1_weighted": 0.343973 + }, + { + "accuracy": 0.383838, + "f1": 0.382537, + "f1_weighted": 0.394319 + }, + { + "accuracy": 0.282828, + "f1": 0.270333, + "f1_weighted": 0.262599 + }, + { + "accuracy": 0.353535, + "f1": 0.358002, + "f1_weighted": 0.354604 + } + ], + "main_score": 0.336364, + "hf_subset": "ces_Latn", + "languages": [ + "ces-Latn" + ] + }, + { + "accuracy": 0.268687, + "f1": 0.258604, + "f1_weighted": 0.266665, + "scores_per_experiment": [ + { + "accuracy": 0.353535, + "f1": 0.319434, + "f1_weighted": 0.355161 + }, + { + "accuracy": 0.262626, + "f1": 0.270842, + "f1_weighted": 0.240882 + }, + { + "accuracy": 0.171717, + "f1": 0.158952, + "f1_weighted": 0.163416 + }, + { + "accuracy": 0.272727, + "f1": 0.264063, + "f1_weighted": 0.272181 + }, + { + "accuracy": 0.262626, + "f1": 0.256615, + "f1_weighted": 0.258117 + }, + { + "accuracy": 0.242424, + "f1": 0.217563, + "f1_weighted": 0.257854 + }, + { + "accuracy": 0.222222, + "f1": 0.218648, + "f1_weighted": 0.224477 + }, + { + "accuracy": 0.323232, + "f1": 0.324109, + "f1_weighted": 0.316269 + }, + { + "accuracy": 0.292929, + "f1": 0.284264, + "f1_weighted": 0.291446 + }, + { + "accuracy": 0.282828, + "f1": 0.27155, + "f1_weighted": 0.286849 + } + ], + "main_score": 0.268687, + "hf_subset": "cjk_Latn", + "languages": [ + "cjk-Latn" + ] + }, + { + "accuracy": 0.187879, + "f1": 0.158098, + "f1_weighted": 0.164908, + "scores_per_experiment": [ + { + "accuracy": 0.252525, + "f1": 0.190249, + "f1_weighted": 0.214873 + }, + { + "accuracy": 0.060606, + "f1": 0.056424, + "f1_weighted": 0.050556 + }, + { + "accuracy": 0.171717, + "f1": 0.143176, + "f1_weighted": 0.154253 + }, + { + "accuracy": 0.252525, + "f1": 0.21396, + "f1_weighted": 0.243558 + }, + { + "accuracy": 0.171717, + "f1": 0.14568, + "f1_weighted": 0.150039 + }, + { + "accuracy": 0.20202, + "f1": 0.188269, + "f1_weighted": 0.184783 + }, + { + "accuracy": 0.171717, + "f1": 0.153686, + "f1_weighted": 0.16642 + }, + { + "accuracy": 0.242424, + "f1": 0.191671, + "f1_weighted": 0.20005 + }, + { + "accuracy": 0.151515, + "f1": 0.142016, + "f1_weighted": 0.126181 + }, + { + "accuracy": 0.20202, + "f1": 0.155849, + "f1_weighted": 0.158362 + } + ], + "main_score": 0.187879, + "hf_subset": "ckb_Arab", + "languages": [ + "ckb-Arab" + ] + }, + { + "accuracy": 0.320202, + "f1": 0.312997, + "f1_weighted": 0.323136, + "scores_per_experiment": [ + { + "accuracy": 0.313131, + "f1": 0.32174, + "f1_weighted": 0.326816 + }, + { + "accuracy": 0.323232, + "f1": 0.311149, + "f1_weighted": 0.330644 + }, + { + "accuracy": 0.222222, + "f1": 0.211407, + "f1_weighted": 0.195603 + }, + { + "accuracy": 0.363636, + "f1": 0.329043, + "f1_weighted": 0.357449 + }, + { + "accuracy": 0.282828, + "f1": 0.289874, + "f1_weighted": 0.279602 + }, + { + "accuracy": 0.353535, + "f1": 0.333757, + "f1_weighted": 0.368111 + }, + { + "accuracy": 0.313131, + "f1": 0.304948, + "f1_weighted": 0.314923 + }, + { + "accuracy": 0.353535, + "f1": 0.345793, + "f1_weighted": 0.362619 + }, + { + "accuracy": 0.323232, + "f1": 0.335101, + "f1_weighted": 0.344918 + }, + { + "accuracy": 0.353535, + "f1": 0.347158, + "f1_weighted": 0.350673 + } + ], + "main_score": 0.320202, + "hf_subset": "crh_Latn", + "languages": [ + "crh-Latn" + ] + }, + { + "accuracy": 0.311111, + "f1": 0.293987, + "f1_weighted": 0.310259, + "scores_per_experiment": [ + { + "accuracy": 0.323232, + "f1": 0.299249, + "f1_weighted": 0.31122 + }, + { + "accuracy": 0.232323, + "f1": 0.248338, + "f1_weighted": 0.23242 + }, + { + "accuracy": 0.222222, + "f1": 0.206556, + "f1_weighted": 0.201802 + }, + { + "accuracy": 0.383838, + "f1": 0.367819, + "f1_weighted": 0.379071 + }, + { + "accuracy": 0.252525, + "f1": 0.220463, + "f1_weighted": 0.257985 + }, + { + "accuracy": 0.343434, + "f1": 0.345156, + "f1_weighted": 0.343702 + }, + { + "accuracy": 0.292929, + "f1": 0.265929, + "f1_weighted": 0.291924 + }, + { + "accuracy": 0.353535, + "f1": 0.3584, + "f1_weighted": 0.379504 + }, + { + "accuracy": 0.323232, + "f1": 0.284226, + "f1_weighted": 0.321269 + }, + { + "accuracy": 0.383838, + "f1": 0.34373, + "f1_weighted": 0.383691 + } + ], + "main_score": 0.311111, + "hf_subset": "cym_Latn", + "languages": [ + "cym-Latn" + ] + }, + { + "accuracy": 0.488889, + "f1": 0.480216, + "f1_weighted": 0.4891, + "scores_per_experiment": [ + { + "accuracy": 0.474747, + "f1": 0.480205, + "f1_weighted": 0.481301 + }, + { + "accuracy": 0.454545, + "f1": 0.432294, + "f1_weighted": 0.464287 + }, + { + "accuracy": 0.464646, + "f1": 0.481797, + "f1_weighted": 0.440859 + }, + { + "accuracy": 0.545455, + "f1": 0.511105, + "f1_weighted": 0.535576 + }, + { + "accuracy": 0.474747, + "f1": 0.458451, + "f1_weighted": 0.485875 + }, + { + "accuracy": 0.505051, + "f1": 0.489214, + "f1_weighted": 0.510235 + }, + { + "accuracy": 0.515152, + "f1": 0.513849, + "f1_weighted": 0.51852 + }, + { + "accuracy": 0.474747, + "f1": 0.469004, + "f1_weighted": 0.478033 + }, + { + "accuracy": 0.474747, + "f1": 0.45906, + "f1_weighted": 0.469858 + }, + { + "accuracy": 0.505051, + "f1": 0.507181, + "f1_weighted": 0.506454 + } + ], + "main_score": 0.488889, + "hf_subset": "dan_Latn", + "languages": [ + "dan-Latn" + ] + }, + { + "accuracy": 0.583838, + "f1": 0.577369, + "f1_weighted": 0.585403, + "scores_per_experiment": [ + { + "accuracy": 0.59596, + "f1": 0.584477, + "f1_weighted": 0.602639 + }, + { + "accuracy": 0.59596, + "f1": 0.591808, + "f1_weighted": 0.59237 + }, + { + "accuracy": 0.525253, + "f1": 0.548566, + "f1_weighted": 0.507607 + }, + { + "accuracy": 0.656566, + "f1": 0.640454, + "f1_weighted": 0.654755 + }, + { + "accuracy": 0.555556, + "f1": 0.537558, + "f1_weighted": 0.564912 + }, + { + "accuracy": 0.575758, + "f1": 0.572253, + "f1_weighted": 0.594041 + }, + { + "accuracy": 0.59596, + "f1": 0.581764, + "f1_weighted": 0.600525 + }, + { + "accuracy": 0.565657, + "f1": 0.548792, + "f1_weighted": 0.567627 + }, + { + "accuracy": 0.515152, + "f1": 0.515312, + "f1_weighted": 0.517131 + }, + { + "accuracy": 0.656566, + "f1": 0.652707, + "f1_weighted": 0.652426 + } + ], + "main_score": 0.583838, + "hf_subset": "deu_Latn", + "languages": [ + "deu-Latn" + ] + }, + { + "accuracy": 0.372727, + "f1": 0.357212, + "f1_weighted": 0.380337, + "scores_per_experiment": [ + { + "accuracy": 0.313131, + "f1": 0.305159, + "f1_weighted": 0.323026 + }, + { + "accuracy": 0.40404, + "f1": 0.370911, + "f1_weighted": 0.39916 + }, + { + "accuracy": 0.313131, + "f1": 0.326984, + "f1_weighted": 0.322124 + }, + { + "accuracy": 0.444444, + "f1": 0.450897, + "f1_weighted": 0.457735 + }, + { + "accuracy": 0.343434, + "f1": 0.31861, + "f1_weighted": 0.353468 + }, + { + "accuracy": 0.434343, + "f1": 0.45303, + "f1_weighted": 0.446056 + }, + { + "accuracy": 0.353535, + "f1": 0.321447, + "f1_weighted": 0.361724 + }, + { + "accuracy": 0.393939, + "f1": 0.37003, + "f1_weighted": 0.404491 + }, + { + "accuracy": 0.343434, + "f1": 0.309878, + "f1_weighted": 0.34124 + }, + { + "accuracy": 0.383838, + "f1": 0.345173, + "f1_weighted": 0.394344 + } + ], + "main_score": 0.372727, + "hf_subset": "dik_Latn", + "languages": [ + "dik-Latn" + ] + }, + { + "accuracy": 0.281818, + "f1": 0.271702, + "f1_weighted": 0.281739, + "scores_per_experiment": [ + { + "accuracy": 0.30303, + "f1": 0.297849, + "f1_weighted": 0.301271 + }, + { + "accuracy": 0.343434, + "f1": 0.349087, + "f1_weighted": 0.33547 + }, + { + "accuracy": 0.242424, + "f1": 0.246784, + "f1_weighted": 0.240943 + }, + { + "accuracy": 0.242424, + "f1": 0.237301, + "f1_weighted": 0.251328 + }, + { + "accuracy": 0.292929, + "f1": 0.262716, + "f1_weighted": 0.296123 + }, + { + "accuracy": 0.282828, + "f1": 0.268836, + "f1_weighted": 0.285563 + }, + { + "accuracy": 0.262626, + "f1": 0.2586, + "f1_weighted": 0.272501 + }, + { + "accuracy": 0.323232, + "f1": 0.295309, + "f1_weighted": 0.30694 + }, + { + "accuracy": 0.242424, + "f1": 0.229617, + "f1_weighted": 0.249847 + }, + { + "accuracy": 0.282828, + "f1": 0.270922, + "f1_weighted": 0.277405 + } + ], + "main_score": 0.281818, + "hf_subset": "dyu_Latn", + "languages": [ + "dyu-Latn" + ] + }, + { + "accuracy": 0.150505, + "f1": 0.091861, + "f1_weighted": 0.105789, + "scores_per_experiment": [ + { + "accuracy": 0.212121, + "f1": 0.14207, + "f1_weighted": 0.1535 + }, + { + "accuracy": 0.242424, + "f1": 0.094715, + "f1_weighted": 0.148632 + }, + { + "accuracy": 0.060606, + "f1": 0.042837, + "f1_weighted": 0.035487 + }, + { + "accuracy": 0.141414, + "f1": 0.104063, + "f1_weighted": 0.112124 + }, + { + "accuracy": 0.171717, + "f1": 0.130444, + "f1_weighted": 0.159919 + }, + { + "accuracy": 0.161616, + "f1": 0.093692, + "f1_weighted": 0.095107 + }, + { + "accuracy": 0.141414, + "f1": 0.076672, + "f1_weighted": 0.085698 + }, + { + "accuracy": 0.090909, + "f1": 0.056132, + "f1_weighted": 0.060474 + }, + { + "accuracy": 0.191919, + "f1": 0.126205, + "f1_weighted": 0.137235 + }, + { + "accuracy": 0.090909, + "f1": 0.051778, + "f1_weighted": 0.069709 + } + ], + "main_score": 0.150505, + "hf_subset": "dzo_Tibt", + "languages": [ + "dzo-Tibt" + ] + }, + { + "accuracy": 0.184848, + "f1": 0.147124, + "f1_weighted": 0.150048, + "scores_per_experiment": [ + { + "accuracy": 0.191919, + "f1": 0.124529, + "f1_weighted": 0.131813 + }, + { + "accuracy": 0.171717, + "f1": 0.147549, + "f1_weighted": 0.135385 + }, + { + "accuracy": 0.10101, + "f1": 0.104522, + "f1_weighted": 0.082287 + }, + { + "accuracy": 0.252525, + "f1": 0.237457, + "f1_weighted": 0.238373 + }, + { + "accuracy": 0.10101, + "f1": 0.08134, + "f1_weighted": 0.06983 + }, + { + "accuracy": 0.131313, + "f1": 0.107769, + "f1_weighted": 0.12263 + }, + { + "accuracy": 0.252525, + "f1": 0.187588, + "f1_weighted": 0.195941 + }, + { + "accuracy": 0.191919, + "f1": 0.175614, + "f1_weighted": 0.194225 + }, + { + "accuracy": 0.232323, + "f1": 0.155081, + "f1_weighted": 0.173431 + }, + { + "accuracy": 0.222222, + "f1": 0.149788, + "f1_weighted": 0.156564 + } + ], + "main_score": 0.184848, + "hf_subset": "ell_Grek", + "languages": [ + "ell-Grek" + ] + }, + { + "accuracy": 0.683838, + "f1": 0.664022, + "f1_weighted": 0.683618, + "scores_per_experiment": [ + { + "accuracy": 0.646465, + "f1": 0.610313, + "f1_weighted": 0.643004 + }, + { + "accuracy": 0.686869, + "f1": 0.683519, + "f1_weighted": 0.690156 + }, + { + "accuracy": 0.545455, + "f1": 0.532957, + "f1_weighted": 0.534123 + }, + { + "accuracy": 0.757576, + "f1": 0.746451, + "f1_weighted": 0.760179 + }, + { + "accuracy": 0.666667, + "f1": 0.622092, + "f1_weighted": 0.659525 + }, + { + "accuracy": 0.717172, + "f1": 0.696831, + "f1_weighted": 0.723908 + }, + { + "accuracy": 0.686869, + "f1": 0.684223, + "f1_weighted": 0.690321 + }, + { + "accuracy": 0.717172, + "f1": 0.688625, + "f1_weighted": 0.71674 + }, + { + "accuracy": 0.69697, + "f1": 0.68573, + "f1_weighted": 0.702121 + }, + { + "accuracy": 0.717172, + "f1": 0.689483, + "f1_weighted": 0.716107 + } + ], + "main_score": 0.683838, + "hf_subset": "eng_Latn", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.429293, + "f1": 0.406371, + "f1_weighted": 0.437742, + "scores_per_experiment": [ + { + "accuracy": 0.434343, + "f1": 0.392794, + "f1_weighted": 0.453082 + }, + { + "accuracy": 0.444444, + "f1": 0.420813, + "f1_weighted": 0.455537 + }, + { + "accuracy": 0.414141, + "f1": 0.384188, + "f1_weighted": 0.417224 + }, + { + "accuracy": 0.434343, + "f1": 0.407588, + "f1_weighted": 0.44229 + }, + { + "accuracy": 0.414141, + "f1": 0.38661, + "f1_weighted": 0.423586 + }, + { + "accuracy": 0.434343, + "f1": 0.414572, + "f1_weighted": 0.445402 + }, + { + "accuracy": 0.414141, + "f1": 0.38304, + "f1_weighted": 0.414681 + }, + { + "accuracy": 0.434343, + "f1": 0.423928, + "f1_weighted": 0.453848 + }, + { + "accuracy": 0.464646, + "f1": 0.449493, + "f1_weighted": 0.469684 + }, + { + "accuracy": 0.40404, + "f1": 0.400685, + "f1_weighted": 0.402089 + } + ], + "main_score": 0.429293, + "hf_subset": "epo_Latn", + "languages": [ + "epo-Latn" + ] + }, + { + "accuracy": 0.276768, + "f1": 0.26111, + "f1_weighted": 0.275996, + "scores_per_experiment": [ + { + "accuracy": 0.333333, + "f1": 0.328108, + "f1_weighted": 0.330616 + }, + { + "accuracy": 0.262626, + "f1": 0.237686, + "f1_weighted": 0.261289 + }, + { + "accuracy": 0.262626, + "f1": 0.256262, + "f1_weighted": 0.24479 + }, + { + "accuracy": 0.252525, + "f1": 0.244983, + "f1_weighted": 0.263684 + }, + { + "accuracy": 0.292929, + "f1": 0.279784, + "f1_weighted": 0.294642 + }, + { + "accuracy": 0.232323, + "f1": 0.219647, + "f1_weighted": 0.229541 + }, + { + "accuracy": 0.30303, + "f1": 0.279479, + "f1_weighted": 0.309196 + }, + { + "accuracy": 0.282828, + "f1": 0.266312, + "f1_weighted": 0.281087 + }, + { + "accuracy": 0.252525, + "f1": 0.230745, + "f1_weighted": 0.240713 + }, + { + "accuracy": 0.292929, + "f1": 0.268096, + "f1_weighted": 0.304406 + } + ], + "main_score": 0.276768, + "hf_subset": "est_Latn", + "languages": [ + "est-Latn" + ] + }, + { + "accuracy": 0.306061, + "f1": 0.295129, + "f1_weighted": 0.309536, + "scores_per_experiment": [ + { + "accuracy": 0.323232, + "f1": 0.325392, + "f1_weighted": 0.329643 + }, + { + "accuracy": 0.292929, + "f1": 0.288514, + "f1_weighted": 0.292926 + }, + { + "accuracy": 0.252525, + "f1": 0.255026, + "f1_weighted": 0.253589 + }, + { + "accuracy": 0.393939, + "f1": 0.359496, + "f1_weighted": 0.38405 + }, + { + "accuracy": 0.363636, + "f1": 0.350843, + "f1_weighted": 0.364943 + }, + { + "accuracy": 0.252525, + "f1": 0.242194, + "f1_weighted": 0.258882 + }, + { + "accuracy": 0.282828, + "f1": 0.268246, + "f1_weighted": 0.282489 + }, + { + "accuracy": 0.313131, + "f1": 0.316972, + "f1_weighted": 0.347052 + }, + { + "accuracy": 0.282828, + "f1": 0.252752, + "f1_weighted": 0.273815 + }, + { + "accuracy": 0.30303, + "f1": 0.291853, + "f1_weighted": 0.307969 + } + ], + "main_score": 0.306061, + "hf_subset": "eus_Latn", + "languages": [ + "eus-Latn" + ] + }, + { + "accuracy": 0.29697, + "f1": 0.285451, + "f1_weighted": 0.295169, + "scores_per_experiment": [ + { + "accuracy": 0.323232, + "f1": 0.315598, + "f1_weighted": 0.315562 + }, + { + "accuracy": 0.232323, + "f1": 0.214265, + "f1_weighted": 0.215649 + }, + { + "accuracy": 0.323232, + "f1": 0.309622, + "f1_weighted": 0.324822 + }, + { + "accuracy": 0.30303, + "f1": 0.308011, + "f1_weighted": 0.318634 + }, + { + "accuracy": 0.262626, + "f1": 0.248748, + "f1_weighted": 0.254281 + }, + { + "accuracy": 0.292929, + "f1": 0.295043, + "f1_weighted": 0.306229 + }, + { + "accuracy": 0.30303, + "f1": 0.271644, + "f1_weighted": 0.296788 + }, + { + "accuracy": 0.333333, + "f1": 0.318974, + "f1_weighted": 0.331866 + }, + { + "accuracy": 0.333333, + "f1": 0.318973, + "f1_weighted": 0.319372 + }, + { + "accuracy": 0.262626, + "f1": 0.253627, + "f1_weighted": 0.268488 + } + ], + "main_score": 0.29697, + "hf_subset": "ewe_Latn", + "languages": [ + "ewe-Latn" + ] + }, + { + "accuracy": 0.356566, + "f1": 0.327025, + "f1_weighted": 0.359798, + "scores_per_experiment": [ + { + "accuracy": 0.474747, + "f1": 0.421065, + "f1_weighted": 0.473835 + }, + { + "accuracy": 0.292929, + "f1": 0.282113, + "f1_weighted": 0.281233 + }, + { + "accuracy": 0.313131, + "f1": 0.301494, + "f1_weighted": 0.314101 + }, + { + "accuracy": 0.333333, + "f1": 0.318671, + "f1_weighted": 0.327574 + }, + { + "accuracy": 0.383838, + "f1": 0.338714, + "f1_weighted": 0.395389 + }, + { + "accuracy": 0.30303, + "f1": 0.292932, + "f1_weighted": 0.311685 + }, + { + "accuracy": 0.393939, + "f1": 0.352895, + "f1_weighted": 0.414024 + }, + { + "accuracy": 0.424242, + "f1": 0.365688, + "f1_weighted": 0.417275 + }, + { + "accuracy": 0.30303, + "f1": 0.27291, + "f1_weighted": 0.307311 + }, + { + "accuracy": 0.343434, + "f1": 0.323766, + "f1_weighted": 0.355548 + } + ], + "main_score": 0.356566, + "hf_subset": "fao_Latn", + "languages": [ + "fao-Latn" + ] + }, + { + "accuracy": 0.382828, + "f1": 0.35957, + "f1_weighted": 0.382254, + "scores_per_experiment": [ + { + "accuracy": 0.434343, + "f1": 0.440251, + "f1_weighted": 0.440669 + }, + { + "accuracy": 0.282828, + "f1": 0.271525, + "f1_weighted": 0.279146 + }, + { + "accuracy": 0.353535, + "f1": 0.34212, + "f1_weighted": 0.338145 + }, + { + "accuracy": 0.40404, + "f1": 0.365457, + "f1_weighted": 0.407898 + }, + { + "accuracy": 0.373737, + "f1": 0.345242, + "f1_weighted": 0.369584 + }, + { + "accuracy": 0.373737, + "f1": 0.346247, + "f1_weighted": 0.380623 + }, + { + "accuracy": 0.40404, + "f1": 0.35296, + "f1_weighted": 0.401456 + }, + { + "accuracy": 0.30303, + "f1": 0.286272, + "f1_weighted": 0.30712 + }, + { + "accuracy": 0.414141, + "f1": 0.383175, + "f1_weighted": 0.40764 + }, + { + "accuracy": 0.484848, + "f1": 0.462454, + "f1_weighted": 0.49026 + } + ], + "main_score": 0.382828, + "hf_subset": "fij_Latn", + "languages": [ + "fij-Latn" + ] + }, + { + "accuracy": 0.326263, + "f1": 0.306567, + "f1_weighted": 0.337112, + "scores_per_experiment": [ + { + "accuracy": 0.343434, + "f1": 0.298322, + "f1_weighted": 0.341876 + }, + { + "accuracy": 0.313131, + "f1": 0.284073, + "f1_weighted": 0.331088 + }, + { + "accuracy": 0.262626, + "f1": 0.261439, + "f1_weighted": 0.255515 + }, + { + "accuracy": 0.444444, + "f1": 0.41937, + "f1_weighted": 0.45935 + }, + { + "accuracy": 0.252525, + "f1": 0.23134, + "f1_weighted": 0.26854 + }, + { + "accuracy": 0.262626, + "f1": 0.255873, + "f1_weighted": 0.274518 + }, + { + "accuracy": 0.40404, + "f1": 0.37351, + "f1_weighted": 0.403607 + }, + { + "accuracy": 0.333333, + "f1": 0.315477, + "f1_weighted": 0.352249 + }, + { + "accuracy": 0.313131, + "f1": 0.305738, + "f1_weighted": 0.319446 + }, + { + "accuracy": 0.333333, + "f1": 0.320534, + "f1_weighted": 0.364929 + } + ], + "main_score": 0.326263, + "hf_subset": "fin_Latn", + "languages": [ + "fin-Latn" + ] + }, + { + "accuracy": 0.358586, + "f1": 0.339675, + "f1_weighted": 0.353745, + "scores_per_experiment": [ + { + "accuracy": 0.353535, + "f1": 0.338048, + "f1_weighted": 0.348999 + }, + { + "accuracy": 0.30303, + "f1": 0.315111, + "f1_weighted": 0.295748 + }, + { + "accuracy": 0.272727, + "f1": 0.259819, + "f1_weighted": 0.279555 + }, + { + "accuracy": 0.414141, + "f1": 0.415081, + "f1_weighted": 0.411388 + }, + { + "accuracy": 0.292929, + "f1": 0.266569, + "f1_weighted": 0.272954 + }, + { + "accuracy": 0.383838, + "f1": 0.376417, + "f1_weighted": 0.384913 + }, + { + "accuracy": 0.393939, + "f1": 0.355984, + "f1_weighted": 0.387884 + }, + { + "accuracy": 0.494949, + "f1": 0.463567, + "f1_weighted": 0.490152 + }, + { + "accuracy": 0.353535, + "f1": 0.344567, + "f1_weighted": 0.352959 + }, + { + "accuracy": 0.323232, + "f1": 0.261584, + "f1_weighted": 0.312897 + } + ], + "main_score": 0.358586, + "hf_subset": "fon_Latn", + "languages": [ + "fon-Latn" + ] + }, + { + "accuracy": 0.611111, + "f1": 0.60937, + "f1_weighted": 0.618821, + "scores_per_experiment": [ + { + "accuracy": 0.656566, + "f1": 0.622873, + "f1_weighted": 0.664133 + }, + { + "accuracy": 0.555556, + "f1": 0.551229, + "f1_weighted": 0.562048 + }, + { + "accuracy": 0.515152, + "f1": 0.520519, + "f1_weighted": 0.504458 + }, + { + "accuracy": 0.707071, + "f1": 0.698798, + "f1_weighted": 0.709516 + }, + { + "accuracy": 0.575758, + "f1": 0.594349, + "f1_weighted": 0.582804 + }, + { + "accuracy": 0.616162, + "f1": 0.61611, + "f1_weighted": 0.63887 + }, + { + "accuracy": 0.616162, + "f1": 0.614092, + "f1_weighted": 0.620435 + }, + { + "accuracy": 0.585859, + "f1": 0.587791, + "f1_weighted": 0.606004 + }, + { + "accuracy": 0.575758, + "f1": 0.581568, + "f1_weighted": 0.584739 + }, + { + "accuracy": 0.707071, + "f1": 0.706371, + "f1_weighted": 0.715202 + } + ], + "main_score": 0.611111, + "hf_subset": "fra_Latn", + "languages": [ + "fra-Latn" + ] + }, + { + "accuracy": 0.464646, + "f1": 0.441026, + "f1_weighted": 0.475369, + "scores_per_experiment": [ + { + "accuracy": 0.575758, + "f1": 0.553839, + "f1_weighted": 0.574781 + }, + { + "accuracy": 0.444444, + "f1": 0.40855, + "f1_weighted": 0.457666 + }, + { + "accuracy": 0.393939, + "f1": 0.409261, + "f1_weighted": 0.391603 + }, + { + "accuracy": 0.555556, + "f1": 0.531008, + "f1_weighted": 0.569278 + }, + { + "accuracy": 0.444444, + "f1": 0.40627, + "f1_weighted": 0.451093 + }, + { + "accuracy": 0.474747, + "f1": 0.463987, + "f1_weighted": 0.486795 + }, + { + "accuracy": 0.393939, + "f1": 0.369812, + "f1_weighted": 0.410618 + }, + { + "accuracy": 0.414141, + "f1": 0.400907, + "f1_weighted": 0.430482 + }, + { + "accuracy": 0.414141, + "f1": 0.388118, + "f1_weighted": 0.426355 + }, + { + "accuracy": 0.535354, + "f1": 0.478504, + "f1_weighted": 0.55502 + } + ], + "main_score": 0.464646, + "hf_subset": "fur_Latn", + "languages": [ + "fur-Latn" + ] + }, + { + "accuracy": 0.321212, + "f1": 0.316907, + "f1_weighted": 0.32381, + "scores_per_experiment": [ + { + "accuracy": 0.333333, + "f1": 0.333654, + "f1_weighted": 0.334318 + }, + { + "accuracy": 0.353535, + "f1": 0.363755, + "f1_weighted": 0.358004 + }, + { + "accuracy": 0.323232, + "f1": 0.322826, + "f1_weighted": 0.327127 + }, + { + "accuracy": 0.383838, + "f1": 0.387755, + "f1_weighted": 0.397667 + }, + { + "accuracy": 0.252525, + "f1": 0.237989, + "f1_weighted": 0.256376 + }, + { + "accuracy": 0.333333, + "f1": 0.329823, + "f1_weighted": 0.33629 + }, + { + "accuracy": 0.373737, + "f1": 0.343766, + "f1_weighted": 0.38429 + }, + { + "accuracy": 0.333333, + "f1": 0.329326, + "f1_weighted": 0.328639 + }, + { + "accuracy": 0.282828, + "f1": 0.282429, + "f1_weighted": 0.288911 + }, + { + "accuracy": 0.242424, + "f1": 0.237742, + "f1_weighted": 0.226475 + } + ], + "main_score": 0.321212, + "hf_subset": "fuv_Latn", + "languages": [ + "fuv-Latn" + ] + }, + { + "accuracy": 0.241414, + "f1": 0.230295, + "f1_weighted": 0.243388, + "scores_per_experiment": [ + { + "accuracy": 0.252525, + "f1": 0.248182, + "f1_weighted": 0.244867 + }, + { + "accuracy": 0.272727, + "f1": 0.27154, + "f1_weighted": 0.260343 + }, + { + "accuracy": 0.232323, + "f1": 0.222654, + "f1_weighted": 0.233302 + }, + { + "accuracy": 0.242424, + "f1": 0.210418, + "f1_weighted": 0.237642 + }, + { + "accuracy": 0.272727, + "f1": 0.26117, + "f1_weighted": 0.277047 + }, + { + "accuracy": 0.191919, + "f1": 0.176206, + "f1_weighted": 0.205401 + }, + { + "accuracy": 0.252525, + "f1": 0.242032, + "f1_weighted": 0.251632 + }, + { + "accuracy": 0.232323, + "f1": 0.231773, + "f1_weighted": 0.240536 + }, + { + "accuracy": 0.262626, + "f1": 0.255936, + "f1_weighted": 0.282224 + }, + { + "accuracy": 0.20202, + "f1": 0.183039, + "f1_weighted": 0.20089 + } + ], + "main_score": 0.241414, + "hf_subset": "gaz_Latn", + "languages": [ + "gaz-Latn" + ] + }, + { + "accuracy": 0.251515, + "f1": 0.247137, + "f1_weighted": 0.247912, + "scores_per_experiment": [ + { + "accuracy": 0.282828, + "f1": 0.302202, + "f1_weighted": 0.279144 + }, + { + "accuracy": 0.242424, + "f1": 0.239318, + "f1_weighted": 0.257622 + }, + { + "accuracy": 0.232323, + "f1": 0.236149, + "f1_weighted": 0.209135 + }, + { + "accuracy": 0.262626, + "f1": 0.250594, + "f1_weighted": 0.2518 + }, + { + "accuracy": 0.171717, + "f1": 0.161606, + "f1_weighted": 0.177556 + }, + { + "accuracy": 0.212121, + "f1": 0.201844, + "f1_weighted": 0.206312 + }, + { + "accuracy": 0.262626, + "f1": 0.255416, + "f1_weighted": 0.25062 + }, + { + "accuracy": 0.313131, + "f1": 0.287484, + "f1_weighted": 0.301722 + }, + { + "accuracy": 0.282828, + "f1": 0.282372, + "f1_weighted": 0.28815 + }, + { + "accuracy": 0.252525, + "f1": 0.25438, + "f1_weighted": 0.257055 + } + ], + "main_score": 0.251515, + "hf_subset": "gla_Latn", + "languages": [ + "gla-Latn" + ] + }, + { + "accuracy": 0.270707, + "f1": 0.261739, + "f1_weighted": 0.275649, + "scores_per_experiment": [ + { + "accuracy": 0.222222, + "f1": 0.218984, + "f1_weighted": 0.213451 + }, + { + "accuracy": 0.262626, + "f1": 0.255706, + "f1_weighted": 0.275328 + }, + { + "accuracy": 0.252525, + "f1": 0.243663, + "f1_weighted": 0.250482 + }, + { + "accuracy": 0.313131, + "f1": 0.313099, + "f1_weighted": 0.319028 + }, + { + "accuracy": 0.262626, + "f1": 0.254475, + "f1_weighted": 0.284513 + }, + { + "accuracy": 0.282828, + "f1": 0.268358, + "f1_weighted": 0.285453 + }, + { + "accuracy": 0.272727, + "f1": 0.289381, + "f1_weighted": 0.265389 + }, + { + "accuracy": 0.282828, + "f1": 0.271489, + "f1_weighted": 0.286743 + }, + { + "accuracy": 0.272727, + "f1": 0.234799, + "f1_weighted": 0.279463 + }, + { + "accuracy": 0.282828, + "f1": 0.267436, + "f1_weighted": 0.296637 + } + ], + "main_score": 0.270707, + "hf_subset": "gle_Latn", + "languages": [ + "gle-Latn" + ] + }, + { + "accuracy": 0.472727, + "f1": 0.451405, + "f1_weighted": 0.480802, + "scores_per_experiment": [ + { + "accuracy": 0.505051, + "f1": 0.47622, + "f1_weighted": 0.512706 + }, + { + "accuracy": 0.494949, + "f1": 0.457058, + "f1_weighted": 0.503462 + }, + { + "accuracy": 0.40404, + "f1": 0.394721, + "f1_weighted": 0.406674 + }, + { + "accuracy": 0.565657, + "f1": 0.545061, + "f1_weighted": 0.565969 + }, + { + "accuracy": 0.444444, + "f1": 0.412661, + "f1_weighted": 0.45906 + }, + { + "accuracy": 0.40404, + "f1": 0.400525, + "f1_weighted": 0.414425 + }, + { + "accuracy": 0.474747, + "f1": 0.483966, + "f1_weighted": 0.495668 + }, + { + "accuracy": 0.474747, + "f1": 0.460078, + "f1_weighted": 0.483862 + }, + { + "accuracy": 0.464646, + "f1": 0.416532, + "f1_weighted": 0.462761 + }, + { + "accuracy": 0.494949, + "f1": 0.467222, + "f1_weighted": 0.503437 + } + ], + "main_score": 0.472727, + "hf_subset": "glg_Latn", + "languages": [ + "glg-Latn" + ] + }, + { + "accuracy": 0.365657, + "f1": 0.35809, + "f1_weighted": 0.368093, + "scores_per_experiment": [ + { + "accuracy": 0.393939, + "f1": 0.388861, + "f1_weighted": 0.384689 + }, + { + "accuracy": 0.353535, + "f1": 0.335711, + "f1_weighted": 0.360556 + }, + { + "accuracy": 0.363636, + "f1": 0.364188, + "f1_weighted": 0.377455 + }, + { + "accuracy": 0.313131, + "f1": 0.315227, + "f1_weighted": 0.31817 + }, + { + "accuracy": 0.414141, + "f1": 0.398785, + "f1_weighted": 0.406992 + }, + { + "accuracy": 0.40404, + "f1": 0.401681, + "f1_weighted": 0.400796 + }, + { + "accuracy": 0.313131, + "f1": 0.321596, + "f1_weighted": 0.314621 + }, + { + "accuracy": 0.353535, + "f1": 0.322363, + "f1_weighted": 0.356682 + }, + { + "accuracy": 0.414141, + "f1": 0.403499, + "f1_weighted": 0.421096 + }, + { + "accuracy": 0.333333, + "f1": 0.328985, + "f1_weighted": 0.339877 + } + ], + "main_score": 0.365657, + "hf_subset": "grn_Latn", + "languages": [ + "grn-Latn" + ] + }, + { + "accuracy": 0.181818, + "f1": 0.143225, + "f1_weighted": 0.145577, + "scores_per_experiment": [ + { + "accuracy": 0.252525, + "f1": 0.184259, + "f1_weighted": 0.188618 + }, + { + "accuracy": 0.161616, + "f1": 0.146484, + "f1_weighted": 0.135501 + }, + { + "accuracy": 0.161616, + "f1": 0.158416, + "f1_weighted": 0.161288 + }, + { + "accuracy": 0.212121, + "f1": 0.182289, + "f1_weighted": 0.163872 + }, + { + "accuracy": 0.090909, + "f1": 0.073086, + "f1_weighted": 0.073526 + }, + { + "accuracy": 0.141414, + "f1": 0.117509, + "f1_weighted": 0.114019 + }, + { + "accuracy": 0.181818, + "f1": 0.170071, + "f1_weighted": 0.168232 + }, + { + "accuracy": 0.171717, + "f1": 0.111001, + "f1_weighted": 0.123328 + }, + { + "accuracy": 0.292929, + "f1": 0.165553, + "f1_weighted": 0.207533 + }, + { + "accuracy": 0.151515, + "f1": 0.123582, + "f1_weighted": 0.119849 + } + ], + "main_score": 0.181818, + "hf_subset": "guj_Gujr", + "languages": [ + "guj-Gujr" + ] + }, + { + "accuracy": 0.292929, + "f1": 0.278554, + "f1_weighted": 0.295471, + "scores_per_experiment": [ + { + "accuracy": 0.363636, + "f1": 0.349228, + "f1_weighted": 0.359582 + }, + { + "accuracy": 0.252525, + "f1": 0.255846, + "f1_weighted": 0.253315 + }, + { + "accuracy": 0.242424, + "f1": 0.231356, + "f1_weighted": 0.239852 + }, + { + "accuracy": 0.262626, + "f1": 0.255847, + "f1_weighted": 0.268531 + }, + { + "accuracy": 0.262626, + "f1": 0.259417, + "f1_weighted": 0.261153 + }, + { + "accuracy": 0.333333, + "f1": 0.289596, + "f1_weighted": 0.335948 + }, + { + "accuracy": 0.313131, + "f1": 0.315629, + "f1_weighted": 0.322161 + }, + { + "accuracy": 0.282828, + "f1": 0.255394, + "f1_weighted": 0.281316 + }, + { + "accuracy": 0.30303, + "f1": 0.27891, + "f1_weighted": 0.307682 + }, + { + "accuracy": 0.313131, + "f1": 0.294313, + "f1_weighted": 0.325168 + } + ], + "main_score": 0.292929, + "hf_subset": "hat_Latn", + "languages": [ + "hat-Latn" + ] + }, + { + "accuracy": 0.30202, + "f1": 0.287867, + "f1_weighted": 0.301539, + "scores_per_experiment": [ + { + "accuracy": 0.292929, + "f1": 0.28392, + "f1_weighted": 0.288395 + }, + { + "accuracy": 0.292929, + "f1": 0.303007, + "f1_weighted": 0.287222 + }, + { + "accuracy": 0.282828, + "f1": 0.267171, + "f1_weighted": 0.274898 + }, + { + "accuracy": 0.343434, + "f1": 0.316149, + "f1_weighted": 0.342048 + }, + { + "accuracy": 0.333333, + "f1": 0.294359, + "f1_weighted": 0.335422 + }, + { + "accuracy": 0.292929, + "f1": 0.285331, + "f1_weighted": 0.280572 + }, + { + "accuracy": 0.252525, + "f1": 0.245165, + "f1_weighted": 0.260223 + }, + { + "accuracy": 0.343434, + "f1": 0.326206, + "f1_weighted": 0.349741 + }, + { + "accuracy": 0.232323, + "f1": 0.210655, + "f1_weighted": 0.244751 + }, + { + "accuracy": 0.353535, + "f1": 0.346703, + "f1_weighted": 0.352117 + } + ], + "main_score": 0.30202, + "hf_subset": "hau_Latn", + "languages": [ + "hau-Latn" + ] + }, + { + "accuracy": 0.20202, + "f1": 0.162981, + "f1_weighted": 0.170989, + "scores_per_experiment": [ + { + "accuracy": 0.232323, + "f1": 0.145671, + "f1_weighted": 0.157382 + }, + { + "accuracy": 0.242424, + "f1": 0.18962, + "f1_weighted": 0.223257 + }, + { + "accuracy": 0.131313, + "f1": 0.127, + "f1_weighted": 0.105967 + }, + { + "accuracy": 0.191919, + "f1": 0.168828, + "f1_weighted": 0.151539 + }, + { + "accuracy": 0.20202, + "f1": 0.12953, + "f1_weighted": 0.145857 + }, + { + "accuracy": 0.212121, + "f1": 0.194246, + "f1_weighted": 0.189702 + }, + { + "accuracy": 0.232323, + "f1": 0.202389, + "f1_weighted": 0.217571 + }, + { + "accuracy": 0.181818, + "f1": 0.1445, + "f1_weighted": 0.170549 + }, + { + "accuracy": 0.191919, + "f1": 0.186479, + "f1_weighted": 0.208712 + }, + { + "accuracy": 0.20202, + "f1": 0.141543, + "f1_weighted": 0.139352 + } + ], + "main_score": 0.20202, + "hf_subset": "heb_Hebr", + "languages": [ + "heb-Hebr" + ] + }, + { + "accuracy": 0.192929, + "f1": 0.166954, + "f1_weighted": 0.173149, + "scores_per_experiment": [ + { + "accuracy": 0.242424, + "f1": 0.177759, + "f1_weighted": 0.179715 + }, + { + "accuracy": 0.181818, + "f1": 0.141324, + "f1_weighted": 0.143355 + }, + { + "accuracy": 0.151515, + "f1": 0.132049, + "f1_weighted": 0.140208 + }, + { + "accuracy": 0.222222, + "f1": 0.213599, + "f1_weighted": 0.188806 + }, + { + "accuracy": 0.10101, + "f1": 0.088409, + "f1_weighted": 0.084366 + }, + { + "accuracy": 0.151515, + "f1": 0.147045, + "f1_weighted": 0.167215 + }, + { + "accuracy": 0.242424, + "f1": 0.184343, + "f1_weighted": 0.221015 + }, + { + "accuracy": 0.222222, + "f1": 0.23486, + "f1_weighted": 0.21456 + }, + { + "accuracy": 0.161616, + "f1": 0.130137, + "f1_weighted": 0.138537 + }, + { + "accuracy": 0.252525, + "f1": 0.220012, + "f1_weighted": 0.253717 + } + ], + "main_score": 0.192929, + "hf_subset": "hin_Deva", + "languages": [ + "hin-Deva" + ] + }, + { + "accuracy": 0.190909, + "f1": 0.141634, + "f1_weighted": 0.153082, + "scores_per_experiment": [ + { + "accuracy": 0.171717, + "f1": 0.168162, + "f1_weighted": 0.164693 + }, + { + "accuracy": 0.151515, + "f1": 0.100668, + "f1_weighted": 0.109221 + }, + { + "accuracy": 0.222222, + "f1": 0.183764, + "f1_weighted": 0.203904 + }, + { + "accuracy": 0.191919, + "f1": 0.197321, + "f1_weighted": 0.167571 + }, + { + "accuracy": 0.141414, + "f1": 0.102095, + "f1_weighted": 0.106827 + }, + { + "accuracy": 0.212121, + "f1": 0.139078, + "f1_weighted": 0.192072 + }, + { + "accuracy": 0.212121, + "f1": 0.127044, + "f1_weighted": 0.14587 + }, + { + "accuracy": 0.20202, + "f1": 0.145586, + "f1_weighted": 0.15165 + }, + { + "accuracy": 0.20202, + "f1": 0.126304, + "f1_weighted": 0.155251 + }, + { + "accuracy": 0.20202, + "f1": 0.126322, + "f1_weighted": 0.133757 + } + ], + "main_score": 0.190909, + "hf_subset": "hne_Deva", + "languages": [ + "hne-Deva" + ] + }, + { + "accuracy": 0.344444, + "f1": 0.327657, + "f1_weighted": 0.351808, + "scores_per_experiment": [ + { + "accuracy": 0.333333, + "f1": 0.323865, + "f1_weighted": 0.351743 + }, + { + "accuracy": 0.353535, + "f1": 0.341981, + "f1_weighted": 0.378798 + }, + { + "accuracy": 0.30303, + "f1": 0.314952, + "f1_weighted": 0.298167 + }, + { + "accuracy": 0.30303, + "f1": 0.299267, + "f1_weighted": 0.304028 + }, + { + "accuracy": 0.333333, + "f1": 0.325161, + "f1_weighted": 0.34053 + }, + { + "accuracy": 0.313131, + "f1": 0.29353, + "f1_weighted": 0.317207 + }, + { + "accuracy": 0.373737, + "f1": 0.343427, + "f1_weighted": 0.38114 + }, + { + "accuracy": 0.40404, + "f1": 0.349093, + "f1_weighted": 0.400542 + }, + { + "accuracy": 0.414141, + "f1": 0.377329, + "f1_weighted": 0.411848 + }, + { + "accuracy": 0.313131, + "f1": 0.307964, + "f1_weighted": 0.334078 + } + ], + "main_score": 0.344444, + "hf_subset": "hrv_Latn", + "languages": [ + "hrv-Latn" + ] + }, + { + "accuracy": 0.272727, + "f1": 0.266392, + "f1_weighted": 0.27916, + "scores_per_experiment": [ + { + "accuracy": 0.333333, + "f1": 0.336381, + "f1_weighted": 0.328912 + }, + { + "accuracy": 0.242424, + "f1": 0.239347, + "f1_weighted": 0.236983 + }, + { + "accuracy": 0.232323, + "f1": 0.234695, + "f1_weighted": 0.231759 + }, + { + "accuracy": 0.242424, + "f1": 0.22283, + "f1_weighted": 0.245161 + }, + { + "accuracy": 0.333333, + "f1": 0.310767, + "f1_weighted": 0.339004 + }, + { + "accuracy": 0.313131, + "f1": 0.290742, + "f1_weighted": 0.330442 + }, + { + "accuracy": 0.242424, + "f1": 0.236, + "f1_weighted": 0.268227 + }, + { + "accuracy": 0.242424, + "f1": 0.254976, + "f1_weighted": 0.253982 + }, + { + "accuracy": 0.313131, + "f1": 0.305454, + "f1_weighted": 0.322652 + }, + { + "accuracy": 0.232323, + "f1": 0.232726, + "f1_weighted": 0.234481 + } + ], + "main_score": 0.272727, + "hf_subset": "hun_Latn", + "languages": [ + "hun-Latn" + ] + }, + { + "accuracy": 0.19899, + "f1": 0.164369, + "f1_weighted": 0.167202, + "scores_per_experiment": [ + { + "accuracy": 0.20202, + "f1": 0.164541, + "f1_weighted": 0.158719 + }, + { + "accuracy": 0.171717, + "f1": 0.144115, + "f1_weighted": 0.153246 + }, + { + "accuracy": 0.10101, + "f1": 0.071412, + "f1_weighted": 0.083514 + }, + { + "accuracy": 0.252525, + "f1": 0.246232, + "f1_weighted": 0.259533 + }, + { + "accuracy": 0.161616, + "f1": 0.116068, + "f1_weighted": 0.136672 + }, + { + "accuracy": 0.20202, + "f1": 0.161956, + "f1_weighted": 0.152798 + }, + { + "accuracy": 0.20202, + "f1": 0.154963, + "f1_weighted": 0.141722 + }, + { + "accuracy": 0.282828, + "f1": 0.261065, + "f1_weighted": 0.26206 + }, + { + "accuracy": 0.20202, + "f1": 0.187995, + "f1_weighted": 0.184073 + }, + { + "accuracy": 0.212121, + "f1": 0.135348, + "f1_weighted": 0.139681 + } + ], + "main_score": 0.19899, + "hf_subset": "hye_Armn", + "languages": [ + "hye-Armn" + ] + }, + { + "accuracy": 0.30303, + "f1": 0.296633, + "f1_weighted": 0.304237, + "scores_per_experiment": [ + { + "accuracy": 0.323232, + "f1": 0.324998, + "f1_weighted": 0.320239 + }, + { + "accuracy": 0.292929, + "f1": 0.291229, + "f1_weighted": 0.296346 + }, + { + "accuracy": 0.252525, + "f1": 0.243894, + "f1_weighted": 0.25885 + }, + { + "accuracy": 0.343434, + "f1": 0.321116, + "f1_weighted": 0.341553 + }, + { + "accuracy": 0.292929, + "f1": 0.271035, + "f1_weighted": 0.284147 + }, + { + "accuracy": 0.242424, + "f1": 0.24665, + "f1_weighted": 0.235963 + }, + { + "accuracy": 0.292929, + "f1": 0.288546, + "f1_weighted": 0.303012 + }, + { + "accuracy": 0.343434, + "f1": 0.337618, + "f1_weighted": 0.349088 + }, + { + "accuracy": 0.30303, + "f1": 0.300824, + "f1_weighted": 0.310418 + }, + { + "accuracy": 0.343434, + "f1": 0.340422, + "f1_weighted": 0.342757 + } + ], + "main_score": 0.30303, + "hf_subset": "ibo_Latn", + "languages": [ + "ibo-Latn" + ] + }, + { + "accuracy": 0.460606, + "f1": 0.428795, + "f1_weighted": 0.468733, + "scores_per_experiment": [ + { + "accuracy": 0.494949, + "f1": 0.465326, + "f1_weighted": 0.505075 + }, + { + "accuracy": 0.474747, + "f1": 0.437306, + "f1_weighted": 0.480622 + }, + { + "accuracy": 0.40404, + "f1": 0.397889, + "f1_weighted": 0.39837 + }, + { + "accuracy": 0.494949, + "f1": 0.458346, + "f1_weighted": 0.499204 + }, + { + "accuracy": 0.363636, + "f1": 0.339961, + "f1_weighted": 0.391098 + }, + { + "accuracy": 0.474747, + "f1": 0.428404, + "f1_weighted": 0.476215 + }, + { + "accuracy": 0.494949, + "f1": 0.461093, + "f1_weighted": 0.487565 + }, + { + "accuracy": 0.525253, + "f1": 0.493183, + "f1_weighted": 0.545744 + }, + { + "accuracy": 0.414141, + "f1": 0.36751, + "f1_weighted": 0.413679 + }, + { + "accuracy": 0.464646, + "f1": 0.438928, + "f1_weighted": 0.489757 + } + ], + "main_score": 0.460606, + "hf_subset": "ilo_Latn", + "languages": [ + "ilo-Latn" + ] + }, + { + "accuracy": 0.380808, + "f1": 0.361872, + "f1_weighted": 0.38596, + "scores_per_experiment": [ + { + "accuracy": 0.474747, + "f1": 0.473624, + "f1_weighted": 0.484993 + }, + { + "accuracy": 0.353535, + "f1": 0.322465, + "f1_weighted": 0.358977 + }, + { + "accuracy": 0.343434, + "f1": 0.322208, + "f1_weighted": 0.341423 + }, + { + "accuracy": 0.393939, + "f1": 0.377029, + "f1_weighted": 0.402992 + }, + { + "accuracy": 0.272727, + "f1": 0.246329, + "f1_weighted": 0.28864 + }, + { + "accuracy": 0.393939, + "f1": 0.385234, + "f1_weighted": 0.389141 + }, + { + "accuracy": 0.414141, + "f1": 0.396477, + "f1_weighted": 0.427899 + }, + { + "accuracy": 0.373737, + "f1": 0.360237, + "f1_weighted": 0.381935 + }, + { + "accuracy": 0.353535, + "f1": 0.309422, + "f1_weighted": 0.341997 + }, + { + "accuracy": 0.434343, + "f1": 0.425696, + "f1_weighted": 0.4416 + } + ], + "main_score": 0.380808, + "hf_subset": "ind_Latn", + "languages": [ + "ind-Latn" + ] + }, + { + "accuracy": 0.284848, + "f1": 0.264261, + "f1_weighted": 0.285538, + "scores_per_experiment": [ + { + "accuracy": 0.292929, + "f1": 0.27797, + "f1_weighted": 0.302516 + }, + { + "accuracy": 0.181818, + "f1": 0.181929, + "f1_weighted": 0.182363 + }, + { + "accuracy": 0.272727, + "f1": 0.250293, + "f1_weighted": 0.282211 + }, + { + "accuracy": 0.282828, + "f1": 0.285458, + "f1_weighted": 0.283239 + }, + { + "accuracy": 0.292929, + "f1": 0.259588, + "f1_weighted": 0.291536 + }, + { + "accuracy": 0.313131, + "f1": 0.290292, + "f1_weighted": 0.320051 + }, + { + "accuracy": 0.30303, + "f1": 0.243903, + "f1_weighted": 0.279219 + }, + { + "accuracy": 0.323232, + "f1": 0.310178, + "f1_weighted": 0.327361 + }, + { + "accuracy": 0.313131, + "f1": 0.262121, + "f1_weighted": 0.298546 + }, + { + "accuracy": 0.272727, + "f1": 0.280873, + "f1_weighted": 0.288343 + } + ], + "main_score": 0.284848, + "hf_subset": "isl_Latn", + "languages": [ + "isl-Latn" + ] + }, + { + "accuracy": 0.535354, + "f1": 0.510093, + "f1_weighted": 0.536167, + "scores_per_experiment": [ + { + "accuracy": 0.545455, + "f1": 0.528838, + "f1_weighted": 0.555739 + }, + { + "accuracy": 0.484848, + "f1": 0.474948, + "f1_weighted": 0.491633 + }, + { + "accuracy": 0.505051, + "f1": 0.500657, + "f1_weighted": 0.510515 + }, + { + "accuracy": 0.616162, + "f1": 0.571982, + "f1_weighted": 0.609955 + }, + { + "accuracy": 0.454545, + "f1": 0.42304, + "f1_weighted": 0.462439 + }, + { + "accuracy": 0.515152, + "f1": 0.504176, + "f1_weighted": 0.514214 + }, + { + "accuracy": 0.565657, + "f1": 0.54876, + "f1_weighted": 0.566126 + }, + { + "accuracy": 0.565657, + "f1": 0.528168, + "f1_weighted": 0.562925 + }, + { + "accuracy": 0.525253, + "f1": 0.490686, + "f1_weighted": 0.519373 + }, + { + "accuracy": 0.575758, + "f1": 0.52968, + "f1_weighted": 0.568754 + } + ], + "main_score": 0.535354, + "hf_subset": "ita_Latn", + "languages": [ + "ita-Latn" + ] + }, + { + "accuracy": 0.339394, + "f1": 0.327528, + "f1_weighted": 0.342324, + "scores_per_experiment": [ + { + "accuracy": 0.414141, + "f1": 0.415131, + "f1_weighted": 0.414522 + }, + { + "accuracy": 0.313131, + "f1": 0.280356, + "f1_weighted": 0.321748 + }, + { + "accuracy": 0.30303, + "f1": 0.314883, + "f1_weighted": 0.302478 + }, + { + "accuracy": 0.343434, + "f1": 0.323707, + "f1_weighted": 0.347133 + }, + { + "accuracy": 0.222222, + "f1": 0.214997, + "f1_weighted": 0.232523 + }, + { + "accuracy": 0.333333, + "f1": 0.314977, + "f1_weighted": 0.326374 + }, + { + "accuracy": 0.333333, + "f1": 0.321753, + "f1_weighted": 0.336962 + }, + { + "accuracy": 0.40404, + "f1": 0.376506, + "f1_weighted": 0.404853 + }, + { + "accuracy": 0.30303, + "f1": 0.288207, + "f1_weighted": 0.305069 + }, + { + "accuracy": 0.424242, + "f1": 0.424766, + "f1_weighted": 0.431579 + } + ], + "main_score": 0.339394, + "hf_subset": "jav_Latn", + "languages": [ + "jav-Latn" + ] + }, + { + "accuracy": 0.220202, + "f1": 0.145254, + "f1_weighted": 0.162168, + "scores_per_experiment": [ + { + "accuracy": 0.232323, + "f1": 0.134033, + "f1_weighted": 0.15093 + }, + { + "accuracy": 0.171717, + "f1": 0.118711, + "f1_weighted": 0.10143 + }, + { + "accuracy": 0.111111, + "f1": 0.113226, + "f1_weighted": 0.116459 + }, + { + "accuracy": 0.292929, + "f1": 0.189669, + "f1_weighted": 0.21561 + }, + { + "accuracy": 0.232323, + "f1": 0.155593, + "f1_weighted": 0.177116 + }, + { + "accuracy": 0.232323, + "f1": 0.133877, + "f1_weighted": 0.174232 + }, + { + "accuracy": 0.232323, + "f1": 0.139309, + "f1_weighted": 0.170818 + }, + { + "accuracy": 0.262626, + "f1": 0.163781, + "f1_weighted": 0.188406 + }, + { + "accuracy": 0.222222, + "f1": 0.158182, + "f1_weighted": 0.17719 + }, + { + "accuracy": 0.212121, + "f1": 0.146155, + "f1_weighted": 0.149489 + } + ], + "main_score": 0.220202, + "hf_subset": "jpn_Jpan", + "languages": [ + "jpn-Jpan" + ] + }, + { + "accuracy": 0.250505, + "f1": 0.241218, + "f1_weighted": 0.243121, + "scores_per_experiment": [ + { + "accuracy": 0.282828, + "f1": 0.257784, + "f1_weighted": 0.2727 + }, + { + "accuracy": 0.20202, + "f1": 0.213779, + "f1_weighted": 0.172793 + }, + { + "accuracy": 0.171717, + "f1": 0.150535, + "f1_weighted": 0.16735 + }, + { + "accuracy": 0.323232, + "f1": 0.326992, + "f1_weighted": 0.324015 + }, + { + "accuracy": 0.343434, + "f1": 0.329816, + "f1_weighted": 0.342533 + }, + { + "accuracy": 0.232323, + "f1": 0.209772, + "f1_weighted": 0.227886 + }, + { + "accuracy": 0.212121, + "f1": 0.211833, + "f1_weighted": 0.203552 + }, + { + "accuracy": 0.272727, + "f1": 0.259878, + "f1_weighted": 0.268885 + }, + { + "accuracy": 0.232323, + "f1": 0.228133, + "f1_weighted": 0.237278 + }, + { + "accuracy": 0.232323, + "f1": 0.223661, + "f1_weighted": 0.214221 + } + ], + "main_score": 0.250505, + "hf_subset": "kab_Latn", + "languages": [ + "kab-Latn" + ] + }, + { + "accuracy": 0.359596, + "f1": 0.352201, + "f1_weighted": 0.369558, + "scores_per_experiment": [ + { + "accuracy": 0.383838, + "f1": 0.391454, + "f1_weighted": 0.400284 + }, + { + "accuracy": 0.323232, + "f1": 0.318109, + "f1_weighted": 0.328176 + }, + { + "accuracy": 0.282828, + "f1": 0.296725, + "f1_weighted": 0.289074 + }, + { + "accuracy": 0.414141, + "f1": 0.406053, + "f1_weighted": 0.419914 + }, + { + "accuracy": 0.444444, + "f1": 0.425753, + "f1_weighted": 0.462667 + }, + { + "accuracy": 0.383838, + "f1": 0.359044, + "f1_weighted": 0.364525 + }, + { + "accuracy": 0.313131, + "f1": 0.290557, + "f1_weighted": 0.322389 + }, + { + "accuracy": 0.333333, + "f1": 0.33621, + "f1_weighted": 0.355136 + }, + { + "accuracy": 0.333333, + "f1": 0.326325, + "f1_weighted": 0.35443 + }, + { + "accuracy": 0.383838, + "f1": 0.371784, + "f1_weighted": 0.398989 + } + ], + "main_score": 0.359596, + "hf_subset": "kac_Latn", + "languages": [ + "kac-Latn" + ] + }, + { + "accuracy": 0.320202, + "f1": 0.313893, + "f1_weighted": 0.322571, + "scores_per_experiment": [ + { + "accuracy": 0.313131, + "f1": 0.303772, + "f1_weighted": 0.311505 + }, + { + "accuracy": 0.222222, + "f1": 0.211242, + "f1_weighted": 0.238715 + }, + { + "accuracy": 0.282828, + "f1": 0.295339, + "f1_weighted": 0.276875 + }, + { + "accuracy": 0.323232, + "f1": 0.313422, + "f1_weighted": 0.303441 + }, + { + "accuracy": 0.313131, + "f1": 0.294481, + "f1_weighted": 0.312671 + }, + { + "accuracy": 0.323232, + "f1": 0.32238, + "f1_weighted": 0.325761 + }, + { + "accuracy": 0.292929, + "f1": 0.285592, + "f1_weighted": 0.287778 + }, + { + "accuracy": 0.363636, + "f1": 0.354979, + "f1_weighted": 0.382449 + }, + { + "accuracy": 0.40404, + "f1": 0.395249, + "f1_weighted": 0.408473 + }, + { + "accuracy": 0.363636, + "f1": 0.362481, + "f1_weighted": 0.378038 + } + ], + "main_score": 0.320202, + "hf_subset": "kam_Latn", + "languages": [ + "kam-Latn" + ] + }, + { + "accuracy": 0.181818, + "f1": 0.146631, + "f1_weighted": 0.149956, + "scores_per_experiment": [ + { + "accuracy": 0.232323, + "f1": 0.165314, + "f1_weighted": 0.191552 + }, + { + "accuracy": 0.212121, + "f1": 0.160694, + "f1_weighted": 0.159505 + }, + { + "accuracy": 0.161616, + "f1": 0.167046, + "f1_weighted": 0.145149 + }, + { + "accuracy": 0.181818, + "f1": 0.169409, + "f1_weighted": 0.153777 + }, + { + "accuracy": 0.141414, + "f1": 0.113375, + "f1_weighted": 0.089295 + }, + { + "accuracy": 0.161616, + "f1": 0.119718, + "f1_weighted": 0.144486 + }, + { + "accuracy": 0.181818, + "f1": 0.134802, + "f1_weighted": 0.150992 + }, + { + "accuracy": 0.212121, + "f1": 0.145388, + "f1_weighted": 0.187273 + }, + { + "accuracy": 0.151515, + "f1": 0.146565, + "f1_weighted": 0.135734 + }, + { + "accuracy": 0.181818, + "f1": 0.144, + "f1_weighted": 0.141795 + } + ], + "main_score": 0.181818, + "hf_subset": "kan_Knda", + "languages": [ + "kan-Knda" + ] + }, + { + "accuracy": 0.225253, + "f1": 0.185278, + "f1_weighted": 0.190665, + "scores_per_experiment": [ + { + "accuracy": 0.20202, + "f1": 0.162468, + "f1_weighted": 0.159421 + }, + { + "accuracy": 0.141414, + "f1": 0.090265, + "f1_weighted": 0.092038 + }, + { + "accuracy": 0.171717, + "f1": 0.161451, + "f1_weighted": 0.127796 + }, + { + "accuracy": 0.282828, + "f1": 0.234782, + "f1_weighted": 0.266045 + }, + { + "accuracy": 0.191919, + "f1": 0.191733, + "f1_weighted": 0.169978 + }, + { + "accuracy": 0.313131, + "f1": 0.255949, + "f1_weighted": 0.282521 + }, + { + "accuracy": 0.272727, + "f1": 0.219537, + "f1_weighted": 0.210319 + }, + { + "accuracy": 0.272727, + "f1": 0.215631, + "f1_weighted": 0.255185 + }, + { + "accuracy": 0.252525, + "f1": 0.188823, + "f1_weighted": 0.203657 + }, + { + "accuracy": 0.151515, + "f1": 0.132142, + "f1_weighted": 0.139686 + } + ], + "main_score": 0.225253, + "hf_subset": "kas_Deva", + "languages": [ + "kas-Deva" + ] + }, + { + "accuracy": 0.211111, + "f1": 0.169852, + "f1_weighted": 0.176821, + "scores_per_experiment": [ + { + "accuracy": 0.252525, + "f1": 0.173709, + "f1_weighted": 0.179544 + }, + { + "accuracy": 0.20202, + "f1": 0.164305, + "f1_weighted": 0.142423 + }, + { + "accuracy": 0.10101, + "f1": 0.079257, + "f1_weighted": 0.069646 + }, + { + "accuracy": 0.272727, + "f1": 0.22181, + "f1_weighted": 0.254787 + }, + { + "accuracy": 0.141414, + "f1": 0.136485, + "f1_weighted": 0.115086 + }, + { + "accuracy": 0.191919, + "f1": 0.163088, + "f1_weighted": 0.168333 + }, + { + "accuracy": 0.222222, + "f1": 0.158531, + "f1_weighted": 0.204071 + }, + { + "accuracy": 0.313131, + "f1": 0.244197, + "f1_weighted": 0.262022 + }, + { + "accuracy": 0.212121, + "f1": 0.189962, + "f1_weighted": 0.208857 + }, + { + "accuracy": 0.20202, + "f1": 0.167175, + "f1_weighted": 0.16344 + } + ], + "main_score": 0.211111, + "hf_subset": "kat_Geor", + "languages": [ + "kat-Geor" + ] + }, + { + "accuracy": 0.210101, + "f1": 0.185919, + "f1_weighted": 0.194874, + "scores_per_experiment": [ + { + "accuracy": 0.212121, + "f1": 0.143827, + "f1_weighted": 0.153042 + }, + { + "accuracy": 0.232323, + "f1": 0.196826, + "f1_weighted": 0.2227 + }, + { + "accuracy": 0.161616, + "f1": 0.150338, + "f1_weighted": 0.120894 + }, + { + "accuracy": 0.282828, + "f1": 0.229385, + "f1_weighted": 0.257644 + }, + { + "accuracy": 0.222222, + "f1": 0.201918, + "f1_weighted": 0.216376 + }, + { + "accuracy": 0.191919, + "f1": 0.191954, + "f1_weighted": 0.191639 + }, + { + "accuracy": 0.232323, + "f1": 0.224374, + "f1_weighted": 0.248997 + }, + { + "accuracy": 0.161616, + "f1": 0.152296, + "f1_weighted": 0.151793 + }, + { + "accuracy": 0.171717, + "f1": 0.166369, + "f1_weighted": 0.154622 + }, + { + "accuracy": 0.232323, + "f1": 0.201901, + "f1_weighted": 0.231031 + } + ], + "main_score": 0.210101, + "hf_subset": "kaz_Cyrl", + "languages": [ + "kaz-Cyrl" + ] + }, + { + "accuracy": 0.287879, + "f1": 0.267419, + "f1_weighted": 0.288238, + "scores_per_experiment": [ + { + "accuracy": 0.323232, + "f1": 0.300901, + "f1_weighted": 0.326217 + }, + { + "accuracy": 0.242424, + "f1": 0.237769, + "f1_weighted": 0.235768 + }, + { + "accuracy": 0.252525, + "f1": 0.242398, + "f1_weighted": 0.238515 + }, + { + "accuracy": 0.353535, + "f1": 0.331471, + "f1_weighted": 0.354958 + }, + { + "accuracy": 0.262626, + "f1": 0.227195, + "f1_weighted": 0.268398 + }, + { + "accuracy": 0.262626, + "f1": 0.251201, + "f1_weighted": 0.25984 + }, + { + "accuracy": 0.272727, + "f1": 0.229645, + "f1_weighted": 0.264817 + }, + { + "accuracy": 0.353535, + "f1": 0.348279, + "f1_weighted": 0.352136 + }, + { + "accuracy": 0.272727, + "f1": 0.246372, + "f1_weighted": 0.275158 + }, + { + "accuracy": 0.282828, + "f1": 0.258962, + "f1_weighted": 0.306571 + } + ], + "main_score": 0.287879, + "hf_subset": "kbp_Latn", + "languages": [ + "kbp-Latn" + ] + }, + { + "accuracy": 0.40202, + "f1": 0.38268, + "f1_weighted": 0.404212, + "scores_per_experiment": [ + { + "accuracy": 0.40404, + "f1": 0.37628, + "f1_weighted": 0.399773 + }, + { + "accuracy": 0.343434, + "f1": 0.340222, + "f1_weighted": 0.346799 + }, + { + "accuracy": 0.363636, + "f1": 0.362559, + "f1_weighted": 0.375703 + }, + { + "accuracy": 0.484848, + "f1": 0.461733, + "f1_weighted": 0.48852 + }, + { + "accuracy": 0.414141, + "f1": 0.388491, + "f1_weighted": 0.4192 + }, + { + "accuracy": 0.393939, + "f1": 0.357765, + "f1_weighted": 0.39502 + }, + { + "accuracy": 0.373737, + "f1": 0.356438, + "f1_weighted": 0.362131 + }, + { + "accuracy": 0.464646, + "f1": 0.418579, + "f1_weighted": 0.474801 + }, + { + "accuracy": 0.363636, + "f1": 0.360531, + "f1_weighted": 0.351769 + }, + { + "accuracy": 0.414141, + "f1": 0.404207, + "f1_weighted": 0.428407 + } + ], + "main_score": 0.40202, + "hf_subset": "kea_Latn", + "languages": [ + "kea-Latn" + ] + }, + { + "accuracy": 0.226263, + "f1": 0.197136, + "f1_weighted": 0.220275, + "scores_per_experiment": [ + { + "accuracy": 0.262626, + "f1": 0.174546, + "f1_weighted": 0.225507 + }, + { + "accuracy": 0.242424, + "f1": 0.229286, + "f1_weighted": 0.259482 + }, + { + "accuracy": 0.141414, + "f1": 0.139966, + "f1_weighted": 0.130495 + }, + { + "accuracy": 0.282828, + "f1": 0.249582, + "f1_weighted": 0.263013 + }, + { + "accuracy": 0.131313, + "f1": 0.12331, + "f1_weighted": 0.127899 + }, + { + "accuracy": 0.272727, + "f1": 0.220086, + "f1_weighted": 0.281921 + }, + { + "accuracy": 0.262626, + "f1": 0.238984, + "f1_weighted": 0.235779 + }, + { + "accuracy": 0.282828, + "f1": 0.270743, + "f1_weighted": 0.301243 + }, + { + "accuracy": 0.171717, + "f1": 0.133446, + "f1_weighted": 0.150797 + }, + { + "accuracy": 0.212121, + "f1": 0.191407, + "f1_weighted": 0.226611 + } + ], + "main_score": 0.226263, + "hf_subset": "khk_Cyrl", + "languages": [ + "khk-Cyrl" + ] + }, + { + "accuracy": 0.229293, + "f1": 0.196638, + "f1_weighted": 0.205509, + "scores_per_experiment": [ + { + "accuracy": 0.20202, + "f1": 0.141766, + "f1_weighted": 0.147584 + }, + { + "accuracy": 0.20202, + "f1": 0.161505, + "f1_weighted": 0.188836 + }, + { + "accuracy": 0.222222, + "f1": 0.200766, + "f1_weighted": 0.198528 + }, + { + "accuracy": 0.212121, + "f1": 0.187347, + "f1_weighted": 0.170206 + }, + { + "accuracy": 0.161616, + "f1": 0.137251, + "f1_weighted": 0.144205 + }, + { + "accuracy": 0.313131, + "f1": 0.24429, + "f1_weighted": 0.2699 + }, + { + "accuracy": 0.252525, + "f1": 0.236576, + "f1_weighted": 0.240593 + }, + { + "accuracy": 0.262626, + "f1": 0.225807, + "f1_weighted": 0.250892 + }, + { + "accuracy": 0.242424, + "f1": 0.210096, + "f1_weighted": 0.229556 + }, + { + "accuracy": 0.222222, + "f1": 0.220974, + "f1_weighted": 0.214786 + } + ], + "main_score": 0.229293, + "hf_subset": "khm_Khmr", + "languages": [ + "khm-Khmr" + ] + }, + { + "accuracy": 0.353535, + "f1": 0.327059, + "f1_weighted": 0.35738, + "scores_per_experiment": [ + { + "accuracy": 0.353535, + "f1": 0.350645, + "f1_weighted": 0.367466 + }, + { + "accuracy": 0.323232, + "f1": 0.283527, + "f1_weighted": 0.320733 + }, + { + "accuracy": 0.333333, + "f1": 0.30283, + "f1_weighted": 0.332158 + }, + { + "accuracy": 0.383838, + "f1": 0.380573, + "f1_weighted": 0.394363 + }, + { + "accuracy": 0.353535, + "f1": 0.32005, + "f1_weighted": 0.355928 + }, + { + "accuracy": 0.313131, + "f1": 0.287965, + "f1_weighted": 0.309425 + }, + { + "accuracy": 0.373737, + "f1": 0.329754, + "f1_weighted": 0.365785 + }, + { + "accuracy": 0.353535, + "f1": 0.315903, + "f1_weighted": 0.351394 + }, + { + "accuracy": 0.373737, + "f1": 0.338366, + "f1_weighted": 0.386654 + }, + { + "accuracy": 0.373737, + "f1": 0.360975, + "f1_weighted": 0.389898 + } + ], + "main_score": 0.353535, + "hf_subset": "kik_Latn", + "languages": [ + "kik-Latn" + ] + }, + { + "accuracy": 0.280808, + "f1": 0.277896, + "f1_weighted": 0.279551, + "scores_per_experiment": [ + { + "accuracy": 0.292929, + "f1": 0.292712, + "f1_weighted": 0.297186 + }, + { + "accuracy": 0.232323, + "f1": 0.237144, + "f1_weighted": 0.218945 + }, + { + "accuracy": 0.232323, + "f1": 0.226352, + "f1_weighted": 0.231349 + }, + { + "accuracy": 0.262626, + "f1": 0.251339, + "f1_weighted": 0.246952 + }, + { + "accuracy": 0.333333, + "f1": 0.323483, + "f1_weighted": 0.345288 + }, + { + "accuracy": 0.181818, + "f1": 0.185051, + "f1_weighted": 0.182045 + }, + { + "accuracy": 0.323232, + "f1": 0.303943, + "f1_weighted": 0.306658 + }, + { + "accuracy": 0.333333, + "f1": 0.346373, + "f1_weighted": 0.325738 + }, + { + "accuracy": 0.292929, + "f1": 0.281301, + "f1_weighted": 0.307434 + }, + { + "accuracy": 0.323232, + "f1": 0.331258, + "f1_weighted": 0.333911 + } + ], + "main_score": 0.280808, + "hf_subset": "kin_Latn", + "languages": [ + "kin-Latn" + ] + }, + { + "accuracy": 0.185859, + "f1": 0.167331, + "f1_weighted": 0.185704, + "scores_per_experiment": [ + { + "accuracy": 0.282828, + "f1": 0.221344, + "f1_weighted": 0.252041 + }, + { + "accuracy": 0.151515, + "f1": 0.155297, + "f1_weighted": 0.150664 + }, + { + "accuracy": 0.161616, + "f1": 0.173379, + "f1_weighted": 0.176912 + }, + { + "accuracy": 0.242424, + "f1": 0.221843, + "f1_weighted": 0.246232 + }, + { + "accuracy": 0.121212, + "f1": 0.09689, + "f1_weighted": 0.115715 + }, + { + "accuracy": 0.171717, + "f1": 0.152544, + "f1_weighted": 0.166897 + }, + { + "accuracy": 0.181818, + "f1": 0.169367, + "f1_weighted": 0.193067 + }, + { + "accuracy": 0.191919, + "f1": 0.160331, + "f1_weighted": 0.19198 + }, + { + "accuracy": 0.20202, + "f1": 0.189171, + "f1_weighted": 0.205497 + }, + { + "accuracy": 0.151515, + "f1": 0.133146, + "f1_weighted": 0.15803 + } + ], + "main_score": 0.185859, + "hf_subset": "kir_Cyrl", + "languages": [ + "kir-Cyrl" + ] + }, + { + "accuracy": 0.267677, + "f1": 0.259136, + "f1_weighted": 0.267841, + "scores_per_experiment": [ + { + "accuracy": 0.262626, + "f1": 0.262698, + "f1_weighted": 0.263816 + }, + { + "accuracy": 0.262626, + "f1": 0.251, + "f1_weighted": 0.250315 + }, + { + "accuracy": 0.222222, + "f1": 0.219253, + "f1_weighted": 0.221247 + }, + { + "accuracy": 0.333333, + "f1": 0.323103, + "f1_weighted": 0.325572 + }, + { + "accuracy": 0.232323, + "f1": 0.212462, + "f1_weighted": 0.221426 + }, + { + "accuracy": 0.323232, + "f1": 0.326785, + "f1_weighted": 0.332656 + }, + { + "accuracy": 0.232323, + "f1": 0.215903, + "f1_weighted": 0.234034 + }, + { + "accuracy": 0.353535, + "f1": 0.331251, + "f1_weighted": 0.357799 + }, + { + "accuracy": 0.232323, + "f1": 0.231516, + "f1_weighted": 0.235954 + }, + { + "accuracy": 0.222222, + "f1": 0.217386, + "f1_weighted": 0.235594 + } + ], + "main_score": 0.267677, + "hf_subset": "kmb_Latn", + "languages": [ + "kmb-Latn" + ] + }, + { + "accuracy": 0.272727, + "f1": 0.244653, + "f1_weighted": 0.274697, + "scores_per_experiment": [ + { + "accuracy": 0.272727, + "f1": 0.250588, + "f1_weighted": 0.276386 + }, + { + "accuracy": 0.232323, + "f1": 0.212728, + "f1_weighted": 0.234339 + }, + { + "accuracy": 0.212121, + "f1": 0.20884, + "f1_weighted": 0.217939 + }, + { + "accuracy": 0.292929, + "f1": 0.266233, + "f1_weighted": 0.290185 + }, + { + "accuracy": 0.30303, + "f1": 0.27078, + "f1_weighted": 0.313586 + }, + { + "accuracy": 0.282828, + "f1": 0.22656, + "f1_weighted": 0.287085 + }, + { + "accuracy": 0.333333, + "f1": 0.306339, + "f1_weighted": 0.341011 + }, + { + "accuracy": 0.343434, + "f1": 0.293859, + "f1_weighted": 0.331985 + }, + { + "accuracy": 0.232323, + "f1": 0.203063, + "f1_weighted": 0.225537 + }, + { + "accuracy": 0.222222, + "f1": 0.207538, + "f1_weighted": 0.228913 + } + ], + "main_score": 0.272727, + "hf_subset": "kmr_Latn", + "languages": [ + "kmr-Latn" + ] + }, + { + "accuracy": 0.40101, + "f1": 0.385708, + "f1_weighted": 0.403886, + "scores_per_experiment": [ + { + "accuracy": 0.414141, + "f1": 0.371851, + "f1_weighted": 0.414359 + }, + { + "accuracy": 0.363636, + "f1": 0.346062, + "f1_weighted": 0.359543 + }, + { + "accuracy": 0.393939, + "f1": 0.381127, + "f1_weighted": 0.404716 + }, + { + "accuracy": 0.464646, + "f1": 0.435659, + "f1_weighted": 0.461563 + }, + { + "accuracy": 0.373737, + "f1": 0.378071, + "f1_weighted": 0.391877 + }, + { + "accuracy": 0.383838, + "f1": 0.373741, + "f1_weighted": 0.382001 + }, + { + "accuracy": 0.424242, + "f1": 0.431603, + "f1_weighted": 0.421211 + }, + { + "accuracy": 0.40404, + "f1": 0.404654, + "f1_weighted": 0.421987 + }, + { + "accuracy": 0.40404, + "f1": 0.370243, + "f1_weighted": 0.399064 + }, + { + "accuracy": 0.383838, + "f1": 0.364069, + "f1_weighted": 0.382537 + } + ], + "main_score": 0.40101, + "hf_subset": "knc_Latn", + "languages": [ + "knc-Latn" + ] + }, + { + "accuracy": 0.380808, + "f1": 0.362552, + "f1_weighted": 0.385491, + "scores_per_experiment": [ + { + "accuracy": 0.434343, + "f1": 0.444751, + "f1_weighted": 0.432345 + }, + { + "accuracy": 0.333333, + "f1": 0.319333, + "f1_weighted": 0.32067 + }, + { + "accuracy": 0.434343, + "f1": 0.435009, + "f1_weighted": 0.451404 + }, + { + "accuracy": 0.373737, + "f1": 0.338595, + "f1_weighted": 0.36033 + }, + { + "accuracy": 0.343434, + "f1": 0.302891, + "f1_weighted": 0.351079 + }, + { + "accuracy": 0.343434, + "f1": 0.329743, + "f1_weighted": 0.361219 + }, + { + "accuracy": 0.393939, + "f1": 0.380361, + "f1_weighted": 0.398298 + }, + { + "accuracy": 0.434343, + "f1": 0.430523, + "f1_weighted": 0.452255 + }, + { + "accuracy": 0.363636, + "f1": 0.340458, + "f1_weighted": 0.370868 + }, + { + "accuracy": 0.353535, + "f1": 0.303854, + "f1_weighted": 0.356441 + } + ], + "main_score": 0.380808, + "hf_subset": "kon_Latn", + "languages": [ + "kon-Latn" + ] + }, + { + "accuracy": 0.234343, + "f1": 0.186921, + "f1_weighted": 0.200824, + "scores_per_experiment": [ + { + "accuracy": 0.252525, + "f1": 0.206909, + "f1_weighted": 0.208633 + }, + { + "accuracy": 0.191919, + "f1": 0.101485, + "f1_weighted": 0.121965 + }, + { + "accuracy": 0.151515, + "f1": 0.147708, + "f1_weighted": 0.145688 + }, + { + "accuracy": 0.313131, + "f1": 0.272941, + "f1_weighted": 0.286099 + }, + { + "accuracy": 0.161616, + "f1": 0.144875, + "f1_weighted": 0.140584 + }, + { + "accuracy": 0.272727, + "f1": 0.172735, + "f1_weighted": 0.210352 + }, + { + "accuracy": 0.222222, + "f1": 0.200442, + "f1_weighted": 0.203208 + }, + { + "accuracy": 0.30303, + "f1": 0.25001, + "f1_weighted": 0.270674 + }, + { + "accuracy": 0.272727, + "f1": 0.180904, + "f1_weighted": 0.220584 + }, + { + "accuracy": 0.20202, + "f1": 0.191196, + "f1_weighted": 0.200453 + } + ], + "main_score": 0.234343, + "hf_subset": "kor_Hang", + "languages": [ + "kor-Hang" + ] + }, + { + "accuracy": 0.256566, + "f1": 0.219265, + "f1_weighted": 0.233234, + "scores_per_experiment": [ + { + "accuracy": 0.262626, + "f1": 0.230887, + "f1_weighted": 0.24502 + }, + { + "accuracy": 0.252525, + "f1": 0.202071, + "f1_weighted": 0.229806 + }, + { + "accuracy": 0.242424, + "f1": 0.186971, + "f1_weighted": 0.200089 + }, + { + "accuracy": 0.191919, + "f1": 0.178598, + "f1_weighted": 0.156635 + }, + { + "accuracy": 0.262626, + "f1": 0.213156, + "f1_weighted": 0.248507 + }, + { + "accuracy": 0.272727, + "f1": 0.204267, + "f1_weighted": 0.244471 + }, + { + "accuracy": 0.262626, + "f1": 0.216304, + "f1_weighted": 0.208288 + }, + { + "accuracy": 0.262626, + "f1": 0.25538, + "f1_weighted": 0.259702 + }, + { + "accuracy": 0.272727, + "f1": 0.284401, + "f1_weighted": 0.270754 + }, + { + "accuracy": 0.282828, + "f1": 0.220616, + "f1_weighted": 0.269065 + } + ], + "main_score": 0.256566, + "hf_subset": "lao_Laoo", + "languages": [ + "lao-Laoo" + ] + }, + { + "accuracy": 0.382828, + "f1": 0.353878, + "f1_weighted": 0.385487, + "scores_per_experiment": [ + { + "accuracy": 0.393939, + "f1": 0.376888, + "f1_weighted": 0.391711 + }, + { + "accuracy": 0.434343, + "f1": 0.373998, + "f1_weighted": 0.426468 + }, + { + "accuracy": 0.30303, + "f1": 0.276077, + "f1_weighted": 0.292505 + }, + { + "accuracy": 0.505051, + "f1": 0.470117, + "f1_weighted": 0.507934 + }, + { + "accuracy": 0.313131, + "f1": 0.267053, + "f1_weighted": 0.332983 + }, + { + "accuracy": 0.444444, + "f1": 0.432669, + "f1_weighted": 0.455211 + }, + { + "accuracy": 0.333333, + "f1": 0.314278, + "f1_weighted": 0.317941 + }, + { + "accuracy": 0.373737, + "f1": 0.348291, + "f1_weighted": 0.373714 + }, + { + "accuracy": 0.353535, + "f1": 0.330077, + "f1_weighted": 0.373609 + }, + { + "accuracy": 0.373737, + "f1": 0.349334, + "f1_weighted": 0.382792 + } + ], + "main_score": 0.382828, + "hf_subset": "lij_Latn", + "languages": [ + "lij-Latn" + ] + }, + { + "accuracy": 0.431313, + "f1": 0.422649, + "f1_weighted": 0.438189, + "scores_per_experiment": [ + { + "accuracy": 0.414141, + "f1": 0.393598, + "f1_weighted": 0.416621 + }, + { + "accuracy": 0.424242, + "f1": 0.417176, + "f1_weighted": 0.426786 + }, + { + "accuracy": 0.414141, + "f1": 0.406892, + "f1_weighted": 0.410413 + }, + { + "accuracy": 0.474747, + "f1": 0.482927, + "f1_weighted": 0.49546 + }, + { + "accuracy": 0.454545, + "f1": 0.440132, + "f1_weighted": 0.46198 + }, + { + "accuracy": 0.444444, + "f1": 0.41844, + "f1_weighted": 0.467113 + }, + { + "accuracy": 0.444444, + "f1": 0.436659, + "f1_weighted": 0.460452 + }, + { + "accuracy": 0.414141, + "f1": 0.414704, + "f1_weighted": 0.414016 + }, + { + "accuracy": 0.393939, + "f1": 0.386981, + "f1_weighted": 0.39381 + }, + { + "accuracy": 0.434343, + "f1": 0.428978, + "f1_weighted": 0.435238 + } + ], + "main_score": 0.431313, + "hf_subset": "lim_Latn", + "languages": [ + "lim-Latn" + ] + }, + { + "accuracy": 0.371717, + "f1": 0.36018, + "f1_weighted": 0.37868, + "scores_per_experiment": [ + { + "accuracy": 0.353535, + "f1": 0.367726, + "f1_weighted": 0.368766 + }, + { + "accuracy": 0.313131, + "f1": 0.325731, + "f1_weighted": 0.323345 + }, + { + "accuracy": 0.373737, + "f1": 0.35807, + "f1_weighted": 0.383011 + }, + { + "accuracy": 0.343434, + "f1": 0.349963, + "f1_weighted": 0.369398 + }, + { + "accuracy": 0.424242, + "f1": 0.409776, + "f1_weighted": 0.437361 + }, + { + "accuracy": 0.353535, + "f1": 0.343743, + "f1_weighted": 0.343857 + }, + { + "accuracy": 0.343434, + "f1": 0.315727, + "f1_weighted": 0.340861 + }, + { + "accuracy": 0.464646, + "f1": 0.419868, + "f1_weighted": 0.474599 + }, + { + "accuracy": 0.30303, + "f1": 0.299794, + "f1_weighted": 0.286911 + }, + { + "accuracy": 0.444444, + "f1": 0.411405, + "f1_weighted": 0.458687 + } + ], + "main_score": 0.371717, + "hf_subset": "lin_Latn", + "languages": [ + "lin-Latn" + ] + }, + { + "accuracy": 0.30101, + "f1": 0.289088, + "f1_weighted": 0.298869, + "scores_per_experiment": [ + { + "accuracy": 0.292929, + "f1": 0.291265, + "f1_weighted": 0.294075 + }, + { + "accuracy": 0.282828, + "f1": 0.269202, + "f1_weighted": 0.277954 + }, + { + "accuracy": 0.333333, + "f1": 0.321644, + "f1_weighted": 0.316962 + }, + { + "accuracy": 0.363636, + "f1": 0.361787, + "f1_weighted": 0.361498 + }, + { + "accuracy": 0.30303, + "f1": 0.278939, + "f1_weighted": 0.295679 + }, + { + "accuracy": 0.262626, + "f1": 0.248761, + "f1_weighted": 0.254047 + }, + { + "accuracy": 0.313131, + "f1": 0.299793, + "f1_weighted": 0.307876 + }, + { + "accuracy": 0.313131, + "f1": 0.289096, + "f1_weighted": 0.316362 + }, + { + "accuracy": 0.252525, + "f1": 0.234705, + "f1_weighted": 0.270648 + }, + { + "accuracy": 0.292929, + "f1": 0.295684, + "f1_weighted": 0.293591 + } + ], + "main_score": 0.30101, + "hf_subset": "lit_Latn", + "languages": [ + "lit-Latn" + ] + }, + { + "accuracy": 0.388889, + "f1": 0.363877, + "f1_weighted": 0.391522, + "scores_per_experiment": [ + { + "accuracy": 0.373737, + "f1": 0.347726, + "f1_weighted": 0.386055 + }, + { + "accuracy": 0.444444, + "f1": 0.424001, + "f1_weighted": 0.449897 + }, + { + "accuracy": 0.313131, + "f1": 0.298775, + "f1_weighted": 0.29867 + }, + { + "accuracy": 0.414141, + "f1": 0.375436, + "f1_weighted": 0.412971 + }, + { + "accuracy": 0.414141, + "f1": 0.380452, + "f1_weighted": 0.413873 + }, + { + "accuracy": 0.383838, + "f1": 0.360457, + "f1_weighted": 0.383673 + }, + { + "accuracy": 0.353535, + "f1": 0.335763, + "f1_weighted": 0.352167 + }, + { + "accuracy": 0.424242, + "f1": 0.396826, + "f1_weighted": 0.432849 + }, + { + "accuracy": 0.424242, + "f1": 0.381115, + "f1_weighted": 0.433619 + }, + { + "accuracy": 0.343434, + "f1": 0.338222, + "f1_weighted": 0.351444 + } + ], + "main_score": 0.388889, + "hf_subset": "lmo_Latn", + "languages": [ + "lmo-Latn" + ] + }, + { + "accuracy": 0.342424, + "f1": 0.326893, + "f1_weighted": 0.337598, + "scores_per_experiment": [ + { + "accuracy": 0.343434, + "f1": 0.311108, + "f1_weighted": 0.347449 + }, + { + "accuracy": 0.373737, + "f1": 0.348186, + "f1_weighted": 0.373447 + }, + { + "accuracy": 0.272727, + "f1": 0.267719, + "f1_weighted": 0.253414 + }, + { + "accuracy": 0.292929, + "f1": 0.275573, + "f1_weighted": 0.294663 + }, + { + "accuracy": 0.363636, + "f1": 0.365557, + "f1_weighted": 0.353877 + }, + { + "accuracy": 0.333333, + "f1": 0.299885, + "f1_weighted": 0.317407 + }, + { + "accuracy": 0.444444, + "f1": 0.422399, + "f1_weighted": 0.444484 + }, + { + "accuracy": 0.343434, + "f1": 0.330226, + "f1_weighted": 0.343635 + }, + { + "accuracy": 0.313131, + "f1": 0.320348, + "f1_weighted": 0.309459 + }, + { + "accuracy": 0.343434, + "f1": 0.32793, + "f1_weighted": 0.338149 + } + ], + "main_score": 0.342424, + "hf_subset": "ltg_Latn", + "languages": [ + "ltg-Latn" + ] + }, + { + "accuracy": 0.434343, + "f1": 0.421796, + "f1_weighted": 0.439148, + "scores_per_experiment": [ + { + "accuracy": 0.424242, + "f1": 0.425401, + "f1_weighted": 0.427455 + }, + { + "accuracy": 0.484848, + "f1": 0.483993, + "f1_weighted": 0.484353 + }, + { + "accuracy": 0.343434, + "f1": 0.348778, + "f1_weighted": 0.345767 + }, + { + "accuracy": 0.464646, + "f1": 0.442957, + "f1_weighted": 0.469613 + }, + { + "accuracy": 0.444444, + "f1": 0.424677, + "f1_weighted": 0.449979 + }, + { + "accuracy": 0.505051, + "f1": 0.477322, + "f1_weighted": 0.514256 + }, + { + "accuracy": 0.373737, + "f1": 0.367849, + "f1_weighted": 0.397699 + }, + { + "accuracy": 0.464646, + "f1": 0.44933, + "f1_weighted": 0.473208 + }, + { + "accuracy": 0.383838, + "f1": 0.353001, + "f1_weighted": 0.374544 + }, + { + "accuracy": 0.454545, + "f1": 0.444652, + "f1_weighted": 0.454602 + } + ], + "main_score": 0.434343, + "hf_subset": "ltz_Latn", + "languages": [ + "ltz-Latn" + ] + }, + { + "accuracy": 0.326263, + "f1": 0.310513, + "f1_weighted": 0.327149, + "scores_per_experiment": [ + { + "accuracy": 0.323232, + "f1": 0.308752, + "f1_weighted": 0.338909 + }, + { + "accuracy": 0.323232, + "f1": 0.325839, + "f1_weighted": 0.328437 + }, + { + "accuracy": 0.242424, + "f1": 0.211209, + "f1_weighted": 0.221614 + }, + { + "accuracy": 0.323232, + "f1": 0.31069, + "f1_weighted": 0.341364 + }, + { + "accuracy": 0.313131, + "f1": 0.297853, + "f1_weighted": 0.317958 + }, + { + "accuracy": 0.333333, + "f1": 0.324225, + "f1_weighted": 0.326677 + }, + { + "accuracy": 0.292929, + "f1": 0.265851, + "f1_weighted": 0.293132 + }, + { + "accuracy": 0.393939, + "f1": 0.3683, + "f1_weighted": 0.392057 + }, + { + "accuracy": 0.373737, + "f1": 0.358062, + "f1_weighted": 0.36213 + }, + { + "accuracy": 0.343434, + "f1": 0.334349, + "f1_weighted": 0.349212 + } + ], + "main_score": 0.326263, + "hf_subset": "lua_Latn", + "languages": [ + "lua-Latn" + ] + }, + { + "accuracy": 0.29596, + "f1": 0.29163, + "f1_weighted": 0.29489, + "scores_per_experiment": [ + { + "accuracy": 0.292929, + "f1": 0.298038, + "f1_weighted": 0.286793 + }, + { + "accuracy": 0.222222, + "f1": 0.217555, + "f1_weighted": 0.215062 + }, + { + "accuracy": 0.20202, + "f1": 0.197674, + "f1_weighted": 0.188298 + }, + { + "accuracy": 0.343434, + "f1": 0.33894, + "f1_weighted": 0.351239 + }, + { + "accuracy": 0.343434, + "f1": 0.336411, + "f1_weighted": 0.367769 + }, + { + "accuracy": 0.292929, + "f1": 0.299541, + "f1_weighted": 0.299561 + }, + { + "accuracy": 0.313131, + "f1": 0.300462, + "f1_weighted": 0.299701 + }, + { + "accuracy": 0.363636, + "f1": 0.352977, + "f1_weighted": 0.35927 + }, + { + "accuracy": 0.272727, + "f1": 0.268311, + "f1_weighted": 0.263543 + }, + { + "accuracy": 0.313131, + "f1": 0.306388, + "f1_weighted": 0.317661 + } + ], + "main_score": 0.29596, + "hf_subset": "lug_Latn", + "languages": [ + "lug-Latn" + ] + }, + { + "accuracy": 0.29798, + "f1": 0.297586, + "f1_weighted": 0.302546, + "scores_per_experiment": [ + { + "accuracy": 0.30303, + "f1": 0.315023, + "f1_weighted": 0.311175 + }, + { + "accuracy": 0.272727, + "f1": 0.273408, + "f1_weighted": 0.264283 + }, + { + "accuracy": 0.292929, + "f1": 0.286306, + "f1_weighted": 0.299958 + }, + { + "accuracy": 0.323232, + "f1": 0.320667, + "f1_weighted": 0.341508 + }, + { + "accuracy": 0.232323, + "f1": 0.229186, + "f1_weighted": 0.232156 + }, + { + "accuracy": 0.323232, + "f1": 0.340984, + "f1_weighted": 0.324652 + }, + { + "accuracy": 0.282828, + "f1": 0.272131, + "f1_weighted": 0.286366 + }, + { + "accuracy": 0.363636, + "f1": 0.349382, + "f1_weighted": 0.383748 + }, + { + "accuracy": 0.252525, + "f1": 0.248746, + "f1_weighted": 0.237601 + }, + { + "accuracy": 0.333333, + "f1": 0.340026, + "f1_weighted": 0.344012 + } + ], + "main_score": 0.29798, + "hf_subset": "luo_Latn", + "languages": [ + "luo-Latn" + ] + }, + { + "accuracy": 0.430303, + "f1": 0.404602, + "f1_weighted": 0.440473, + "scores_per_experiment": [ + { + "accuracy": 0.494949, + "f1": 0.475785, + "f1_weighted": 0.511098 + }, + { + "accuracy": 0.333333, + "f1": 0.304533, + "f1_weighted": 0.340244 + }, + { + "accuracy": 0.333333, + "f1": 0.327521, + "f1_weighted": 0.340746 + }, + { + "accuracy": 0.505051, + "f1": 0.461787, + "f1_weighted": 0.505276 + }, + { + "accuracy": 0.474747, + "f1": 0.450233, + "f1_weighted": 0.490615 + }, + { + "accuracy": 0.414141, + "f1": 0.381179, + "f1_weighted": 0.42489 + }, + { + "accuracy": 0.515152, + "f1": 0.467248, + "f1_weighted": 0.517868 + }, + { + "accuracy": 0.40404, + "f1": 0.391048, + "f1_weighted": 0.430102 + }, + { + "accuracy": 0.363636, + "f1": 0.341847, + "f1_weighted": 0.366957 + }, + { + "accuracy": 0.464646, + "f1": 0.444837, + "f1_weighted": 0.47693 + } + ], + "main_score": 0.430303, + "hf_subset": "lus_Latn", + "languages": [ + "lus-Latn" + ] + }, + { + "accuracy": 0.29798, + "f1": 0.28683, + "f1_weighted": 0.302375, + "scores_per_experiment": [ + { + "accuracy": 0.292929, + "f1": 0.276321, + "f1_weighted": 0.294387 + }, + { + "accuracy": 0.323232, + "f1": 0.291811, + "f1_weighted": 0.320031 + }, + { + "accuracy": 0.282828, + "f1": 0.28625, + "f1_weighted": 0.277024 + }, + { + "accuracy": 0.292929, + "f1": 0.286197, + "f1_weighted": 0.301384 + }, + { + "accuracy": 0.313131, + "f1": 0.307622, + "f1_weighted": 0.315947 + }, + { + "accuracy": 0.282828, + "f1": 0.273139, + "f1_weighted": 0.296539 + }, + { + "accuracy": 0.282828, + "f1": 0.273507, + "f1_weighted": 0.288151 + }, + { + "accuracy": 0.373737, + "f1": 0.372403, + "f1_weighted": 0.377818 + }, + { + "accuracy": 0.252525, + "f1": 0.243689, + "f1_weighted": 0.270512 + }, + { + "accuracy": 0.282828, + "f1": 0.257365, + "f1_weighted": 0.281959 + } + ], + "main_score": 0.29798, + "hf_subset": "lvs_Latn", + "languages": [ + "lvs-Latn" + ] + }, + { + "accuracy": 0.186869, + "f1": 0.152402, + "f1_weighted": 0.154755, + "scores_per_experiment": [ + { + "accuracy": 0.181818, + "f1": 0.157444, + "f1_weighted": 0.154794 + }, + { + "accuracy": 0.141414, + "f1": 0.098718, + "f1_weighted": 0.10545 + }, + { + "accuracy": 0.131313, + "f1": 0.1287, + "f1_weighted": 0.10782 + }, + { + "accuracy": 0.181818, + "f1": 0.183374, + "f1_weighted": 0.158746 + }, + { + "accuracy": 0.181818, + "f1": 0.183987, + "f1_weighted": 0.166433 + }, + { + "accuracy": 0.242424, + "f1": 0.153807, + "f1_weighted": 0.197204 + }, + { + "accuracy": 0.20202, + "f1": 0.144619, + "f1_weighted": 0.149412 + }, + { + "accuracy": 0.232323, + "f1": 0.183418, + "f1_weighted": 0.192837 + }, + { + "accuracy": 0.171717, + "f1": 0.138373, + "f1_weighted": 0.14893 + }, + { + "accuracy": 0.20202, + "f1": 0.151585, + "f1_weighted": 0.165922 + } + ], + "main_score": 0.186869, + "hf_subset": "mag_Deva", + "languages": [ + "mag-Deva" + ] + }, + { + "accuracy": 0.20303, + "f1": 0.160314, + "f1_weighted": 0.171597, + "scores_per_experiment": [ + { + "accuracy": 0.222222, + "f1": 0.177841, + "f1_weighted": 0.189776 + }, + { + "accuracy": 0.191919, + "f1": 0.11693, + "f1_weighted": 0.133427 + }, + { + "accuracy": 0.20202, + "f1": 0.14867, + "f1_weighted": 0.175688 + }, + { + "accuracy": 0.20202, + "f1": 0.182963, + "f1_weighted": 0.163767 + }, + { + "accuracy": 0.111111, + "f1": 0.100306, + "f1_weighted": 0.095495 + }, + { + "accuracy": 0.181818, + "f1": 0.175917, + "f1_weighted": 0.169974 + }, + { + "accuracy": 0.232323, + "f1": 0.180848, + "f1_weighted": 0.17942 + }, + { + "accuracy": 0.292929, + "f1": 0.21348, + "f1_weighted": 0.258696 + }, + { + "accuracy": 0.222222, + "f1": 0.167145, + "f1_weighted": 0.197443 + }, + { + "accuracy": 0.171717, + "f1": 0.139044, + "f1_weighted": 0.152287 + } + ], + "main_score": 0.20303, + "hf_subset": "mai_Deva", + "languages": [ + "mai-Deva" + ] + }, + { + "accuracy": 0.20202, + "f1": 0.154687, + "f1_weighted": 0.165423, + "scores_per_experiment": [ + { + "accuracy": 0.222222, + "f1": 0.157075, + "f1_weighted": 0.165849 + }, + { + "accuracy": 0.121212, + "f1": 0.067665, + "f1_weighted": 0.059406 + }, + { + "accuracy": 0.090909, + "f1": 0.090292, + "f1_weighted": 0.068558 + }, + { + "accuracy": 0.232323, + "f1": 0.222983, + "f1_weighted": 0.198063 + }, + { + "accuracy": 0.121212, + "f1": 0.12111, + "f1_weighted": 0.100828 + }, + { + "accuracy": 0.30303, + "f1": 0.214403, + "f1_weighted": 0.262038 + }, + { + "accuracy": 0.242424, + "f1": 0.182303, + "f1_weighted": 0.216457 + }, + { + "accuracy": 0.272727, + "f1": 0.169838, + "f1_weighted": 0.227273 + }, + { + "accuracy": 0.222222, + "f1": 0.166136, + "f1_weighted": 0.180491 + }, + { + "accuracy": 0.191919, + "f1": 0.155068, + "f1_weighted": 0.175271 + } + ], + "main_score": 0.20202, + "hf_subset": "mal_Mlym", + "languages": [ + "mal-Mlym" + ] + }, + { + "accuracy": 0.151515, + "f1": 0.124131, + "f1_weighted": 0.127443, + "scores_per_experiment": [ + { + "accuracy": 0.121212, + "f1": 0.094218, + "f1_weighted": 0.093987 + }, + { + "accuracy": 0.121212, + "f1": 0.094994, + "f1_weighted": 0.115687 + }, + { + "accuracy": 0.121212, + "f1": 0.119127, + "f1_weighted": 0.101484 + }, + { + "accuracy": 0.181818, + "f1": 0.160343, + "f1_weighted": 0.163715 + }, + { + "accuracy": 0.131313, + "f1": 0.112214, + "f1_weighted": 0.082385 + }, + { + "accuracy": 0.212121, + "f1": 0.123835, + "f1_weighted": 0.166824 + }, + { + "accuracy": 0.161616, + "f1": 0.13699, + "f1_weighted": 0.139649 + }, + { + "accuracy": 0.141414, + "f1": 0.127578, + "f1_weighted": 0.125025 + }, + { + "accuracy": 0.171717, + "f1": 0.148659, + "f1_weighted": 0.165945 + }, + { + "accuracy": 0.151515, + "f1": 0.123354, + "f1_weighted": 0.119727 + } + ], + "main_score": 0.151515, + "hf_subset": "mar_Deva", + "languages": [ + "mar-Deva" + ] + }, + { + "accuracy": 0.373737, + "f1": 0.363384, + "f1_weighted": 0.374444, + "scores_per_experiment": [ + { + "accuracy": 0.393939, + "f1": 0.414801, + "f1_weighted": 0.396013 + }, + { + "accuracy": 0.414141, + "f1": 0.380777, + "f1_weighted": 0.422989 + }, + { + "accuracy": 0.353535, + "f1": 0.340345, + "f1_weighted": 0.353751 + }, + { + "accuracy": 0.343434, + "f1": 0.343719, + "f1_weighted": 0.341156 + }, + { + "accuracy": 0.282828, + "f1": 0.263619, + "f1_weighted": 0.276266 + }, + { + "accuracy": 0.393939, + "f1": 0.383947, + "f1_weighted": 0.390501 + }, + { + "accuracy": 0.353535, + "f1": 0.344101, + "f1_weighted": 0.346103 + }, + { + "accuracy": 0.40404, + "f1": 0.402896, + "f1_weighted": 0.412267 + }, + { + "accuracy": 0.444444, + "f1": 0.40417, + "f1_weighted": 0.43875 + }, + { + "accuracy": 0.353535, + "f1": 0.355464, + "f1_weighted": 0.366641 + } + ], + "main_score": 0.373737, + "hf_subset": "min_Latn", + "languages": [ + "min-Latn" + ] + }, + { + "accuracy": 0.252525, + "f1": 0.233712, + "f1_weighted": 0.243488, + "scores_per_experiment": [ + { + "accuracy": 0.343434, + "f1": 0.301818, + "f1_weighted": 0.311692 + }, + { + "accuracy": 0.222222, + "f1": 0.185702, + "f1_weighted": 0.203251 + }, + { + "accuracy": 0.252525, + "f1": 0.252558, + "f1_weighted": 0.256507 + }, + { + "accuracy": 0.272727, + "f1": 0.259408, + "f1_weighted": 0.281737 + }, + { + "accuracy": 0.171717, + "f1": 0.143865, + "f1_weighted": 0.161787 + }, + { + "accuracy": 0.20202, + "f1": 0.18901, + "f1_weighted": 0.208232 + }, + { + "accuracy": 0.262626, + "f1": 0.262136, + "f1_weighted": 0.256134 + }, + { + "accuracy": 0.272727, + "f1": 0.26196, + "f1_weighted": 0.264863 + }, + { + "accuracy": 0.272727, + "f1": 0.243222, + "f1_weighted": 0.253425 + }, + { + "accuracy": 0.252525, + "f1": 0.237441, + "f1_weighted": 0.237254 + } + ], + "main_score": 0.252525, + "hf_subset": "mkd_Cyrl", + "languages": [ + "mkd-Cyrl" + ] + }, + { + "accuracy": 0.3, + "f1": 0.290349, + "f1_weighted": 0.306166, + "scores_per_experiment": [ + { + "accuracy": 0.333333, + "f1": 0.32975, + "f1_weighted": 0.356894 + }, + { + "accuracy": 0.262626, + "f1": 0.258241, + "f1_weighted": 0.264067 + }, + { + "accuracy": 0.252525, + "f1": 0.241707, + "f1_weighted": 0.231516 + }, + { + "accuracy": 0.373737, + "f1": 0.335289, + "f1_weighted": 0.36475 + }, + { + "accuracy": 0.30303, + "f1": 0.290227, + "f1_weighted": 0.310662 + }, + { + "accuracy": 0.323232, + "f1": 0.323867, + "f1_weighted": 0.324691 + }, + { + "accuracy": 0.292929, + "f1": 0.283703, + "f1_weighted": 0.293531 + }, + { + "accuracy": 0.292929, + "f1": 0.28302, + "f1_weighted": 0.307349 + }, + { + "accuracy": 0.292929, + "f1": 0.293674, + "f1_weighted": 0.323428 + }, + { + "accuracy": 0.272727, + "f1": 0.264012, + "f1_weighted": 0.284776 + } + ], + "main_score": 0.3, + "hf_subset": "mlt_Latn", + "languages": [ + "mlt-Latn" + ] + }, + { + "accuracy": 0.219192, + "f1": 0.185227, + "f1_weighted": 0.191666, + "scores_per_experiment": [ + { + "accuracy": 0.252525, + "f1": 0.212147, + "f1_weighted": 0.202289 + }, + { + "accuracy": 0.262626, + "f1": 0.200153, + "f1_weighted": 0.218468 + }, + { + "accuracy": 0.141414, + "f1": 0.136146, + "f1_weighted": 0.137086 + }, + { + "accuracy": 0.232323, + "f1": 0.222901, + "f1_weighted": 0.233255 + }, + { + "accuracy": 0.212121, + "f1": 0.189066, + "f1_weighted": 0.186408 + }, + { + "accuracy": 0.20202, + "f1": 0.176663, + "f1_weighted": 0.192135 + }, + { + "accuracy": 0.191919, + "f1": 0.150212, + "f1_weighted": 0.143812 + }, + { + "accuracy": 0.323232, + "f1": 0.234099, + "f1_weighted": 0.279573 + }, + { + "accuracy": 0.191919, + "f1": 0.17757, + "f1_weighted": 0.161858 + }, + { + "accuracy": 0.181818, + "f1": 0.153308, + "f1_weighted": 0.161778 + } + ], + "main_score": 0.219192, + "hf_subset": "mni_Beng", + "languages": [ + "mni-Beng" + ] + }, + { + "accuracy": 0.318182, + "f1": 0.3038, + "f1_weighted": 0.324048, + "scores_per_experiment": [ + { + "accuracy": 0.343434, + "f1": 0.327036, + "f1_weighted": 0.363053 + }, + { + "accuracy": 0.30303, + "f1": 0.298773, + "f1_weighted": 0.305351 + }, + { + "accuracy": 0.262626, + "f1": 0.252805, + "f1_weighted": 0.264889 + }, + { + "accuracy": 0.373737, + "f1": 0.353093, + "f1_weighted": 0.369896 + }, + { + "accuracy": 0.252525, + "f1": 0.249862, + "f1_weighted": 0.263893 + }, + { + "accuracy": 0.373737, + "f1": 0.342495, + "f1_weighted": 0.365517 + }, + { + "accuracy": 0.292929, + "f1": 0.284129, + "f1_weighted": 0.308094 + }, + { + "accuracy": 0.353535, + "f1": 0.336338, + "f1_weighted": 0.356221 + }, + { + "accuracy": 0.30303, + "f1": 0.281983, + "f1_weighted": 0.312991 + }, + { + "accuracy": 0.323232, + "f1": 0.311483, + "f1_weighted": 0.330577 + } + ], + "main_score": 0.318182, + "hf_subset": "mos_Latn", + "languages": [ + "mos-Latn" + ] + }, + { + "accuracy": 0.356566, + "f1": 0.336828, + "f1_weighted": 0.361223, + "scores_per_experiment": [ + { + "accuracy": 0.40404, + "f1": 0.396284, + "f1_weighted": 0.410836 + }, + { + "accuracy": 0.373737, + "f1": 0.365048, + "f1_weighted": 0.378808 + }, + { + "accuracy": 0.333333, + "f1": 0.309455, + "f1_weighted": 0.324556 + }, + { + "accuracy": 0.333333, + "f1": 0.31224, + "f1_weighted": 0.332096 + }, + { + "accuracy": 0.353535, + "f1": 0.310761, + "f1_weighted": 0.352455 + }, + { + "accuracy": 0.333333, + "f1": 0.303409, + "f1_weighted": 0.337101 + }, + { + "accuracy": 0.333333, + "f1": 0.299805, + "f1_weighted": 0.337342 + }, + { + "accuracy": 0.393939, + "f1": 0.375891, + "f1_weighted": 0.40001 + }, + { + "accuracy": 0.343434, + "f1": 0.317911, + "f1_weighted": 0.357617 + }, + { + "accuracy": 0.363636, + "f1": 0.37748, + "f1_weighted": 0.381406 + } + ], + "main_score": 0.356566, + "hf_subset": "mri_Latn", + "languages": [ + "mri-Latn" + ] + }, + { + "accuracy": 0.159596, + "f1": 0.102846, + "f1_weighted": 0.119096, + "scores_per_experiment": [ + { + "accuracy": 0.191919, + "f1": 0.110244, + "f1_weighted": 0.130233 + }, + { + "accuracy": 0.131313, + "f1": 0.081367, + "f1_weighted": 0.078174 + }, + { + "accuracy": 0.121212, + "f1": 0.076597, + "f1_weighted": 0.079806 + }, + { + "accuracy": 0.161616, + "f1": 0.107906, + "f1_weighted": 0.10881 + }, + { + "accuracy": 0.151515, + "f1": 0.088652, + "f1_weighted": 0.113162 + }, + { + "accuracy": 0.131313, + "f1": 0.094315, + "f1_weighted": 0.125697 + }, + { + "accuracy": 0.181818, + "f1": 0.159697, + "f1_weighted": 0.165538 + }, + { + "accuracy": 0.191919, + "f1": 0.055394, + "f1_weighted": 0.097918 + }, + { + "accuracy": 0.141414, + "f1": 0.101661, + "f1_weighted": 0.12638 + }, + { + "accuracy": 0.191919, + "f1": 0.152626, + "f1_weighted": 0.165241 + } + ], + "main_score": 0.159596, + "hf_subset": "mya_Mymr", + "languages": [ + "mya-Mymr" + ] + }, + { + "accuracy": 0.459596, + "f1": 0.440181, + "f1_weighted": 0.469548, + "scores_per_experiment": [ + { + "accuracy": 0.484848, + "f1": 0.464765, + "f1_weighted": 0.494815 + }, + { + "accuracy": 0.464646, + "f1": 0.452901, + "f1_weighted": 0.480353 + }, + { + "accuracy": 0.383838, + "f1": 0.385612, + "f1_weighted": 0.388911 + }, + { + "accuracy": 0.545455, + "f1": 0.504678, + "f1_weighted": 0.541635 + }, + { + "accuracy": 0.474747, + "f1": 0.452611, + "f1_weighted": 0.478222 + }, + { + "accuracy": 0.494949, + "f1": 0.487318, + "f1_weighted": 0.51765 + }, + { + "accuracy": 0.40404, + "f1": 0.384531, + "f1_weighted": 0.419087 + }, + { + "accuracy": 0.454545, + "f1": 0.453692, + "f1_weighted": 0.478117 + }, + { + "accuracy": 0.40404, + "f1": 0.369642, + "f1_weighted": 0.406627 + }, + { + "accuracy": 0.484848, + "f1": 0.446056, + "f1_weighted": 0.490065 + } + ], + "main_score": 0.459596, + "hf_subset": "nld_Latn", + "languages": [ + "nld-Latn" + ] + }, + { + "accuracy": 0.415152, + "f1": 0.391178, + "f1_weighted": 0.421149, + "scores_per_experiment": [ + { + "accuracy": 0.454545, + "f1": 0.442966, + "f1_weighted": 0.467119 + }, + { + "accuracy": 0.393939, + "f1": 0.358682, + "f1_weighted": 0.414102 + }, + { + "accuracy": 0.373737, + "f1": 0.367757, + "f1_weighted": 0.359213 + }, + { + "accuracy": 0.424242, + "f1": 0.388841, + "f1_weighted": 0.425902 + }, + { + "accuracy": 0.363636, + "f1": 0.341573, + "f1_weighted": 0.378845 + }, + { + "accuracy": 0.424242, + "f1": 0.399596, + "f1_weighted": 0.428903 + }, + { + "accuracy": 0.434343, + "f1": 0.401029, + "f1_weighted": 0.443721 + }, + { + "accuracy": 0.515152, + "f1": 0.466776, + "f1_weighted": 0.520281 + }, + { + "accuracy": 0.383838, + "f1": 0.372915, + "f1_weighted": 0.387481 + }, + { + "accuracy": 0.383838, + "f1": 0.371643, + "f1_weighted": 0.385923 + } + ], + "main_score": 0.415152, + "hf_subset": "nno_Latn", + "languages": [ + "nno-Latn" + ] + }, + { + "accuracy": 0.459596, + "f1": 0.444673, + "f1_weighted": 0.463646, + "scores_per_experiment": [ + { + "accuracy": 0.505051, + "f1": 0.493355, + "f1_weighted": 0.518415 + }, + { + "accuracy": 0.40404, + "f1": 0.392238, + "f1_weighted": 0.412883 + }, + { + "accuracy": 0.40404, + "f1": 0.395073, + "f1_weighted": 0.414841 + }, + { + "accuracy": 0.464646, + "f1": 0.44169, + "f1_weighted": 0.453915 + }, + { + "accuracy": 0.464646, + "f1": 0.40571, + "f1_weighted": 0.463975 + }, + { + "accuracy": 0.484848, + "f1": 0.476152, + "f1_weighted": 0.487308 + }, + { + "accuracy": 0.484848, + "f1": 0.479752, + "f1_weighted": 0.489177 + }, + { + "accuracy": 0.525253, + "f1": 0.515387, + "f1_weighted": 0.538488 + }, + { + "accuracy": 0.393939, + "f1": 0.384622, + "f1_weighted": 0.394969 + }, + { + "accuracy": 0.464646, + "f1": 0.462749, + "f1_weighted": 0.462493 + } + ], + "main_score": 0.459596, + "hf_subset": "nob_Latn", + "languages": [ + "nob-Latn" + ] + }, + { + "accuracy": 0.19596, + "f1": 0.16343, + "f1_weighted": 0.167525, + "scores_per_experiment": [ + { + "accuracy": 0.252525, + "f1": 0.231654, + "f1_weighted": 0.229442 + }, + { + "accuracy": 0.212121, + "f1": 0.133221, + "f1_weighted": 0.155077 + }, + { + "accuracy": 0.141414, + "f1": 0.124504, + "f1_weighted": 0.099567 + }, + { + "accuracy": 0.181818, + "f1": 0.153192, + "f1_weighted": 0.152461 + }, + { + "accuracy": 0.20202, + "f1": 0.179842, + "f1_weighted": 0.159876 + }, + { + "accuracy": 0.161616, + "f1": 0.128415, + "f1_weighted": 0.140308 + }, + { + "accuracy": 0.191919, + "f1": 0.157758, + "f1_weighted": 0.154193 + }, + { + "accuracy": 0.222222, + "f1": 0.191577, + "f1_weighted": 0.222717 + }, + { + "accuracy": 0.181818, + "f1": 0.139821, + "f1_weighted": 0.152204 + }, + { + "accuracy": 0.212121, + "f1": 0.194312, + "f1_weighted": 0.209407 + } + ], + "main_score": 0.19596, + "hf_subset": "npi_Deva", + "languages": [ + "npi-Deva" + ] + }, + { + "accuracy": 0.184848, + "f1": 0.137517, + "f1_weighted": 0.15623, + "scores_per_experiment": [ + { + "accuracy": 0.191919, + "f1": 0.136664, + "f1_weighted": 0.174528 + }, + { + "accuracy": 0.141414, + "f1": 0.111414, + "f1_weighted": 0.142767 + }, + { + "accuracy": 0.212121, + "f1": 0.202174, + "f1_weighted": 0.197222 + }, + { + "accuracy": 0.212121, + "f1": 0.150332, + "f1_weighted": 0.161248 + }, + { + "accuracy": 0.212121, + "f1": 0.151968, + "f1_weighted": 0.177618 + }, + { + "accuracy": 0.131313, + "f1": 0.107152, + "f1_weighted": 0.103943 + }, + { + "accuracy": 0.222222, + "f1": 0.140217, + "f1_weighted": 0.178739 + }, + { + "accuracy": 0.191919, + "f1": 0.105946, + "f1_weighted": 0.155063 + }, + { + "accuracy": 0.131313, + "f1": 0.102662, + "f1_weighted": 0.091621 + }, + { + "accuracy": 0.20202, + "f1": 0.166636, + "f1_weighted": 0.179548 + } + ], + "main_score": 0.184848, + "hf_subset": "nqo_Nkoo", + "languages": [ + "nqo-Nkoo" + ] + }, + { + "accuracy": 0.336364, + "f1": 0.329137, + "f1_weighted": 0.339198, + "scores_per_experiment": [ + { + "accuracy": 0.282828, + "f1": 0.289413, + "f1_weighted": 0.290537 + }, + { + "accuracy": 0.353535, + "f1": 0.349431, + "f1_weighted": 0.348822 + }, + { + "accuracy": 0.272727, + "f1": 0.277509, + "f1_weighted": 0.248798 + }, + { + "accuracy": 0.343434, + "f1": 0.317922, + "f1_weighted": 0.339314 + }, + { + "accuracy": 0.373737, + "f1": 0.345304, + "f1_weighted": 0.366967 + }, + { + "accuracy": 0.282828, + "f1": 0.252999, + "f1_weighted": 0.291946 + }, + { + "accuracy": 0.434343, + "f1": 0.444121, + "f1_weighted": 0.441431 + }, + { + "accuracy": 0.313131, + "f1": 0.325047, + "f1_weighted": 0.324699 + }, + { + "accuracy": 0.333333, + "f1": 0.320555, + "f1_weighted": 0.350181 + }, + { + "accuracy": 0.373737, + "f1": 0.369072, + "f1_weighted": 0.389286 + } + ], + "main_score": 0.336364, + "hf_subset": "nso_Latn", + "languages": [ + "nso-Latn" + ] + }, + { + "accuracy": 0.243434, + "f1": 0.222186, + "f1_weighted": 0.243071, + "scores_per_experiment": [ + { + "accuracy": 0.222222, + "f1": 0.210049, + "f1_weighted": 0.203951 + }, + { + "accuracy": 0.252525, + "f1": 0.228378, + "f1_weighted": 0.217931 + }, + { + "accuracy": 0.151515, + "f1": 0.148123, + "f1_weighted": 0.15742 + }, + { + "accuracy": 0.313131, + "f1": 0.268244, + "f1_weighted": 0.328537 + }, + { + "accuracy": 0.242424, + "f1": 0.240329, + "f1_weighted": 0.246025 + }, + { + "accuracy": 0.252525, + "f1": 0.216393, + "f1_weighted": 0.262079 + }, + { + "accuracy": 0.212121, + "f1": 0.190972, + "f1_weighted": 0.208525 + }, + { + "accuracy": 0.292929, + "f1": 0.259462, + "f1_weighted": 0.30195 + }, + { + "accuracy": 0.222222, + "f1": 0.192799, + "f1_weighted": 0.222378 + }, + { + "accuracy": 0.272727, + "f1": 0.26711, + "f1_weighted": 0.281913 + } + ], + "main_score": 0.243434, + "hf_subset": "nus_Latn", + "languages": [ + "nus-Latn" + ] + }, + { + "accuracy": 0.371717, + "f1": 0.344278, + "f1_weighted": 0.380345, + "scores_per_experiment": [ + { + "accuracy": 0.414141, + "f1": 0.410495, + "f1_weighted": 0.438197 + }, + { + "accuracy": 0.343434, + "f1": 0.323444, + "f1_weighted": 0.349563 + }, + { + "accuracy": 0.252525, + "f1": 0.212198, + "f1_weighted": 0.246549 + }, + { + "accuracy": 0.40404, + "f1": 0.383896, + "f1_weighted": 0.405714 + }, + { + "accuracy": 0.383838, + "f1": 0.35692, + "f1_weighted": 0.401721 + }, + { + "accuracy": 0.363636, + "f1": 0.334582, + "f1_weighted": 0.37102 + }, + { + "accuracy": 0.424242, + "f1": 0.382753, + "f1_weighted": 0.445922 + }, + { + "accuracy": 0.424242, + "f1": 0.402289, + "f1_weighted": 0.439537 + }, + { + "accuracy": 0.282828, + "f1": 0.261149, + "f1_weighted": 0.284182 + }, + { + "accuracy": 0.424242, + "f1": 0.375048, + "f1_weighted": 0.421043 + } + ], + "main_score": 0.371717, + "hf_subset": "nya_Latn", + "languages": [ + "nya-Latn" + ] + }, + { + "accuracy": 0.468687, + "f1": 0.445067, + "f1_weighted": 0.470083, + "scores_per_experiment": [ + { + "accuracy": 0.525253, + "f1": 0.496971, + "f1_weighted": 0.526553 + }, + { + "accuracy": 0.474747, + "f1": 0.448472, + "f1_weighted": 0.47046 + }, + { + "accuracy": 0.393939, + "f1": 0.389495, + "f1_weighted": 0.3862 + }, + { + "accuracy": 0.505051, + "f1": 0.494807, + "f1_weighted": 0.506482 + }, + { + "accuracy": 0.484848, + "f1": 0.467263, + "f1_weighted": 0.491681 + }, + { + "accuracy": 0.464646, + "f1": 0.425627, + "f1_weighted": 0.469313 + }, + { + "accuracy": 0.464646, + "f1": 0.437981, + "f1_weighted": 0.474036 + }, + { + "accuracy": 0.424242, + "f1": 0.3938, + "f1_weighted": 0.429867 + }, + { + "accuracy": 0.434343, + "f1": 0.395832, + "f1_weighted": 0.427931 + }, + { + "accuracy": 0.515152, + "f1": 0.500426, + "f1_weighted": 0.518306 + } + ], + "main_score": 0.468687, + "hf_subset": "oci_Latn", + "languages": [ + "oci-Latn" + ] + }, + { + "accuracy": 0.182828, + "f1": 0.163691, + "f1_weighted": 0.161934, + "scores_per_experiment": [ + { + "accuracy": 0.212121, + "f1": 0.21795, + "f1_weighted": 0.220213 + }, + { + "accuracy": 0.191919, + "f1": 0.159439, + "f1_weighted": 0.14772 + }, + { + "accuracy": 0.141414, + "f1": 0.13775, + "f1_weighted": 0.103703 + }, + { + "accuracy": 0.181818, + "f1": 0.174306, + "f1_weighted": 0.148697 + }, + { + "accuracy": 0.10101, + "f1": 0.076749, + "f1_weighted": 0.061912 + }, + { + "accuracy": 0.151515, + "f1": 0.113573, + "f1_weighted": 0.122328 + }, + { + "accuracy": 0.212121, + "f1": 0.185802, + "f1_weighted": 0.202231 + }, + { + "accuracy": 0.242424, + "f1": 0.221022, + "f1_weighted": 0.240138 + }, + { + "accuracy": 0.181818, + "f1": 0.162563, + "f1_weighted": 0.175133 + }, + { + "accuracy": 0.212121, + "f1": 0.18776, + "f1_weighted": 0.19726 + } + ], + "main_score": 0.182828, + "hf_subset": "ory_Orya", + "languages": [ + "ory-Orya" + ] + }, + { + "accuracy": 0.475758, + "f1": 0.453085, + "f1_weighted": 0.488692, + "scores_per_experiment": [ + { + "accuracy": 0.505051, + "f1": 0.490846, + "f1_weighted": 0.514018 + }, + { + "accuracy": 0.505051, + "f1": 0.48299, + "f1_weighted": 0.519077 + }, + { + "accuracy": 0.434343, + "f1": 0.43443, + "f1_weighted": 0.441428 + }, + { + "accuracy": 0.515152, + "f1": 0.470675, + "f1_weighted": 0.520409 + }, + { + "accuracy": 0.454545, + "f1": 0.440212, + "f1_weighted": 0.478834 + }, + { + "accuracy": 0.494949, + "f1": 0.472078, + "f1_weighted": 0.502141 + }, + { + "accuracy": 0.454545, + "f1": 0.397025, + "f1_weighted": 0.467512 + }, + { + "accuracy": 0.494949, + "f1": 0.467313, + "f1_weighted": 0.508423 + }, + { + "accuracy": 0.373737, + "f1": 0.354015, + "f1_weighted": 0.383355 + }, + { + "accuracy": 0.525253, + "f1": 0.521264, + "f1_weighted": 0.551719 + } + ], + "main_score": 0.475758, + "hf_subset": "pag_Latn", + "languages": [ + "pag-Latn" + ] + }, + { + "accuracy": 0.173737, + "f1": 0.150924, + "f1_weighted": 0.150488, + "scores_per_experiment": [ + { + "accuracy": 0.191919, + "f1": 0.160961, + "f1_weighted": 0.175985 + }, + { + "accuracy": 0.222222, + "f1": 0.202301, + "f1_weighted": 0.190568 + }, + { + "accuracy": 0.131313, + "f1": 0.127739, + "f1_weighted": 0.109689 + }, + { + "accuracy": 0.232323, + "f1": 0.221464, + "f1_weighted": 0.199914 + }, + { + "accuracy": 0.131313, + "f1": 0.130195, + "f1_weighted": 0.113728 + }, + { + "accuracy": 0.171717, + "f1": 0.160922, + "f1_weighted": 0.158594 + }, + { + "accuracy": 0.161616, + "f1": 0.120415, + "f1_weighted": 0.126835 + }, + { + "accuracy": 0.20202, + "f1": 0.16535, + "f1_weighted": 0.178308 + }, + { + "accuracy": 0.191919, + "f1": 0.139793, + "f1_weighted": 0.158732 + }, + { + "accuracy": 0.10101, + "f1": 0.080099, + "f1_weighted": 0.092528 + } + ], + "main_score": 0.173737, + "hf_subset": "pan_Guru", + "languages": [ + "pan-Guru" + ] + }, + { + "accuracy": 0.408081, + "f1": 0.404467, + "f1_weighted": 0.410352, + "scores_per_experiment": [ + { + "accuracy": 0.454545, + "f1": 0.451194, + "f1_weighted": 0.464641 + }, + { + "accuracy": 0.40404, + "f1": 0.393133, + "f1_weighted": 0.39822 + }, + { + "accuracy": 0.40404, + "f1": 0.40473, + "f1_weighted": 0.403754 + }, + { + "accuracy": 0.414141, + "f1": 0.418948, + "f1_weighted": 0.405391 + }, + { + "accuracy": 0.393939, + "f1": 0.396721, + "f1_weighted": 0.412579 + }, + { + "accuracy": 0.474747, + "f1": 0.456871, + "f1_weighted": 0.479377 + }, + { + "accuracy": 0.353535, + "f1": 0.34397, + "f1_weighted": 0.351066 + }, + { + "accuracy": 0.424242, + "f1": 0.416957, + "f1_weighted": 0.437583 + }, + { + "accuracy": 0.333333, + "f1": 0.330649, + "f1_weighted": 0.319863 + }, + { + "accuracy": 0.424242, + "f1": 0.431497, + "f1_weighted": 0.431045 + } + ], + "main_score": 0.408081, + "hf_subset": "pap_Latn", + "languages": [ + "pap-Latn" + ] + }, + { + "accuracy": 0.247475, + "f1": 0.168149, + "f1_weighted": 0.185558, + "scores_per_experiment": [ + { + "accuracy": 0.292929, + "f1": 0.219891, + "f1_weighted": 0.214864 + }, + { + "accuracy": 0.232323, + "f1": 0.103368, + "f1_weighted": 0.125975 + }, + { + "accuracy": 0.151515, + "f1": 0.099607, + "f1_weighted": 0.110863 + }, + { + "accuracy": 0.313131, + "f1": 0.243993, + "f1_weighted": 0.242106 + }, + { + "accuracy": 0.212121, + "f1": 0.099476, + "f1_weighted": 0.11969 + }, + { + "accuracy": 0.222222, + "f1": 0.202972, + "f1_weighted": 0.201137 + }, + { + "accuracy": 0.262626, + "f1": 0.178148, + "f1_weighted": 0.19986 + }, + { + "accuracy": 0.20202, + "f1": 0.151788, + "f1_weighted": 0.16204 + }, + { + "accuracy": 0.30303, + "f1": 0.206343, + "f1_weighted": 0.246652 + }, + { + "accuracy": 0.282828, + "f1": 0.175899, + "f1_weighted": 0.232392 + } + ], + "main_score": 0.247475, + "hf_subset": "pbt_Arab", + "languages": [ + "pbt-Arab" + ] + }, + { + "accuracy": 0.176768, + "f1": 0.131064, + "f1_weighted": 0.132554, + "scores_per_experiment": [ + { + "accuracy": 0.232323, + "f1": 0.145952, + "f1_weighted": 0.156335 + }, + { + "accuracy": 0.141414, + "f1": 0.096426, + "f1_weighted": 0.081214 + }, + { + "accuracy": 0.080808, + "f1": 0.073414, + "f1_weighted": 0.055422 + }, + { + "accuracy": 0.232323, + "f1": 0.206673, + "f1_weighted": 0.208149 + }, + { + "accuracy": 0.181818, + "f1": 0.12273, + "f1_weighted": 0.122889 + }, + { + "accuracy": 0.161616, + "f1": 0.128139, + "f1_weighted": 0.125185 + }, + { + "accuracy": 0.20202, + "f1": 0.17625, + "f1_weighted": 0.179291 + }, + { + "accuracy": 0.212121, + "f1": 0.103206, + "f1_weighted": 0.145435 + }, + { + "accuracy": 0.171717, + "f1": 0.150522, + "f1_weighted": 0.158695 + }, + { + "accuracy": 0.151515, + "f1": 0.107328, + "f1_weighted": 0.092922 + } + ], + "main_score": 0.176768, + "hf_subset": "pes_Arab", + "languages": [ + "pes-Arab" + ] + }, + { + "accuracy": 0.288889, + "f1": 0.277287, + "f1_weighted": 0.290407, + "scores_per_experiment": [ + { + "accuracy": 0.30303, + "f1": 0.295788, + "f1_weighted": 0.304786 + }, + { + "accuracy": 0.242424, + "f1": 0.236138, + "f1_weighted": 0.248991 + }, + { + "accuracy": 0.292929, + "f1": 0.290594, + "f1_weighted": 0.296767 + }, + { + "accuracy": 0.323232, + "f1": 0.289922, + "f1_weighted": 0.325444 + }, + { + "accuracy": 0.323232, + "f1": 0.318048, + "f1_weighted": 0.329091 + }, + { + "accuracy": 0.222222, + "f1": 0.213889, + "f1_weighted": 0.226479 + }, + { + "accuracy": 0.252525, + "f1": 0.247882, + "f1_weighted": 0.250371 + }, + { + "accuracy": 0.292929, + "f1": 0.275571, + "f1_weighted": 0.284878 + }, + { + "accuracy": 0.313131, + "f1": 0.304698, + "f1_weighted": 0.310151 + }, + { + "accuracy": 0.323232, + "f1": 0.300341, + "f1_weighted": 0.327117 + } + ], + "main_score": 0.288889, + "hf_subset": "plt_Latn", + "languages": [ + "plt-Latn" + ] + }, + { + "accuracy": 0.311111, + "f1": 0.295048, + "f1_weighted": 0.307077, + "scores_per_experiment": [ + { + "accuracy": 0.313131, + "f1": 0.323888, + "f1_weighted": 0.310926 + }, + { + "accuracy": 0.252525, + "f1": 0.241641, + "f1_weighted": 0.251011 + }, + { + "accuracy": 0.282828, + "f1": 0.277295, + "f1_weighted": 0.269898 + }, + { + "accuracy": 0.30303, + "f1": 0.279186, + "f1_weighted": 0.30753 + }, + { + "accuracy": 0.30303, + "f1": 0.277105, + "f1_weighted": 0.293713 + }, + { + "accuracy": 0.313131, + "f1": 0.28953, + "f1_weighted": 0.304691 + }, + { + "accuracy": 0.343434, + "f1": 0.32876, + "f1_weighted": 0.338072 + }, + { + "accuracy": 0.353535, + "f1": 0.329873, + "f1_weighted": 0.354816 + }, + { + "accuracy": 0.272727, + "f1": 0.246944, + "f1_weighted": 0.265183 + }, + { + "accuracy": 0.373737, + "f1": 0.356256, + "f1_weighted": 0.374935 + } + ], + "main_score": 0.311111, + "hf_subset": "pol_Latn", + "languages": [ + "pol-Latn" + ] + }, + { + "accuracy": 0.555556, + "f1": 0.533186, + "f1_weighted": 0.557585, + "scores_per_experiment": [ + { + "accuracy": 0.565657, + "f1": 0.546367, + "f1_weighted": 0.57398 + }, + { + "accuracy": 0.525253, + "f1": 0.496524, + "f1_weighted": 0.532209 + }, + { + "accuracy": 0.444444, + "f1": 0.428646, + "f1_weighted": 0.444353 + }, + { + "accuracy": 0.59596, + "f1": 0.583673, + "f1_weighted": 0.589149 + }, + { + "accuracy": 0.545455, + "f1": 0.516084, + "f1_weighted": 0.549273 + }, + { + "accuracy": 0.555556, + "f1": 0.541984, + "f1_weighted": 0.563653 + }, + { + "accuracy": 0.565657, + "f1": 0.552339, + "f1_weighted": 0.555741 + }, + { + "accuracy": 0.555556, + "f1": 0.547427, + "f1_weighted": 0.568223 + }, + { + "accuracy": 0.636364, + "f1": 0.58026, + "f1_weighted": 0.622896 + }, + { + "accuracy": 0.565657, + "f1": 0.538561, + "f1_weighted": 0.576373 + } + ], + "main_score": 0.555556, + "hf_subset": "por_Latn", + "languages": [ + "por-Latn" + ] + }, + { + "accuracy": 0.209091, + "f1": 0.158148, + "f1_weighted": 0.156405, + "scores_per_experiment": [ + { + "accuracy": 0.272727, + "f1": 0.204376, + "f1_weighted": 0.199496 + }, + { + "accuracy": 0.161616, + "f1": 0.122648, + "f1_weighted": 0.102475 + }, + { + "accuracy": 0.121212, + "f1": 0.118926, + "f1_weighted": 0.096523 + }, + { + "accuracy": 0.272727, + "f1": 0.188437, + "f1_weighted": 0.174109 + }, + { + "accuracy": 0.151515, + "f1": 0.124772, + "f1_weighted": 0.107281 + }, + { + "accuracy": 0.181818, + "f1": 0.119408, + "f1_weighted": 0.125974 + }, + { + "accuracy": 0.232323, + "f1": 0.189814, + "f1_weighted": 0.209457 + }, + { + "accuracy": 0.30303, + "f1": 0.20738, + "f1_weighted": 0.229693 + }, + { + "accuracy": 0.181818, + "f1": 0.137179, + "f1_weighted": 0.132893 + }, + { + "accuracy": 0.212121, + "f1": 0.168536, + "f1_weighted": 0.186153 + } + ], + "main_score": 0.209091, + "hf_subset": "prs_Arab", + "languages": [ + "prs-Arab" + ] + }, + { + "accuracy": 0.347475, + "f1": 0.329146, + "f1_weighted": 0.356368, + "scores_per_experiment": [ + { + "accuracy": 0.444444, + "f1": 0.447322, + "f1_weighted": 0.448203 + }, + { + "accuracy": 0.313131, + "f1": 0.30887, + "f1_weighted": 0.346348 + }, + { + "accuracy": 0.30303, + "f1": 0.272912, + "f1_weighted": 0.30242 + }, + { + "accuracy": 0.333333, + "f1": 0.318302, + "f1_weighted": 0.329748 + }, + { + "accuracy": 0.333333, + "f1": 0.30524, + "f1_weighted": 0.348235 + }, + { + "accuracy": 0.383838, + "f1": 0.369433, + "f1_weighted": 0.392867 + }, + { + "accuracy": 0.30303, + "f1": 0.292499, + "f1_weighted": 0.318408 + }, + { + "accuracy": 0.333333, + "f1": 0.301319, + "f1_weighted": 0.339557 + }, + { + "accuracy": 0.323232, + "f1": 0.306312, + "f1_weighted": 0.326841 + }, + { + "accuracy": 0.40404, + "f1": 0.369256, + "f1_weighted": 0.411048 + } + ], + "main_score": 0.347475, + "hf_subset": "quy_Latn", + "languages": [ + "quy-Latn" + ] + }, + { + "accuracy": 0.535354, + "f1": 0.517489, + "f1_weighted": 0.540509, + "scores_per_experiment": [ + { + "accuracy": 0.525253, + "f1": 0.512955, + "f1_weighted": 0.532517 + }, + { + "accuracy": 0.545455, + "f1": 0.539076, + "f1_weighted": 0.554893 + }, + { + "accuracy": 0.474747, + "f1": 0.467149, + "f1_weighted": 0.473413 + }, + { + "accuracy": 0.585859, + "f1": 0.567224, + "f1_weighted": 0.575608 + }, + { + "accuracy": 0.545455, + "f1": 0.524006, + "f1_weighted": 0.557007 + }, + { + "accuracy": 0.565657, + "f1": 0.54156, + "f1_weighted": 0.577715 + }, + { + "accuracy": 0.474747, + "f1": 0.448617, + "f1_weighted": 0.483196 + }, + { + "accuracy": 0.565657, + "f1": 0.534631, + "f1_weighted": 0.570069 + }, + { + "accuracy": 0.525253, + "f1": 0.501176, + "f1_weighted": 0.515761 + }, + { + "accuracy": 0.545455, + "f1": 0.538497, + "f1_weighted": 0.564913 + } + ], + "main_score": 0.535354, + "hf_subset": "ron_Latn", + "languages": [ + "ron-Latn" + ] + }, + { + "accuracy": 0.280808, + "f1": 0.274182, + "f1_weighted": 0.2796, + "scores_per_experiment": [ + { + "accuracy": 0.272727, + "f1": 0.273225, + "f1_weighted": 0.276756 + }, + { + "accuracy": 0.313131, + "f1": 0.310001, + "f1_weighted": 0.30913 + }, + { + "accuracy": 0.222222, + "f1": 0.228242, + "f1_weighted": 0.210639 + }, + { + "accuracy": 0.30303, + "f1": 0.287421, + "f1_weighted": 0.296738 + }, + { + "accuracy": 0.232323, + "f1": 0.231872, + "f1_weighted": 0.259216 + }, + { + "accuracy": 0.232323, + "f1": 0.232434, + "f1_weighted": 0.21198 + }, + { + "accuracy": 0.333333, + "f1": 0.3231, + "f1_weighted": 0.309166 + }, + { + "accuracy": 0.313131, + "f1": 0.308362, + "f1_weighted": 0.324603 + }, + { + "accuracy": 0.252525, + "f1": 0.247199, + "f1_weighted": 0.250837 + }, + { + "accuracy": 0.333333, + "f1": 0.29997, + "f1_weighted": 0.346936 + } + ], + "main_score": 0.280808, + "hf_subset": "run_Latn", + "languages": [ + "run-Latn" + ] + }, + { + "accuracy": 0.215152, + "f1": 0.192037, + "f1_weighted": 0.200925, + "scores_per_experiment": [ + { + "accuracy": 0.313131, + "f1": 0.256381, + "f1_weighted": 0.30081 + }, + { + "accuracy": 0.262626, + "f1": 0.215418, + "f1_weighted": 0.234489 + }, + { + "accuracy": 0.212121, + "f1": 0.219345, + "f1_weighted": 0.196838 + }, + { + "accuracy": 0.262626, + "f1": 0.253617, + "f1_weighted": 0.259794 + }, + { + "accuracy": 0.131313, + "f1": 0.114264, + "f1_weighted": 0.109884 + }, + { + "accuracy": 0.161616, + "f1": 0.155397, + "f1_weighted": 0.169827 + }, + { + "accuracy": 0.171717, + "f1": 0.150208, + "f1_weighted": 0.152306 + }, + { + "accuracy": 0.171717, + "f1": 0.174982, + "f1_weighted": 0.168197 + }, + { + "accuracy": 0.252525, + "f1": 0.18428, + "f1_weighted": 0.208242 + }, + { + "accuracy": 0.212121, + "f1": 0.19648, + "f1_weighted": 0.208866 + } + ], + "main_score": 0.215152, + "hf_subset": "rus_Cyrl", + "languages": [ + "rus-Cyrl" + ] + }, + { + "accuracy": 0.370707, + "f1": 0.352453, + "f1_weighted": 0.359895, + "scores_per_experiment": [ + { + "accuracy": 0.282828, + "f1": 0.267751, + "f1_weighted": 0.279257 + }, + { + "accuracy": 0.363636, + "f1": 0.364719, + "f1_weighted": 0.341235 + }, + { + "accuracy": 0.383838, + "f1": 0.384642, + "f1_weighted": 0.375779 + }, + { + "accuracy": 0.40404, + "f1": 0.364864, + "f1_weighted": 0.390139 + }, + { + "accuracy": 0.373737, + "f1": 0.361491, + "f1_weighted": 0.368061 + }, + { + "accuracy": 0.383838, + "f1": 0.373311, + "f1_weighted": 0.374591 + }, + { + "accuracy": 0.363636, + "f1": 0.308643, + "f1_weighted": 0.342063 + }, + { + "accuracy": 0.444444, + "f1": 0.432827, + "f1_weighted": 0.434272 + }, + { + "accuracy": 0.353535, + "f1": 0.346145, + "f1_weighted": 0.348212 + }, + { + "accuracy": 0.353535, + "f1": 0.320142, + "f1_weighted": 0.345344 + } + ], + "main_score": 0.370707, + "hf_subset": "sag_Latn", + "languages": [ + "sag-Latn" + ] + }, + { + "accuracy": 0.188889, + "f1": 0.159352, + "f1_weighted": 0.16558, + "scores_per_experiment": [ + { + "accuracy": 0.181818, + "f1": 0.155909, + "f1_weighted": 0.151418 + }, + { + "accuracy": 0.161616, + "f1": 0.123804, + "f1_weighted": 0.137138 + }, + { + "accuracy": 0.131313, + "f1": 0.12478, + "f1_weighted": 0.113995 + }, + { + "accuracy": 0.222222, + "f1": 0.222251, + "f1_weighted": 0.201372 + }, + { + "accuracy": 0.181818, + "f1": 0.128042, + "f1_weighted": 0.136177 + }, + { + "accuracy": 0.20202, + "f1": 0.202609, + "f1_weighted": 0.233114 + }, + { + "accuracy": 0.161616, + "f1": 0.125329, + "f1_weighted": 0.134176 + }, + { + "accuracy": 0.262626, + "f1": 0.206451, + "f1_weighted": 0.225038 + }, + { + "accuracy": 0.161616, + "f1": 0.151485, + "f1_weighted": 0.15476 + }, + { + "accuracy": 0.222222, + "f1": 0.152859, + "f1_weighted": 0.168614 + } + ], + "main_score": 0.188889, + "hf_subset": "san_Deva", + "languages": [ + "san-Deva" + ] + }, + { + "accuracy": 0.113131, + "f1": 0.07521, + "f1_weighted": 0.085114, + "scores_per_experiment": [ + { + "accuracy": 0.111111, + "f1": 0.0686, + "f1_weighted": 0.083133 + }, + { + "accuracy": 0.030303, + "f1": 0.020087, + "f1_weighted": 0.018047 + }, + { + "accuracy": 0.080808, + "f1": 0.056654, + "f1_weighted": 0.043204 + }, + { + "accuracy": 0.161616, + "f1": 0.122857, + "f1_weighted": 0.119596 + }, + { + "accuracy": 0.070707, + "f1": 0.051472, + "f1_weighted": 0.04644 + }, + { + "accuracy": 0.121212, + "f1": 0.10988, + "f1_weighted": 0.130319 + }, + { + "accuracy": 0.161616, + "f1": 0.097066, + "f1_weighted": 0.12781 + }, + { + "accuracy": 0.121212, + "f1": 0.048677, + "f1_weighted": 0.064347 + }, + { + "accuracy": 0.191919, + "f1": 0.125207, + "f1_weighted": 0.169223 + }, + { + "accuracy": 0.080808, + "f1": 0.051603, + "f1_weighted": 0.049023 + } + ], + "main_score": 0.113131, + "hf_subset": "sat_Olck", + "languages": [ + "sat-Olck" + ] + }, + { + "accuracy": 0.418182, + "f1": 0.393484, + "f1_weighted": 0.424445, + "scores_per_experiment": [ + { + "accuracy": 0.383838, + "f1": 0.353781, + "f1_weighted": 0.39731 + }, + { + "accuracy": 0.424242, + "f1": 0.410375, + "f1_weighted": 0.441333 + }, + { + "accuracy": 0.30303, + "f1": 0.291581, + "f1_weighted": 0.309147 + }, + { + "accuracy": 0.464646, + "f1": 0.427484, + "f1_weighted": 0.460024 + }, + { + "accuracy": 0.454545, + "f1": 0.407192, + "f1_weighted": 0.465651 + }, + { + "accuracy": 0.444444, + "f1": 0.433972, + "f1_weighted": 0.451342 + }, + { + "accuracy": 0.454545, + "f1": 0.429882, + "f1_weighted": 0.459019 + }, + { + "accuracy": 0.40404, + "f1": 0.373647, + "f1_weighted": 0.404696 + }, + { + "accuracy": 0.474747, + "f1": 0.440411, + "f1_weighted": 0.460855 + }, + { + "accuracy": 0.373737, + "f1": 0.366514, + "f1_weighted": 0.395071 + } + ], + "main_score": 0.418182, + "hf_subset": "scn_Latn", + "languages": [ + "scn-Latn" + ] + }, + { + "accuracy": 0.269697, + "f1": 0.236784, + "f1_weighted": 0.261991, + "scores_per_experiment": [ + { + "accuracy": 0.252525, + "f1": 0.274355, + "f1_weighted": 0.283871 + }, + { + "accuracy": 0.222222, + "f1": 0.185249, + "f1_weighted": 0.196708 + }, + { + "accuracy": 0.212121, + "f1": 0.213988, + "f1_weighted": 0.19567 + }, + { + "accuracy": 0.272727, + "f1": 0.27364, + "f1_weighted": 0.288646 + }, + { + "accuracy": 0.282828, + "f1": 0.227013, + "f1_weighted": 0.271655 + }, + { + "accuracy": 0.262626, + "f1": 0.223063, + "f1_weighted": 0.253135 + }, + { + "accuracy": 0.30303, + "f1": 0.215781, + "f1_weighted": 0.279934 + }, + { + "accuracy": 0.363636, + "f1": 0.307711, + "f1_weighted": 0.353751 + }, + { + "accuracy": 0.262626, + "f1": 0.217727, + "f1_weighted": 0.245857 + }, + { + "accuracy": 0.262626, + "f1": 0.229311, + "f1_weighted": 0.250681 + } + ], + "main_score": 0.269697, + "hf_subset": "shn_Mymr", + "languages": [ + "shn-Mymr" + ] + }, + { + "accuracy": 0.208081, + "f1": 0.152007, + "f1_weighted": 0.170824, + "scores_per_experiment": [ + { + "accuracy": 0.181818, + "f1": 0.152422, + "f1_weighted": 0.163257 + }, + { + "accuracy": 0.151515, + "f1": 0.098963, + "f1_weighted": 0.107563 + }, + { + "accuracy": 0.171717, + "f1": 0.135175, + "f1_weighted": 0.138192 + }, + { + "accuracy": 0.242424, + "f1": 0.228889, + "f1_weighted": 0.232586 + }, + { + "accuracy": 0.111111, + "f1": 0.084227, + "f1_weighted": 0.058588 + }, + { + "accuracy": 0.282828, + "f1": 0.160102, + "f1_weighted": 0.224239 + }, + { + "accuracy": 0.222222, + "f1": 0.144522, + "f1_weighted": 0.157551 + }, + { + "accuracy": 0.262626, + "f1": 0.184628, + "f1_weighted": 0.229295 + }, + { + "accuracy": 0.242424, + "f1": 0.150108, + "f1_weighted": 0.185218 + }, + { + "accuracy": 0.212121, + "f1": 0.181037, + "f1_weighted": 0.211752 + } + ], + "main_score": 0.208081, + "hf_subset": "sin_Sinh", + "languages": [ + "sin-Sinh" + ] + }, + { + "accuracy": 0.30202, + "f1": 0.296247, + "f1_weighted": 0.300892, + "scores_per_experiment": [ + { + "accuracy": 0.434343, + "f1": 0.410416, + "f1_weighted": 0.442607 + }, + { + "accuracy": 0.242424, + "f1": 0.25016, + "f1_weighted": 0.239241 + }, + { + "accuracy": 0.212121, + "f1": 0.207392, + "f1_weighted": 0.192899 + }, + { + "accuracy": 0.363636, + "f1": 0.360244, + "f1_weighted": 0.368876 + }, + { + "accuracy": 0.171717, + "f1": 0.171748, + "f1_weighted": 0.184157 + }, + { + "accuracy": 0.323232, + "f1": 0.316629, + "f1_weighted": 0.316074 + }, + { + "accuracy": 0.282828, + "f1": 0.291443, + "f1_weighted": 0.271486 + }, + { + "accuracy": 0.363636, + "f1": 0.330075, + "f1_weighted": 0.363978 + }, + { + "accuracy": 0.292929, + "f1": 0.29129, + "f1_weighted": 0.29248 + }, + { + "accuracy": 0.333333, + "f1": 0.333076, + "f1_weighted": 0.337123 + } + ], + "main_score": 0.30202, + "hf_subset": "slk_Latn", + "languages": [ + "slk-Latn" + ] + }, + { + "accuracy": 0.314141, + "f1": 0.300283, + "f1_weighted": 0.314182, + "scores_per_experiment": [ + { + "accuracy": 0.30303, + "f1": 0.286541, + "f1_weighted": 0.315351 + }, + { + "accuracy": 0.282828, + "f1": 0.259082, + "f1_weighted": 0.302629 + }, + { + "accuracy": 0.292929, + "f1": 0.296397, + "f1_weighted": 0.290798 + }, + { + "accuracy": 0.323232, + "f1": 0.32749, + "f1_weighted": 0.327118 + }, + { + "accuracy": 0.373737, + "f1": 0.3626, + "f1_weighted": 0.383702 + }, + { + "accuracy": 0.343434, + "f1": 0.336294, + "f1_weighted": 0.334517 + }, + { + "accuracy": 0.292929, + "f1": 0.280503, + "f1_weighted": 0.279223 + }, + { + "accuracy": 0.373737, + "f1": 0.334779, + "f1_weighted": 0.360832 + }, + { + "accuracy": 0.262626, + "f1": 0.24142, + "f1_weighted": 0.242349 + }, + { + "accuracy": 0.292929, + "f1": 0.277723, + "f1_weighted": 0.305304 + } + ], + "main_score": 0.314141, + "hf_subset": "slv_Latn", + "languages": [ + "slv-Latn" + ] + }, + { + "accuracy": 0.39798, + "f1": 0.376853, + "f1_weighted": 0.409249, + "scores_per_experiment": [ + { + "accuracy": 0.414141, + "f1": 0.392561, + "f1_weighted": 0.427447 + }, + { + "accuracy": 0.363636, + "f1": 0.334967, + "f1_weighted": 0.378924 + }, + { + "accuracy": 0.373737, + "f1": 0.363035, + "f1_weighted": 0.3886 + }, + { + "accuracy": 0.393939, + "f1": 0.38378, + "f1_weighted": 0.39782 + }, + { + "accuracy": 0.393939, + "f1": 0.395633, + "f1_weighted": 0.407582 + }, + { + "accuracy": 0.383838, + "f1": 0.363739, + "f1_weighted": 0.390858 + }, + { + "accuracy": 0.434343, + "f1": 0.410891, + "f1_weighted": 0.440933 + }, + { + "accuracy": 0.383838, + "f1": 0.35678, + "f1_weighted": 0.393696 + }, + { + "accuracy": 0.434343, + "f1": 0.388183, + "f1_weighted": 0.44089 + }, + { + "accuracy": 0.40404, + "f1": 0.378959, + "f1_weighted": 0.425736 + } + ], + "main_score": 0.39798, + "hf_subset": "smo_Latn", + "languages": [ + "smo-Latn" + ] + }, + { + "accuracy": 0.308081, + "f1": 0.295389, + "f1_weighted": 0.30529, + "scores_per_experiment": [ + { + "accuracy": 0.313131, + "f1": 0.326291, + "f1_weighted": 0.335045 + }, + { + "accuracy": 0.282828, + "f1": 0.277608, + "f1_weighted": 0.275756 + }, + { + "accuracy": 0.242424, + "f1": 0.227974, + "f1_weighted": 0.244743 + }, + { + "accuracy": 0.333333, + "f1": 0.321006, + "f1_weighted": 0.328994 + }, + { + "accuracy": 0.30303, + "f1": 0.276628, + "f1_weighted": 0.285655 + }, + { + "accuracy": 0.272727, + "f1": 0.260384, + "f1_weighted": 0.261891 + }, + { + "accuracy": 0.323232, + "f1": 0.299934, + "f1_weighted": 0.318849 + }, + { + "accuracy": 0.313131, + "f1": 0.294025, + "f1_weighted": 0.310396 + }, + { + "accuracy": 0.363636, + "f1": 0.367766, + "f1_weighted": 0.365502 + }, + { + "accuracy": 0.333333, + "f1": 0.302274, + "f1_weighted": 0.326064 + } + ], + "main_score": 0.308081, + "hf_subset": "sna_Latn", + "languages": [ + "sna-Latn" + ] + }, + { + "accuracy": 0.212121, + "f1": 0.16157, + "f1_weighted": 0.160909, + "scores_per_experiment": [ + { + "accuracy": 0.292929, + "f1": 0.203081, + "f1_weighted": 0.225424 + }, + { + "accuracy": 0.171717, + "f1": 0.113916, + "f1_weighted": 0.0955 + }, + { + "accuracy": 0.121212, + "f1": 0.120035, + "f1_weighted": 0.114213 + }, + { + "accuracy": 0.262626, + "f1": 0.176047, + "f1_weighted": 0.16563 + }, + { + "accuracy": 0.232323, + "f1": 0.169454, + "f1_weighted": 0.1725 + }, + { + "accuracy": 0.151515, + "f1": 0.09721, + "f1_weighted": 0.095089 + }, + { + "accuracy": 0.212121, + "f1": 0.192689, + "f1_weighted": 0.189956 + }, + { + "accuracy": 0.30303, + "f1": 0.237724, + "f1_weighted": 0.264014 + }, + { + "accuracy": 0.181818, + "f1": 0.155828, + "f1_weighted": 0.127284 + }, + { + "accuracy": 0.191919, + "f1": 0.149715, + "f1_weighted": 0.159482 + } + ], + "main_score": 0.212121, + "hf_subset": "snd_Arab", + "languages": [ + "snd-Arab" + ] + }, + { + "accuracy": 0.283838, + "f1": 0.273975, + "f1_weighted": 0.2825, + "scores_per_experiment": [ + { + "accuracy": 0.313131, + "f1": 0.303231, + "f1_weighted": 0.317634 + }, + { + "accuracy": 0.313131, + "f1": 0.301276, + "f1_weighted": 0.298205 + }, + { + "accuracy": 0.292929, + "f1": 0.270264, + "f1_weighted": 0.285659 + }, + { + "accuracy": 0.282828, + "f1": 0.266652, + "f1_weighted": 0.272867 + }, + { + "accuracy": 0.242424, + "f1": 0.229896, + "f1_weighted": 0.243744 + }, + { + "accuracy": 0.262626, + "f1": 0.242506, + "f1_weighted": 0.269094 + }, + { + "accuracy": 0.292929, + "f1": 0.29789, + "f1_weighted": 0.280142 + }, + { + "accuracy": 0.272727, + "f1": 0.275796, + "f1_weighted": 0.291606 + }, + { + "accuracy": 0.292929, + "f1": 0.281175, + "f1_weighted": 0.289491 + }, + { + "accuracy": 0.272727, + "f1": 0.271062, + "f1_weighted": 0.27656 + } + ], + "main_score": 0.283838, + "hf_subset": "som_Latn", + "languages": [ + "som-Latn" + ] + }, + { + "accuracy": 0.310101, + "f1": 0.305318, + "f1_weighted": 0.312127, + "scores_per_experiment": [ + { + "accuracy": 0.292929, + "f1": 0.308249, + "f1_weighted": 0.307564 + }, + { + "accuracy": 0.292929, + "f1": 0.27293, + "f1_weighted": 0.293928 + }, + { + "accuracy": 0.282828, + "f1": 0.285158, + "f1_weighted": 0.262908 + }, + { + "accuracy": 0.40404, + "f1": 0.412919, + "f1_weighted": 0.411444 + }, + { + "accuracy": 0.232323, + "f1": 0.224738, + "f1_weighted": 0.230913 + }, + { + "accuracy": 0.313131, + "f1": 0.312539, + "f1_weighted": 0.320612 + }, + { + "accuracy": 0.333333, + "f1": 0.297226, + "f1_weighted": 0.339412 + }, + { + "accuracy": 0.323232, + "f1": 0.316498, + "f1_weighted": 0.335614 + }, + { + "accuracy": 0.30303, + "f1": 0.297514, + "f1_weighted": 0.296908 + }, + { + "accuracy": 0.323232, + "f1": 0.325411, + "f1_weighted": 0.32197 + } + ], + "main_score": 0.310101, + "hf_subset": "sot_Latn", + "languages": [ + "sot-Latn" + ] + }, + { + "accuracy": 0.552525, + "f1": 0.53054, + "f1_weighted": 0.555591, + "scores_per_experiment": [ + { + "accuracy": 0.636364, + "f1": 0.607253, + "f1_weighted": 0.638059 + }, + { + "accuracy": 0.535354, + "f1": 0.516958, + "f1_weighted": 0.539042 + }, + { + "accuracy": 0.464646, + "f1": 0.458275, + "f1_weighted": 0.456791 + }, + { + "accuracy": 0.535354, + "f1": 0.490922, + "f1_weighted": 0.529831 + }, + { + "accuracy": 0.555556, + "f1": 0.540724, + "f1_weighted": 0.551928 + }, + { + "accuracy": 0.565657, + "f1": 0.546411, + "f1_weighted": 0.58883 + }, + { + "accuracy": 0.535354, + "f1": 0.519802, + "f1_weighted": 0.534358 + }, + { + "accuracy": 0.575758, + "f1": 0.551253, + "f1_weighted": 0.582319 + }, + { + "accuracy": 0.545455, + "f1": 0.498676, + "f1_weighted": 0.551934 + }, + { + "accuracy": 0.575758, + "f1": 0.57513, + "f1_weighted": 0.582818 + } + ], + "main_score": 0.552525, + "hf_subset": "spa_Latn", + "languages": [ + "spa-Latn" + ] + }, + { + "accuracy": 0.382828, + "f1": 0.36141, + "f1_weighted": 0.384388, + "scores_per_experiment": [ + { + "accuracy": 0.454545, + "f1": 0.415215, + "f1_weighted": 0.456285 + }, + { + "accuracy": 0.323232, + "f1": 0.301046, + "f1_weighted": 0.333048 + }, + { + "accuracy": 0.323232, + "f1": 0.320716, + "f1_weighted": 0.315304 + }, + { + "accuracy": 0.393939, + "f1": 0.387554, + "f1_weighted": 0.39242 + }, + { + "accuracy": 0.373737, + "f1": 0.354413, + "f1_weighted": 0.375851 + }, + { + "accuracy": 0.414141, + "f1": 0.397097, + "f1_weighted": 0.411934 + }, + { + "accuracy": 0.414141, + "f1": 0.4171, + "f1_weighted": 0.415098 + }, + { + "accuracy": 0.40404, + "f1": 0.349086, + "f1_weighted": 0.404068 + }, + { + "accuracy": 0.343434, + "f1": 0.300206, + "f1_weighted": 0.341196 + }, + { + "accuracy": 0.383838, + "f1": 0.371671, + "f1_weighted": 0.398676 + } + ], + "main_score": 0.382828, + "hf_subset": "srd_Latn", + "languages": [ + "srd-Latn" + ] + }, + { + "accuracy": 0.233333, + "f1": 0.205441, + "f1_weighted": 0.215087, + "scores_per_experiment": [ + { + "accuracy": 0.282828, + "f1": 0.275307, + "f1_weighted": 0.274729 + }, + { + "accuracy": 0.232323, + "f1": 0.224492, + "f1_weighted": 0.207578 + }, + { + "accuracy": 0.252525, + "f1": 0.230222, + "f1_weighted": 0.221002 + }, + { + "accuracy": 0.30303, + "f1": 0.230438, + "f1_weighted": 0.269853 + }, + { + "accuracy": 0.222222, + "f1": 0.202027, + "f1_weighted": 0.213449 + }, + { + "accuracy": 0.20202, + "f1": 0.175837, + "f1_weighted": 0.212822 + }, + { + "accuracy": 0.191919, + "f1": 0.142083, + "f1_weighted": 0.15586 + }, + { + "accuracy": 0.20202, + "f1": 0.183593, + "f1_weighted": 0.193554 + }, + { + "accuracy": 0.212121, + "f1": 0.190855, + "f1_weighted": 0.192404 + }, + { + "accuracy": 0.232323, + "f1": 0.199554, + "f1_weighted": 0.209624 + } + ], + "main_score": 0.233333, + "hf_subset": "srp_Cyrl", + "languages": [ + "srp-Cyrl" + ] + }, + { + "accuracy": 0.277778, + "f1": 0.262126, + "f1_weighted": 0.27633, + "scores_per_experiment": [ + { + "accuracy": 0.282828, + "f1": 0.280891, + "f1_weighted": 0.291435 + }, + { + "accuracy": 0.272727, + "f1": 0.277848, + "f1_weighted": 0.27344 + }, + { + "accuracy": 0.242424, + "f1": 0.227934, + "f1_weighted": 0.227571 + }, + { + "accuracy": 0.242424, + "f1": 0.230886, + "f1_weighted": 0.244753 + }, + { + "accuracy": 0.333333, + "f1": 0.306561, + "f1_weighted": 0.330429 + }, + { + "accuracy": 0.242424, + "f1": 0.235051, + "f1_weighted": 0.254052 + }, + { + "accuracy": 0.30303, + "f1": 0.259471, + "f1_weighted": 0.288792 + }, + { + "accuracy": 0.232323, + "f1": 0.223193, + "f1_weighted": 0.22746 + }, + { + "accuracy": 0.343434, + "f1": 0.305327, + "f1_weighted": 0.32608 + }, + { + "accuracy": 0.282828, + "f1": 0.2741, + "f1_weighted": 0.299291 + } + ], + "main_score": 0.277778, + "hf_subset": "ssw_Latn", + "languages": [ + "ssw-Latn" + ] + }, + { + "accuracy": 0.356566, + "f1": 0.348347, + "f1_weighted": 0.36274, + "scores_per_experiment": [ + { + "accuracy": 0.323232, + "f1": 0.337619, + "f1_weighted": 0.340572 + }, + { + "accuracy": 0.383838, + "f1": 0.382588, + "f1_weighted": 0.372699 + }, + { + "accuracy": 0.40404, + "f1": 0.402637, + "f1_weighted": 0.414919 + }, + { + "accuracy": 0.383838, + "f1": 0.371706, + "f1_weighted": 0.388389 + }, + { + "accuracy": 0.353535, + "f1": 0.34049, + "f1_weighted": 0.362329 + }, + { + "accuracy": 0.30303, + "f1": 0.289528, + "f1_weighted": 0.313781 + }, + { + "accuracy": 0.333333, + "f1": 0.317372, + "f1_weighted": 0.329887 + }, + { + "accuracy": 0.383838, + "f1": 0.385956, + "f1_weighted": 0.398829 + }, + { + "accuracy": 0.292929, + "f1": 0.26636, + "f1_weighted": 0.288667 + }, + { + "accuracy": 0.40404, + "f1": 0.389211, + "f1_weighted": 0.417327 + } + ], + "main_score": 0.356566, + "hf_subset": "sun_Latn", + "languages": [ + "sun-Latn" + ] + }, + { + "accuracy": 0.40202, + "f1": 0.391123, + "f1_weighted": 0.40951, + "scores_per_experiment": [ + { + "accuracy": 0.444444, + "f1": 0.42218, + "f1_weighted": 0.465418 + }, + { + "accuracy": 0.383838, + "f1": 0.357055, + "f1_weighted": 0.410085 + }, + { + "accuracy": 0.363636, + "f1": 0.372997, + "f1_weighted": 0.34212 + }, + { + "accuracy": 0.414141, + "f1": 0.392344, + "f1_weighted": 0.411697 + }, + { + "accuracy": 0.363636, + "f1": 0.348, + "f1_weighted": 0.370306 + }, + { + "accuracy": 0.393939, + "f1": 0.3995, + "f1_weighted": 0.405964 + }, + { + "accuracy": 0.393939, + "f1": 0.374777, + "f1_weighted": 0.391482 + }, + { + "accuracy": 0.424242, + "f1": 0.429569, + "f1_weighted": 0.435962 + }, + { + "accuracy": 0.383838, + "f1": 0.368249, + "f1_weighted": 0.400828 + }, + { + "accuracy": 0.454545, + "f1": 0.446554, + "f1_weighted": 0.461234 + } + ], + "main_score": 0.40202, + "hf_subset": "swe_Latn", + "languages": [ + "swe-Latn" + ] + }, + { + "accuracy": 0.308081, + "f1": 0.292715, + "f1_weighted": 0.308891, + "scores_per_experiment": [ + { + "accuracy": 0.333333, + "f1": 0.304308, + "f1_weighted": 0.330696 + }, + { + "accuracy": 0.262626, + "f1": 0.264719, + "f1_weighted": 0.257677 + }, + { + "accuracy": 0.252525, + "f1": 0.249219, + "f1_weighted": 0.253105 + }, + { + "accuracy": 0.373737, + "f1": 0.33945, + "f1_weighted": 0.358409 + }, + { + "accuracy": 0.30303, + "f1": 0.274892, + "f1_weighted": 0.307577 + }, + { + "accuracy": 0.333333, + "f1": 0.31888, + "f1_weighted": 0.339566 + }, + { + "accuracy": 0.232323, + "f1": 0.227546, + "f1_weighted": 0.228203 + }, + { + "accuracy": 0.313131, + "f1": 0.324634, + "f1_weighted": 0.320649 + }, + { + "accuracy": 0.363636, + "f1": 0.34026, + "f1_weighted": 0.369485 + }, + { + "accuracy": 0.313131, + "f1": 0.283237, + "f1_weighted": 0.323548 + } + ], + "main_score": 0.308081, + "hf_subset": "swh_Latn", + "languages": [ + "swh-Latn" + ] + }, + { + "accuracy": 0.268687, + "f1": 0.259594, + "f1_weighted": 0.273922, + "scores_per_experiment": [ + { + "accuracy": 0.272727, + "f1": 0.287303, + "f1_weighted": 0.262913 + }, + { + "accuracy": 0.262626, + "f1": 0.240779, + "f1_weighted": 0.270722 + }, + { + "accuracy": 0.222222, + "f1": 0.224099, + "f1_weighted": 0.219397 + }, + { + "accuracy": 0.262626, + "f1": 0.267137, + "f1_weighted": 0.281724 + }, + { + "accuracy": 0.262626, + "f1": 0.242672, + "f1_weighted": 0.261355 + }, + { + "accuracy": 0.242424, + "f1": 0.214921, + "f1_weighted": 0.248334 + }, + { + "accuracy": 0.292929, + "f1": 0.277851, + "f1_weighted": 0.288713 + }, + { + "accuracy": 0.292929, + "f1": 0.276138, + "f1_weighted": 0.306409 + }, + { + "accuracy": 0.282828, + "f1": 0.261336, + "f1_weighted": 0.293597 + }, + { + "accuracy": 0.292929, + "f1": 0.3037, + "f1_weighted": 0.306055 + } + ], + "main_score": 0.268687, + "hf_subset": "szl_Latn", + "languages": [ + "szl-Latn" + ] + }, + { + "accuracy": 0.193939, + "f1": 0.156664, + "f1_weighted": 0.171772, + "scores_per_experiment": [ + { + "accuracy": 0.242424, + "f1": 0.17238, + "f1_weighted": 0.187812 + }, + { + "accuracy": 0.242424, + "f1": 0.2116, + "f1_weighted": 0.234939 + }, + { + "accuracy": 0.111111, + "f1": 0.101128, + "f1_weighted": 0.075767 + }, + { + "accuracy": 0.232323, + "f1": 0.198973, + "f1_weighted": 0.223799 + }, + { + "accuracy": 0.121212, + "f1": 0.112271, + "f1_weighted": 0.095434 + }, + { + "accuracy": 0.191919, + "f1": 0.140435, + "f1_weighted": 0.16522 + }, + { + "accuracy": 0.20202, + "f1": 0.131434, + "f1_weighted": 0.170576 + }, + { + "accuracy": 0.20202, + "f1": 0.175113, + "f1_weighted": 0.197994 + }, + { + "accuracy": 0.242424, + "f1": 0.183075, + "f1_weighted": 0.225193 + }, + { + "accuracy": 0.151515, + "f1": 0.140232, + "f1_weighted": 0.140989 + } + ], + "main_score": 0.193939, + "hf_subset": "tam_Taml", + "languages": [ + "tam-Taml" + ] + }, + { + "accuracy": 0.2, + "f1": 0.159016, + "f1_weighted": 0.16351, + "scores_per_experiment": [ + { + "accuracy": 0.272727, + "f1": 0.178252, + "f1_weighted": 0.192683 + }, + { + "accuracy": 0.131313, + "f1": 0.112242, + "f1_weighted": 0.107709 + }, + { + "accuracy": 0.111111, + "f1": 0.098578, + "f1_weighted": 0.090747 + }, + { + "accuracy": 0.191919, + "f1": 0.179165, + "f1_weighted": 0.152692 + }, + { + "accuracy": 0.252525, + "f1": 0.201785, + "f1_weighted": 0.197242 + }, + { + "accuracy": 0.151515, + "f1": 0.09742, + "f1_weighted": 0.091361 + }, + { + "accuracy": 0.262626, + "f1": 0.186896, + "f1_weighted": 0.231785 + }, + { + "accuracy": 0.212121, + "f1": 0.189789, + "f1_weighted": 0.200884 + }, + { + "accuracy": 0.252525, + "f1": 0.214539, + "f1_weighted": 0.250008 + }, + { + "accuracy": 0.161616, + "f1": 0.131495, + "f1_weighted": 0.119986 + } + ], + "main_score": 0.2, + "hf_subset": "taq_Tfng", + "languages": [ + "taq-Tfng" + ] + }, + { + "accuracy": 0.20404, + "f1": 0.186637, + "f1_weighted": 0.187281, + "scores_per_experiment": [ + { + "accuracy": 0.272727, + "f1": 0.236494, + "f1_weighted": 0.238841 + }, + { + "accuracy": 0.171717, + "f1": 0.165844, + "f1_weighted": 0.158959 + }, + { + "accuracy": 0.111111, + "f1": 0.117246, + "f1_weighted": 0.112027 + }, + { + "accuracy": 0.212121, + "f1": 0.162987, + "f1_weighted": 0.200109 + }, + { + "accuracy": 0.191919, + "f1": 0.16658, + "f1_weighted": 0.1773 + }, + { + "accuracy": 0.212121, + "f1": 0.193853, + "f1_weighted": 0.196136 + }, + { + "accuracy": 0.242424, + "f1": 0.241992, + "f1_weighted": 0.219983 + }, + { + "accuracy": 0.242424, + "f1": 0.233049, + "f1_weighted": 0.247492 + }, + { + "accuracy": 0.191919, + "f1": 0.184147, + "f1_weighted": 0.161017 + }, + { + "accuracy": 0.191919, + "f1": 0.164177, + "f1_weighted": 0.160948 + } + ], + "main_score": 0.20404, + "hf_subset": "tat_Cyrl", + "languages": [ + "tat-Cyrl" + ] + }, + { + "accuracy": 0.190909, + "f1": 0.160004, + "f1_weighted": 0.161175, + "scores_per_experiment": [ + { + "accuracy": 0.242424, + "f1": 0.144327, + "f1_weighted": 0.158918 + }, + { + "accuracy": 0.151515, + "f1": 0.116188, + "f1_weighted": 0.106922 + }, + { + "accuracy": 0.151515, + "f1": 0.152053, + "f1_weighted": 0.124631 + }, + { + "accuracy": 0.20202, + "f1": 0.197786, + "f1_weighted": 0.179278 + }, + { + "accuracy": 0.171717, + "f1": 0.130716, + "f1_weighted": 0.133902 + }, + { + "accuracy": 0.131313, + "f1": 0.11456, + "f1_weighted": 0.121259 + }, + { + "accuracy": 0.252525, + "f1": 0.186159, + "f1_weighted": 0.196974 + }, + { + "accuracy": 0.232323, + "f1": 0.192752, + "f1_weighted": 0.22655 + }, + { + "accuracy": 0.181818, + "f1": 0.165969, + "f1_weighted": 0.172664 + }, + { + "accuracy": 0.191919, + "f1": 0.199533, + "f1_weighted": 0.190647 + } + ], + "main_score": 0.190909, + "hf_subset": "tel_Telu", + "languages": [ + "tel-Telu" + ] + }, + { + "accuracy": 0.215152, + "f1": 0.195841, + "f1_weighted": 0.211088, + "scores_per_experiment": [ + { + "accuracy": 0.232323, + "f1": 0.187906, + "f1_weighted": 0.216576 + }, + { + "accuracy": 0.222222, + "f1": 0.22865, + "f1_weighted": 0.226875 + }, + { + "accuracy": 0.141414, + "f1": 0.142621, + "f1_weighted": 0.146299 + }, + { + "accuracy": 0.333333, + "f1": 0.308071, + "f1_weighted": 0.338962 + }, + { + "accuracy": 0.10101, + "f1": 0.071069, + "f1_weighted": 0.065308 + }, + { + "accuracy": 0.191919, + "f1": 0.172693, + "f1_weighted": 0.203008 + }, + { + "accuracy": 0.252525, + "f1": 0.20722, + "f1_weighted": 0.234502 + }, + { + "accuracy": 0.212121, + "f1": 0.20197, + "f1_weighted": 0.219963 + }, + { + "accuracy": 0.262626, + "f1": 0.239842, + "f1_weighted": 0.260659 + }, + { + "accuracy": 0.20202, + "f1": 0.198373, + "f1_weighted": 0.198728 + } + ], + "main_score": 0.215152, + "hf_subset": "tgk_Cyrl", + "languages": [ + "tgk-Cyrl" + ] + }, + { + "accuracy": 0.442424, + "f1": 0.414961, + "f1_weighted": 0.448577, + "scores_per_experiment": [ + { + "accuracy": 0.414141, + "f1": 0.405132, + "f1_weighted": 0.422012 + }, + { + "accuracy": 0.444444, + "f1": 0.406469, + "f1_weighted": 0.446773 + }, + { + "accuracy": 0.333333, + "f1": 0.324924, + "f1_weighted": 0.322828 + }, + { + "accuracy": 0.505051, + "f1": 0.463219, + "f1_weighted": 0.492836 + }, + { + "accuracy": 0.393939, + "f1": 0.362964, + "f1_weighted": 0.405752 + }, + { + "accuracy": 0.434343, + "f1": 0.39917, + "f1_weighted": 0.448785 + }, + { + "accuracy": 0.494949, + "f1": 0.443253, + "f1_weighted": 0.500842 + }, + { + "accuracy": 0.515152, + "f1": 0.512392, + "f1_weighted": 0.545943 + }, + { + "accuracy": 0.454545, + "f1": 0.414987, + "f1_weighted": 0.453046 + }, + { + "accuracy": 0.434343, + "f1": 0.417095, + "f1_weighted": 0.446952 + } + ], + "main_score": 0.442424, + "hf_subset": "tgl_Latn", + "languages": [ + "tgl-Latn" + ] + }, + { + "accuracy": 0.235354, + "f1": 0.1716, + "f1_weighted": 0.196266, + "scores_per_experiment": [ + { + "accuracy": 0.20202, + "f1": 0.17699, + "f1_weighted": 0.186186 + }, + { + "accuracy": 0.262626, + "f1": 0.153714, + "f1_weighted": 0.200815 + }, + { + "accuracy": 0.161616, + "f1": 0.135758, + "f1_weighted": 0.151638 + }, + { + "accuracy": 0.282828, + "f1": 0.230003, + "f1_weighted": 0.252516 + }, + { + "accuracy": 0.272727, + "f1": 0.200273, + "f1_weighted": 0.216836 + }, + { + "accuracy": 0.242424, + "f1": 0.131833, + "f1_weighted": 0.172753 + }, + { + "accuracy": 0.212121, + "f1": 0.146906, + "f1_weighted": 0.153188 + }, + { + "accuracy": 0.30303, + "f1": 0.211276, + "f1_weighted": 0.258632 + }, + { + "accuracy": 0.151515, + "f1": 0.130403, + "f1_weighted": 0.131361 + }, + { + "accuracy": 0.262626, + "f1": 0.19884, + "f1_weighted": 0.238735 + } + ], + "main_score": 0.235354, + "hf_subset": "tha_Thai", + "languages": [ + "tha-Thai" + ] + }, + { + "accuracy": 0.215152, + "f1": 0.154758, + "f1_weighted": 0.163813, + "scores_per_experiment": [ + { + "accuracy": 0.242424, + "f1": 0.140217, + "f1_weighted": 0.154603 + }, + { + "accuracy": 0.191919, + "f1": 0.144897, + "f1_weighted": 0.132074 + }, + { + "accuracy": 0.121212, + "f1": 0.098722, + "f1_weighted": 0.077625 + }, + { + "accuracy": 0.30303, + "f1": 0.24536, + "f1_weighted": 0.26375 + }, + { + "accuracy": 0.20202, + "f1": 0.17589, + "f1_weighted": 0.169563 + }, + { + "accuracy": 0.181818, + "f1": 0.129975, + "f1_weighted": 0.123306 + }, + { + "accuracy": 0.242424, + "f1": 0.162187, + "f1_weighted": 0.169065 + }, + { + "accuracy": 0.20202, + "f1": 0.122069, + "f1_weighted": 0.166984 + }, + { + "accuracy": 0.272727, + "f1": 0.164323, + "f1_weighted": 0.218004 + }, + { + "accuracy": 0.191919, + "f1": 0.163944, + "f1_weighted": 0.163151 + } + ], + "main_score": 0.215152, + "hf_subset": "tir_Ethi", + "languages": [ + "tir-Ethi" + ] + }, + { + "accuracy": 0.526263, + "f1": 0.502046, + "f1_weighted": 0.527392, + "scores_per_experiment": [ + { + "accuracy": 0.565657, + "f1": 0.559775, + "f1_weighted": 0.568958 + }, + { + "accuracy": 0.434343, + "f1": 0.365521, + "f1_weighted": 0.425286 + }, + { + "accuracy": 0.434343, + "f1": 0.436671, + "f1_weighted": 0.433787 + }, + { + "accuracy": 0.484848, + "f1": 0.469779, + "f1_weighted": 0.495389 + }, + { + "accuracy": 0.535354, + "f1": 0.509585, + "f1_weighted": 0.540782 + }, + { + "accuracy": 0.575758, + "f1": 0.531829, + "f1_weighted": 0.573583 + }, + { + "accuracy": 0.606061, + "f1": 0.57103, + "f1_weighted": 0.599211 + }, + { + "accuracy": 0.555556, + "f1": 0.533072, + "f1_weighted": 0.562927 + }, + { + "accuracy": 0.494949, + "f1": 0.462215, + "f1_weighted": 0.492951 + }, + { + "accuracy": 0.575758, + "f1": 0.580983, + "f1_weighted": 0.581042 + } + ], + "main_score": 0.526263, + "hf_subset": "tpi_Latn", + "languages": [ + "tpi-Latn" + ] + }, + { + "accuracy": 0.319192, + "f1": 0.302132, + "f1_weighted": 0.320504, + "scores_per_experiment": [ + { + "accuracy": 0.343434, + "f1": 0.331038, + "f1_weighted": 0.353482 + }, + { + "accuracy": 0.30303, + "f1": 0.302919, + "f1_weighted": 0.301108 + }, + { + "accuracy": 0.363636, + "f1": 0.321308, + "f1_weighted": 0.361826 + }, + { + "accuracy": 0.353535, + "f1": 0.339663, + "f1_weighted": 0.36031 + }, + { + "accuracy": 0.30303, + "f1": 0.300277, + "f1_weighted": 0.31295 + }, + { + "accuracy": 0.30303, + "f1": 0.271574, + "f1_weighted": 0.292801 + }, + { + "accuracy": 0.333333, + "f1": 0.320625, + "f1_weighted": 0.339047 + }, + { + "accuracy": 0.363636, + "f1": 0.339606, + "f1_weighted": 0.366307 + }, + { + "accuracy": 0.252525, + "f1": 0.249411, + "f1_weighted": 0.247803 + }, + { + "accuracy": 0.272727, + "f1": 0.244896, + "f1_weighted": 0.269402 + } + ], + "main_score": 0.319192, + "hf_subset": "tsn_Latn", + "languages": [ + "tsn-Latn" + ] + }, + { + "accuracy": 0.332323, + "f1": 0.30807, + "f1_weighted": 0.33805, + "scores_per_experiment": [ + { + "accuracy": 0.353535, + "f1": 0.305074, + "f1_weighted": 0.350267 + }, + { + "accuracy": 0.363636, + "f1": 0.349868, + "f1_weighted": 0.371978 + }, + { + "accuracy": 0.272727, + "f1": 0.254043, + "f1_weighted": 0.254015 + }, + { + "accuracy": 0.333333, + "f1": 0.300179, + "f1_weighted": 0.336117 + }, + { + "accuracy": 0.353535, + "f1": 0.335122, + "f1_weighted": 0.365871 + }, + { + "accuracy": 0.30303, + "f1": 0.28511, + "f1_weighted": 0.311063 + }, + { + "accuracy": 0.282828, + "f1": 0.244481, + "f1_weighted": 0.300105 + }, + { + "accuracy": 0.313131, + "f1": 0.275385, + "f1_weighted": 0.3135 + }, + { + "accuracy": 0.363636, + "f1": 0.356429, + "f1_weighted": 0.383057 + }, + { + "accuracy": 0.383838, + "f1": 0.375005, + "f1_weighted": 0.39453 + } + ], + "main_score": 0.332323, + "hf_subset": "tso_Latn", + "languages": [ + "tso-Latn" + ] + }, + { + "accuracy": 0.314141, + "f1": 0.301031, + "f1_weighted": 0.320813, + "scores_per_experiment": [ + { + "accuracy": 0.30303, + "f1": 0.299218, + "f1_weighted": 0.318376 + }, + { + "accuracy": 0.373737, + "f1": 0.349539, + "f1_weighted": 0.380598 + }, + { + "accuracy": 0.222222, + "f1": 0.208779, + "f1_weighted": 0.227266 + }, + { + "accuracy": 0.333333, + "f1": 0.324723, + "f1_weighted": 0.350348 + }, + { + "accuracy": 0.262626, + "f1": 0.256069, + "f1_weighted": 0.261993 + }, + { + "accuracy": 0.323232, + "f1": 0.310276, + "f1_weighted": 0.32136 + }, + { + "accuracy": 0.333333, + "f1": 0.325837, + "f1_weighted": 0.336215 + }, + { + "accuracy": 0.323232, + "f1": 0.317543, + "f1_weighted": 0.332499 + }, + { + "accuracy": 0.292929, + "f1": 0.279196, + "f1_weighted": 0.299622 + }, + { + "accuracy": 0.373737, + "f1": 0.339132, + "f1_weighted": 0.379855 + } + ], + "main_score": 0.314141, + "hf_subset": "tuk_Latn", + "languages": [ + "tuk-Latn" + ] + }, + { + "accuracy": 0.378788, + "f1": 0.362748, + "f1_weighted": 0.383719, + "scores_per_experiment": [ + { + "accuracy": 0.414141, + "f1": 0.384742, + "f1_weighted": 0.414842 + }, + { + "accuracy": 0.333333, + "f1": 0.346959, + "f1_weighted": 0.353265 + }, + { + "accuracy": 0.353535, + "f1": 0.35097, + "f1_weighted": 0.353944 + }, + { + "accuracy": 0.353535, + "f1": 0.341398, + "f1_weighted": 0.362553 + }, + { + "accuracy": 0.333333, + "f1": 0.272926, + "f1_weighted": 0.313811 + }, + { + "accuracy": 0.343434, + "f1": 0.329474, + "f1_weighted": 0.351191 + }, + { + "accuracy": 0.444444, + "f1": 0.418439, + "f1_weighted": 0.441918 + }, + { + "accuracy": 0.393939, + "f1": 0.402008, + "f1_weighted": 0.404912 + }, + { + "accuracy": 0.424242, + "f1": 0.413703, + "f1_weighted": 0.430524 + }, + { + "accuracy": 0.393939, + "f1": 0.366863, + "f1_weighted": 0.410229 + } + ], + "main_score": 0.378788, + "hf_subset": "tum_Latn", + "languages": [ + "tum-Latn" + ] + }, + { + "accuracy": 0.283838, + "f1": 0.270461, + "f1_weighted": 0.287781, + "scores_per_experiment": [ + { + "accuracy": 0.323232, + "f1": 0.323411, + "f1_weighted": 0.334988 + }, + { + "accuracy": 0.292929, + "f1": 0.282523, + "f1_weighted": 0.301731 + }, + { + "accuracy": 0.262626, + "f1": 0.258123, + "f1_weighted": 0.268352 + }, + { + "accuracy": 0.363636, + "f1": 0.331951, + "f1_weighted": 0.371923 + }, + { + "accuracy": 0.20202, + "f1": 0.185458, + "f1_weighted": 0.210596 + }, + { + "accuracy": 0.252525, + "f1": 0.220903, + "f1_weighted": 0.249654 + }, + { + "accuracy": 0.30303, + "f1": 0.288254, + "f1_weighted": 0.299814 + }, + { + "accuracy": 0.242424, + "f1": 0.225997, + "f1_weighted": 0.244012 + }, + { + "accuracy": 0.30303, + "f1": 0.311836, + "f1_weighted": 0.300331 + }, + { + "accuracy": 0.292929, + "f1": 0.276151, + "f1_weighted": 0.296415 + } + ], + "main_score": 0.283838, + "hf_subset": "tur_Latn", + "languages": [ + "tur-Latn" + ] + }, + { + "accuracy": 0.386869, + "f1": 0.37168, + "f1_weighted": 0.388404, + "scores_per_experiment": [ + { + "accuracy": 0.444444, + "f1": 0.420112, + "f1_weighted": 0.433238 + }, + { + "accuracy": 0.363636, + "f1": 0.333752, + "f1_weighted": 0.361068 + }, + { + "accuracy": 0.333333, + "f1": 0.321279, + "f1_weighted": 0.348253 + }, + { + "accuracy": 0.464646, + "f1": 0.480431, + "f1_weighted": 0.467938 + }, + { + "accuracy": 0.292929, + "f1": 0.280453, + "f1_weighted": 0.300016 + }, + { + "accuracy": 0.363636, + "f1": 0.350771, + "f1_weighted": 0.34795 + }, + { + "accuracy": 0.393939, + "f1": 0.381299, + "f1_weighted": 0.400522 + }, + { + "accuracy": 0.444444, + "f1": 0.421305, + "f1_weighted": 0.457934 + }, + { + "accuracy": 0.353535, + "f1": 0.302871, + "f1_weighted": 0.339012 + }, + { + "accuracy": 0.414141, + "f1": 0.424528, + "f1_weighted": 0.428111 + } + ], + "main_score": 0.386869, + "hf_subset": "twi_Latn", + "languages": [ + "twi-Latn" + ] + }, + { + "accuracy": 0.184848, + "f1": 0.154795, + "f1_weighted": 0.155127, + "scores_per_experiment": [ + { + "accuracy": 0.191919, + "f1": 0.177422, + "f1_weighted": 0.186821 + }, + { + "accuracy": 0.181818, + "f1": 0.138578, + "f1_weighted": 0.139578 + }, + { + "accuracy": 0.121212, + "f1": 0.092686, + "f1_weighted": 0.06861 + }, + { + "accuracy": 0.232323, + "f1": 0.204663, + "f1_weighted": 0.171585 + }, + { + "accuracy": 0.141414, + "f1": 0.127101, + "f1_weighted": 0.118294 + }, + { + "accuracy": 0.242424, + "f1": 0.16696, + "f1_weighted": 0.203197 + }, + { + "accuracy": 0.181818, + "f1": 0.163011, + "f1_weighted": 0.165525 + }, + { + "accuracy": 0.20202, + "f1": 0.191575, + "f1_weighted": 0.186904 + }, + { + "accuracy": 0.181818, + "f1": 0.154924, + "f1_weighted": 0.186719 + }, + { + "accuracy": 0.171717, + "f1": 0.131033, + "f1_weighted": 0.124034 + } + ], + "main_score": 0.184848, + "hf_subset": "tzm_Tfng", + "languages": [ + "tzm-Tfng" + ] + }, + { + "accuracy": 0.20202, + "f1": 0.156742, + "f1_weighted": 0.158931, + "scores_per_experiment": [ + { + "accuracy": 0.282828, + "f1": 0.226213, + "f1_weighted": 0.215192 + }, + { + "accuracy": 0.171717, + "f1": 0.133377, + "f1_weighted": 0.105171 + }, + { + "accuracy": 0.121212, + "f1": 0.112983, + "f1_weighted": 0.117184 + }, + { + "accuracy": 0.292929, + "f1": 0.220638, + "f1_weighted": 0.233652 + }, + { + "accuracy": 0.141414, + "f1": 0.113493, + "f1_weighted": 0.096938 + }, + { + "accuracy": 0.181818, + "f1": 0.14895, + "f1_weighted": 0.160572 + }, + { + "accuracy": 0.181818, + "f1": 0.132316, + "f1_weighted": 0.126952 + }, + { + "accuracy": 0.262626, + "f1": 0.170379, + "f1_weighted": 0.203045 + }, + { + "accuracy": 0.232323, + "f1": 0.196072, + "f1_weighted": 0.223109 + }, + { + "accuracy": 0.151515, + "f1": 0.113004, + "f1_weighted": 0.107498 + } + ], + "main_score": 0.20202, + "hf_subset": "uig_Arab", + "languages": [ + "uig-Arab" + ] + }, + { + "accuracy": 0.225253, + "f1": 0.208027, + "f1_weighted": 0.223231, + "scores_per_experiment": [ + { + "accuracy": 0.282828, + "f1": 0.234382, + "f1_weighted": 0.265239 + }, + { + "accuracy": 0.161616, + "f1": 0.166366, + "f1_weighted": 0.147461 + }, + { + "accuracy": 0.252525, + "f1": 0.252874, + "f1_weighted": 0.255238 + }, + { + "accuracy": 0.262626, + "f1": 0.253304, + "f1_weighted": 0.262104 + }, + { + "accuracy": 0.090909, + "f1": 0.097127, + "f1_weighted": 0.080426 + }, + { + "accuracy": 0.181818, + "f1": 0.147913, + "f1_weighted": 0.186224 + }, + { + "accuracy": 0.313131, + "f1": 0.272066, + "f1_weighted": 0.317744 + }, + { + "accuracy": 0.262626, + "f1": 0.231322, + "f1_weighted": 0.271635 + }, + { + "accuracy": 0.232323, + "f1": 0.236523, + "f1_weighted": 0.231791 + }, + { + "accuracy": 0.212121, + "f1": 0.188391, + "f1_weighted": 0.21445 + } + ], + "main_score": 0.225253, + "hf_subset": "ukr_Cyrl", + "languages": [ + "ukr-Cyrl" + ] + }, + { + "accuracy": 0.311111, + "f1": 0.284285, + "f1_weighted": 0.312693, + "scores_per_experiment": [ + { + "accuracy": 0.383838, + "f1": 0.350245, + "f1_weighted": 0.386075 + }, + { + "accuracy": 0.181818, + "f1": 0.18161, + "f1_weighted": 0.172181 + }, + { + "accuracy": 0.343434, + "f1": 0.332319, + "f1_weighted": 0.33815 + }, + { + "accuracy": 0.373737, + "f1": 0.358583, + "f1_weighted": 0.373534 + }, + { + "accuracy": 0.30303, + "f1": 0.27311, + "f1_weighted": 0.313888 + }, + { + "accuracy": 0.333333, + "f1": 0.273479, + "f1_weighted": 0.33037 + }, + { + "accuracy": 0.242424, + "f1": 0.215894, + "f1_weighted": 0.248695 + }, + { + "accuracy": 0.383838, + "f1": 0.33103, + "f1_weighted": 0.379319 + }, + { + "accuracy": 0.282828, + "f1": 0.267987, + "f1_weighted": 0.302975 + }, + { + "accuracy": 0.282828, + "f1": 0.258591, + "f1_weighted": 0.281748 + } + ], + "main_score": 0.311111, + "hf_subset": "umb_Latn", + "languages": [ + "umb-Latn" + ] + }, + { + "accuracy": 0.184848, + "f1": 0.137435, + "f1_weighted": 0.138457, + "scores_per_experiment": [ + { + "accuracy": 0.232323, + "f1": 0.134558, + "f1_weighted": 0.157269 + }, + { + "accuracy": 0.151515, + "f1": 0.124209, + "f1_weighted": 0.112318 + }, + { + "accuracy": 0.121212, + "f1": 0.102304, + "f1_weighted": 0.078042 + }, + { + "accuracy": 0.262626, + "f1": 0.199348, + "f1_weighted": 0.193884 + }, + { + "accuracy": 0.181818, + "f1": 0.12404, + "f1_weighted": 0.135748 + }, + { + "accuracy": 0.141414, + "f1": 0.098286, + "f1_weighted": 0.111416 + }, + { + "accuracy": 0.191919, + "f1": 0.125013, + "f1_weighted": 0.132639 + }, + { + "accuracy": 0.141414, + "f1": 0.104515, + "f1_weighted": 0.098431 + }, + { + "accuracy": 0.171717, + "f1": 0.136984, + "f1_weighted": 0.131459 + }, + { + "accuracy": 0.252525, + "f1": 0.225098, + "f1_weighted": 0.233363 + } + ], + "main_score": 0.184848, + "hf_subset": "urd_Arab", + "languages": [ + "urd-Arab" + ] + }, + { + "accuracy": 0.277778, + "f1": 0.269329, + "f1_weighted": 0.276915, + "scores_per_experiment": [ + { + "accuracy": 0.333333, + "f1": 0.311562, + "f1_weighted": 0.327746 + }, + { + "accuracy": 0.272727, + "f1": 0.266814, + "f1_weighted": 0.262574 + }, + { + "accuracy": 0.181818, + "f1": 0.159909, + "f1_weighted": 0.16234 + }, + { + "accuracy": 0.313131, + "f1": 0.312614, + "f1_weighted": 0.308419 + }, + { + "accuracy": 0.222222, + "f1": 0.231821, + "f1_weighted": 0.217682 + }, + { + "accuracy": 0.313131, + "f1": 0.314772, + "f1_weighted": 0.312305 + }, + { + "accuracy": 0.30303, + "f1": 0.279399, + "f1_weighted": 0.306506 + }, + { + "accuracy": 0.272727, + "f1": 0.269442, + "f1_weighted": 0.288334 + }, + { + "accuracy": 0.262626, + "f1": 0.254498, + "f1_weighted": 0.26342 + }, + { + "accuracy": 0.30303, + "f1": 0.29246, + "f1_weighted": 0.319826 + } + ], + "main_score": 0.277778, + "hf_subset": "uzn_Latn", + "languages": [ + "uzn-Latn" + ] + }, + { + "accuracy": 0.429293, + "f1": 0.397022, + "f1_weighted": 0.432033, + "scores_per_experiment": [ + { + "accuracy": 0.414141, + "f1": 0.383705, + "f1_weighted": 0.424292 + }, + { + "accuracy": 0.373737, + "f1": 0.335296, + "f1_weighted": 0.379698 + }, + { + "accuracy": 0.373737, + "f1": 0.372149, + "f1_weighted": 0.37884 + }, + { + "accuracy": 0.525253, + "f1": 0.500213, + "f1_weighted": 0.52791 + }, + { + "accuracy": 0.464646, + "f1": 0.407859, + "f1_weighted": 0.470522 + }, + { + "accuracy": 0.363636, + "f1": 0.331596, + "f1_weighted": 0.360403 + }, + { + "accuracy": 0.434343, + "f1": 0.397898, + "f1_weighted": 0.433074 + }, + { + "accuracy": 0.474747, + "f1": 0.439217, + "f1_weighted": 0.469009 + }, + { + "accuracy": 0.40404, + "f1": 0.379566, + "f1_weighted": 0.412553 + }, + { + "accuracy": 0.464646, + "f1": 0.422725, + "f1_weighted": 0.464034 + } + ], + "main_score": 0.429293, + "hf_subset": "vec_Latn", + "languages": [ + "vec-Latn" + ] + }, + { + "accuracy": 0.316162, + "f1": 0.292327, + "f1_weighted": 0.306265, + "scores_per_experiment": [ + { + "accuracy": 0.343434, + "f1": 0.337552, + "f1_weighted": 0.345585 + }, + { + "accuracy": 0.222222, + "f1": 0.231434, + "f1_weighted": 0.206963 + }, + { + "accuracy": 0.252525, + "f1": 0.237201, + "f1_weighted": 0.241146 + }, + { + "accuracy": 0.363636, + "f1": 0.310143, + "f1_weighted": 0.353572 + }, + { + "accuracy": 0.353535, + "f1": 0.333478, + "f1_weighted": 0.354676 + }, + { + "accuracy": 0.323232, + "f1": 0.251976, + "f1_weighted": 0.286587 + }, + { + "accuracy": 0.272727, + "f1": 0.22431, + "f1_weighted": 0.239811 + }, + { + "accuracy": 0.40404, + "f1": 0.39612, + "f1_weighted": 0.406817 + }, + { + "accuracy": 0.323232, + "f1": 0.298764, + "f1_weighted": 0.32601 + }, + { + "accuracy": 0.30303, + "f1": 0.30229, + "f1_weighted": 0.301479 + } + ], + "main_score": 0.316162, + "hf_subset": "vie_Latn", + "languages": [ + "vie-Latn" + ] + }, + { + "accuracy": 0.505051, + "f1": 0.485851, + "f1_weighted": 0.511914, + "scores_per_experiment": [ + { + "accuracy": 0.494949, + "f1": 0.498594, + "f1_weighted": 0.503083 + }, + { + "accuracy": 0.545455, + "f1": 0.507533, + "f1_weighted": 0.54608 + }, + { + "accuracy": 0.464646, + "f1": 0.455917, + "f1_weighted": 0.471144 + }, + { + "accuracy": 0.494949, + "f1": 0.446856, + "f1_weighted": 0.49127 + }, + { + "accuracy": 0.474747, + "f1": 0.460859, + "f1_weighted": 0.488845 + }, + { + "accuracy": 0.565657, + "f1": 0.542783, + "f1_weighted": 0.57206 + }, + { + "accuracy": 0.505051, + "f1": 0.474559, + "f1_weighted": 0.508291 + }, + { + "accuracy": 0.515152, + "f1": 0.492683, + "f1_weighted": 0.528482 + }, + { + "accuracy": 0.474747, + "f1": 0.451791, + "f1_weighted": 0.483225 + }, + { + "accuracy": 0.515152, + "f1": 0.526936, + "f1_weighted": 0.526659 + } + ], + "main_score": 0.505051, + "hf_subset": "war_Latn", + "languages": [ + "war-Latn" + ] + }, + { + "accuracy": 0.331313, + "f1": 0.316415, + "f1_weighted": 0.333908, + "scores_per_experiment": [ + { + "accuracy": 0.434343, + "f1": 0.407252, + "f1_weighted": 0.445993 + }, + { + "accuracy": 0.262626, + "f1": 0.271822, + "f1_weighted": 0.238357 + }, + { + "accuracy": 0.232323, + "f1": 0.223572, + "f1_weighted": 0.255634 + }, + { + "accuracy": 0.353535, + "f1": 0.341775, + "f1_weighted": 0.357785 + }, + { + "accuracy": 0.333333, + "f1": 0.312852, + "f1_weighted": 0.341119 + }, + { + "accuracy": 0.292929, + "f1": 0.290974, + "f1_weighted": 0.302885 + }, + { + "accuracy": 0.282828, + "f1": 0.272335, + "f1_weighted": 0.275667 + }, + { + "accuracy": 0.40404, + "f1": 0.367465, + "f1_weighted": 0.400357 + }, + { + "accuracy": 0.363636, + "f1": 0.345969, + "f1_weighted": 0.368636 + }, + { + "accuracy": 0.353535, + "f1": 0.330137, + "f1_weighted": 0.352647 + } + ], + "main_score": 0.331313, + "hf_subset": "wol_Latn", + "languages": [ + "wol-Latn" + ] + }, + { + "accuracy": 0.325253, + "f1": 0.31671, + "f1_weighted": 0.332573, + "scores_per_experiment": [ + { + "accuracy": 0.313131, + "f1": 0.316538, + "f1_weighted": 0.321158 + }, + { + "accuracy": 0.272727, + "f1": 0.274374, + "f1_weighted": 0.286514 + }, + { + "accuracy": 0.323232, + "f1": 0.321031, + "f1_weighted": 0.328099 + }, + { + "accuracy": 0.363636, + "f1": 0.351732, + "f1_weighted": 0.372093 + }, + { + "accuracy": 0.333333, + "f1": 0.317874, + "f1_weighted": 0.336835 + }, + { + "accuracy": 0.363636, + "f1": 0.347493, + "f1_weighted": 0.363491 + }, + { + "accuracy": 0.292929, + "f1": 0.285702, + "f1_weighted": 0.303218 + }, + { + "accuracy": 0.282828, + "f1": 0.275879, + "f1_weighted": 0.290359 + }, + { + "accuracy": 0.373737, + "f1": 0.352187, + "f1_weighted": 0.381683 + }, + { + "accuracy": 0.333333, + "f1": 0.324292, + "f1_weighted": 0.342277 + } + ], + "main_score": 0.325253, + "hf_subset": "xho_Latn", + "languages": [ + "xho-Latn" + ] + }, + { + "accuracy": 0.209091, + "f1": 0.171574, + "f1_weighted": 0.184724, + "scores_per_experiment": [ + { + "accuracy": 0.242424, + "f1": 0.206643, + "f1_weighted": 0.218963 + }, + { + "accuracy": 0.191919, + "f1": 0.134365, + "f1_weighted": 0.153997 + }, + { + "accuracy": 0.090909, + "f1": 0.094782, + "f1_weighted": 0.075906 + }, + { + "accuracy": 0.252525, + "f1": 0.242537, + "f1_weighted": 0.250642 + }, + { + "accuracy": 0.151515, + "f1": 0.14246, + "f1_weighted": 0.121909 + }, + { + "accuracy": 0.171717, + "f1": 0.125812, + "f1_weighted": 0.139831 + }, + { + "accuracy": 0.212121, + "f1": 0.178336, + "f1_weighted": 0.194938 + }, + { + "accuracy": 0.30303, + "f1": 0.190981, + "f1_weighted": 0.253316 + }, + { + "accuracy": 0.272727, + "f1": 0.23819, + "f1_weighted": 0.2693 + }, + { + "accuracy": 0.20202, + "f1": 0.161637, + "f1_weighted": 0.168437 + } + ], + "main_score": 0.209091, + "hf_subset": "ydd_Hebr", + "languages": [ + "ydd-Hebr" + ] + }, + { + "accuracy": 0.278788, + "f1": 0.267071, + "f1_weighted": 0.283631, + "scores_per_experiment": [ + { + "accuracy": 0.313131, + "f1": 0.310831, + "f1_weighted": 0.294262 + }, + { + "accuracy": 0.262626, + "f1": 0.245942, + "f1_weighted": 0.273799 + }, + { + "accuracy": 0.252525, + "f1": 0.245059, + "f1_weighted": 0.259954 + }, + { + "accuracy": 0.272727, + "f1": 0.266029, + "f1_weighted": 0.286228 + }, + { + "accuracy": 0.30303, + "f1": 0.279529, + "f1_weighted": 0.286848 + }, + { + "accuracy": 0.212121, + "f1": 0.212182, + "f1_weighted": 0.23825 + }, + { + "accuracy": 0.313131, + "f1": 0.280799, + "f1_weighted": 0.296582 + }, + { + "accuracy": 0.323232, + "f1": 0.294021, + "f1_weighted": 0.32242 + }, + { + "accuracy": 0.272727, + "f1": 0.275968, + "f1_weighted": 0.294388 + }, + { + "accuracy": 0.262626, + "f1": 0.260352, + "f1_weighted": 0.283585 + } + ], + "main_score": 0.278788, + "hf_subset": "yor_Latn", + "languages": [ + "yor-Latn" + ] + }, + { + "accuracy": 0.183838, + "f1": 0.138671, + "f1_weighted": 0.133924, + "scores_per_experiment": [ + { + "accuracy": 0.262626, + "f1": 0.195144, + "f1_weighted": 0.210201 + }, + { + "accuracy": 0.181818, + "f1": 0.090672, + "f1_weighted": 0.080814 + }, + { + "accuracy": 0.151515, + "f1": 0.116801, + "f1_weighted": 0.102496 + }, + { + "accuracy": 0.232323, + "f1": 0.179357, + "f1_weighted": 0.187012 + }, + { + "accuracy": 0.151515, + "f1": 0.10165, + "f1_weighted": 0.073571 + }, + { + "accuracy": 0.141414, + "f1": 0.128555, + "f1_weighted": 0.112426 + }, + { + "accuracy": 0.212121, + "f1": 0.166861, + "f1_weighted": 0.148556 + }, + { + "accuracy": 0.161616, + "f1": 0.09644, + "f1_weighted": 0.125298 + }, + { + "accuracy": 0.222222, + "f1": 0.201995, + "f1_weighted": 0.207478 + }, + { + "accuracy": 0.121212, + "f1": 0.109237, + "f1_weighted": 0.091391 + } + ], + "main_score": 0.183838, + "hf_subset": "yue_Hant", + "languages": [ + "yue-Hant" + ] + }, + { + "accuracy": 0.186869, + "f1": 0.128149, + "f1_weighted": 0.133107, + "scores_per_experiment": [ + { + "accuracy": 0.222222, + "f1": 0.153783, + "f1_weighted": 0.165435 + }, + { + "accuracy": 0.171717, + "f1": 0.111491, + "f1_weighted": 0.098733 + }, + { + "accuracy": 0.141414, + "f1": 0.097493, + "f1_weighted": 0.104466 + }, + { + "accuracy": 0.313131, + "f1": 0.267512, + "f1_weighted": 0.297646 + }, + { + "accuracy": 0.151515, + "f1": 0.077264, + "f1_weighted": 0.090249 + }, + { + "accuracy": 0.222222, + "f1": 0.150893, + "f1_weighted": 0.148232 + }, + { + "accuracy": 0.191919, + "f1": 0.109888, + "f1_weighted": 0.121524 + }, + { + "accuracy": 0.151515, + "f1": 0.072562, + "f1_weighted": 0.065095 + }, + { + "accuracy": 0.151515, + "f1": 0.146766, + "f1_weighted": 0.146961 + }, + { + "accuracy": 0.151515, + "f1": 0.093838, + "f1_weighted": 0.092728 + } + ], + "main_score": 0.186869, + "hf_subset": "zho_Hant", + "languages": [ + "zho-Hant" + ] + }, + { + "accuracy": 0.408081, + "f1": 0.385412, + "f1_weighted": 0.413133, + "scores_per_experiment": [ + { + "accuracy": 0.444444, + "f1": 0.4378, + "f1_weighted": 0.445915 + }, + { + "accuracy": 0.434343, + "f1": 0.399178, + "f1_weighted": 0.424506 + }, + { + "accuracy": 0.373737, + "f1": 0.341997, + "f1_weighted": 0.378116 + }, + { + "accuracy": 0.484848, + "f1": 0.460851, + "f1_weighted": 0.48498 + }, + { + "accuracy": 0.383838, + "f1": 0.370928, + "f1_weighted": 0.398084 + }, + { + "accuracy": 0.414141, + "f1": 0.414286, + "f1_weighted": 0.427994 + }, + { + "accuracy": 0.363636, + "f1": 0.313342, + "f1_weighted": 0.380129 + }, + { + "accuracy": 0.414141, + "f1": 0.407366, + "f1_weighted": 0.427356 + }, + { + "accuracy": 0.393939, + "f1": 0.341082, + "f1_weighted": 0.382505 + }, + { + "accuracy": 0.373737, + "f1": 0.367295, + "f1_weighted": 0.381741 + } + ], + "main_score": 0.408081, + "hf_subset": "zsm_Latn", + "languages": [ + "zsm-Latn" + ] + }, + { + "accuracy": 0.287879, + "f1": 0.276308, + "f1_weighted": 0.289062, + "scores_per_experiment": [ + { + "accuracy": 0.262626, + "f1": 0.261074, + "f1_weighted": 0.265682 + }, + { + "accuracy": 0.20202, + "f1": 0.193197, + "f1_weighted": 0.196633 + }, + { + "accuracy": 0.272727, + "f1": 0.258905, + "f1_weighted": 0.273049 + }, + { + "accuracy": 0.262626, + "f1": 0.251718, + "f1_weighted": 0.262466 + }, + { + "accuracy": 0.373737, + "f1": 0.369848, + "f1_weighted": 0.384148 + }, + { + "accuracy": 0.282828, + "f1": 0.271764, + "f1_weighted": 0.277247 + }, + { + "accuracy": 0.252525, + "f1": 0.223471, + "f1_weighted": 0.256067 + }, + { + "accuracy": 0.343434, + "f1": 0.323852, + "f1_weighted": 0.335575 + }, + { + "accuracy": 0.282828, + "f1": 0.275922, + "f1_weighted": 0.284773 + }, + { + "accuracy": 0.343434, + "f1": 0.333333, + "f1_weighted": 0.354978 + } + ], + "main_score": 0.287879, + "hf_subset": "zul_Latn", + "languages": [ + "zul-Latn" + ] + } + ], + "test": [ + { + "accuracy": 0.348529, + "f1": 0.336393, + "f1_weighted": 0.350925, + "scores_per_experiment": [ + { + "accuracy": 0.367647, + "f1": 0.373127, + "f1_weighted": 0.381099 + }, + { + "accuracy": 0.289216, + "f1": 0.276749, + "f1_weighted": 0.293195 + }, + { + "accuracy": 0.348039, + "f1": 0.341672, + "f1_weighted": 0.349976 + }, + { + "accuracy": 0.323529, + "f1": 0.314917, + "f1_weighted": 0.327319 + }, + { + "accuracy": 0.333333, + "f1": 0.324527, + "f1_weighted": 0.332664 + }, + { + "accuracy": 0.313725, + "f1": 0.304441, + "f1_weighted": 0.318199 + }, + { + "accuracy": 0.367647, + "f1": 0.337487, + "f1_weighted": 0.366246 + }, + { + "accuracy": 0.362745, + "f1": 0.349315, + "f1_weighted": 0.356631 + }, + { + "accuracy": 0.357843, + "f1": 0.335988, + "f1_weighted": 0.361775 + }, + { + "accuracy": 0.421569, + "f1": 0.405712, + "f1_weighted": 0.422149 + } + ], + "main_score": 0.348529, + "hf_subset": "ace_Latn", + "languages": [ + "ace-Latn" + ] + }, + { + "accuracy": 0.183333, + "f1": 0.132039, + "f1_weighted": 0.129775, + "scores_per_experiment": [ + { + "accuracy": 0.245098, + "f1": 0.159242, + "f1_weighted": 0.176112 + }, + { + "accuracy": 0.166667, + "f1": 0.132218, + "f1_weighted": 0.129733 + }, + { + "accuracy": 0.147059, + "f1": 0.126653, + "f1_weighted": 0.100129 + }, + { + "accuracy": 0.25, + "f1": 0.168805, + "f1_weighted": 0.169214 + }, + { + "accuracy": 0.191176, + "f1": 0.13589, + "f1_weighted": 0.132226 + }, + { + "accuracy": 0.132353, + "f1": 0.102868, + "f1_weighted": 0.09827 + }, + { + "accuracy": 0.215686, + "f1": 0.147797, + "f1_weighted": 0.151951 + }, + { + "accuracy": 0.166667, + "f1": 0.135731, + "f1_weighted": 0.133668 + }, + { + "accuracy": 0.142157, + "f1": 0.093901, + "f1_weighted": 0.08763 + }, + { + "accuracy": 0.176471, + "f1": 0.117284, + "f1_weighted": 0.118814 + } + ], + "main_score": 0.183333, + "hf_subset": "acm_Arab", + "languages": [ + "acm-Arab" + ] + }, + { + "accuracy": 0.182843, + "f1": 0.125208, + "f1_weighted": 0.125363, + "scores_per_experiment": [ + { + "accuracy": 0.220588, + "f1": 0.130464, + "f1_weighted": 0.138456 + }, + { + "accuracy": 0.191176, + "f1": 0.116152, + "f1_weighted": 0.124288 + }, + { + "accuracy": 0.147059, + "f1": 0.118052, + "f1_weighted": 0.093621 + }, + { + "accuracy": 0.220588, + "f1": 0.121203, + "f1_weighted": 0.123717 + }, + { + "accuracy": 0.20098, + "f1": 0.138555, + "f1_weighted": 0.158102 + }, + { + "accuracy": 0.137255, + "f1": 0.123129, + "f1_weighted": 0.108685 + }, + { + "accuracy": 0.22549, + "f1": 0.146056, + "f1_weighted": 0.15997 + }, + { + "accuracy": 0.151961, + "f1": 0.123494, + "f1_weighted": 0.121331 + }, + { + "accuracy": 0.151961, + "f1": 0.108026, + "f1_weighted": 0.096335 + }, + { + "accuracy": 0.181373, + "f1": 0.126953, + "f1_weighted": 0.12912 + } + ], + "main_score": 0.182843, + "hf_subset": "acq_Arab", + "languages": [ + "acq-Arab" + ] + }, + { + "accuracy": 0.180392, + "f1": 0.128282, + "f1_weighted": 0.129836, + "scores_per_experiment": [ + { + "accuracy": 0.22549, + "f1": 0.131325, + "f1_weighted": 0.148058 + }, + { + "accuracy": 0.181373, + "f1": 0.111962, + "f1_weighted": 0.119704 + }, + { + "accuracy": 0.156863, + "f1": 0.13689, + "f1_weighted": 0.109696 + }, + { + "accuracy": 0.230392, + "f1": 0.151293, + "f1_weighted": 0.156271 + }, + { + "accuracy": 0.20098, + "f1": 0.160869, + "f1_weighted": 0.170784 + }, + { + "accuracy": 0.137255, + "f1": 0.111667, + "f1_weighted": 0.116422 + }, + { + "accuracy": 0.215686, + "f1": 0.150411, + "f1_weighted": 0.159602 + }, + { + "accuracy": 0.147059, + "f1": 0.117955, + "f1_weighted": 0.113409 + }, + { + "accuracy": 0.142157, + "f1": 0.090619, + "f1_weighted": 0.084952 + }, + { + "accuracy": 0.166667, + "f1": 0.119832, + "f1_weighted": 0.119462 + } + ], + "main_score": 0.180392, + "hf_subset": "aeb_Arab", + "languages": [ + "aeb-Arab" + ] + }, + { + "accuracy": 0.377451, + "f1": 0.371587, + "f1_weighted": 0.37443, + "scores_per_experiment": [ + { + "accuracy": 0.421569, + "f1": 0.427962, + "f1_weighted": 0.435541 + }, + { + "accuracy": 0.377451, + "f1": 0.364104, + "f1_weighted": 0.375021 + }, + { + "accuracy": 0.367647, + "f1": 0.369462, + "f1_weighted": 0.373073 + }, + { + "accuracy": 0.426471, + "f1": 0.411047, + "f1_weighted": 0.414696 + }, + { + "accuracy": 0.377451, + "f1": 0.37996, + "f1_weighted": 0.377547 + }, + { + "accuracy": 0.372549, + "f1": 0.346217, + "f1_weighted": 0.36335 + }, + { + "accuracy": 0.333333, + "f1": 0.335064, + "f1_weighted": 0.325164 + }, + { + "accuracy": 0.362745, + "f1": 0.373295, + "f1_weighted": 0.363865 + }, + { + "accuracy": 0.348039, + "f1": 0.324783, + "f1_weighted": 0.324438 + }, + { + "accuracy": 0.387255, + "f1": 0.383974, + "f1_weighted": 0.391605 + } + ], + "main_score": 0.377451, + "hf_subset": "afr_Latn", + "languages": [ + "afr-Latn" + ] + }, + { + "accuracy": 0.165686, + "f1": 0.107458, + "f1_weighted": 0.1149, + "scores_per_experiment": [ + { + "accuracy": 0.205882, + "f1": 0.1206, + "f1_weighted": 0.132251 + }, + { + "accuracy": 0.235294, + "f1": 0.123214, + "f1_weighted": 0.154693 + }, + { + "accuracy": 0.122549, + "f1": 0.092393, + "f1_weighted": 0.072246 + }, + { + "accuracy": 0.210784, + "f1": 0.112143, + "f1_weighted": 0.117235 + }, + { + "accuracy": 0.137255, + "f1": 0.083178, + "f1_weighted": 0.102861 + }, + { + "accuracy": 0.171569, + "f1": 0.164927, + "f1_weighted": 0.154347 + }, + { + "accuracy": 0.147059, + "f1": 0.101691, + "f1_weighted": 0.109486 + }, + { + "accuracy": 0.137255, + "f1": 0.086018, + "f1_weighted": 0.089357 + }, + { + "accuracy": 0.161765, + "f1": 0.10332, + "f1_weighted": 0.122005 + }, + { + "accuracy": 0.127451, + "f1": 0.0871, + "f1_weighted": 0.094525 + } + ], + "main_score": 0.165686, + "hf_subset": "ajp_Arab", + "languages": [ + "ajp-Arab" + ] + }, + { + "accuracy": 0.395588, + "f1": 0.387569, + "f1_weighted": 0.398848, + "scores_per_experiment": [ + { + "accuracy": 0.392157, + "f1": 0.369592, + "f1_weighted": 0.388528 + }, + { + "accuracy": 0.357843, + "f1": 0.35276, + "f1_weighted": 0.353334 + }, + { + "accuracy": 0.377451, + "f1": 0.382125, + "f1_weighted": 0.3952 + }, + { + "accuracy": 0.416667, + "f1": 0.409894, + "f1_weighted": 0.41673 + }, + { + "accuracy": 0.411765, + "f1": 0.418449, + "f1_weighted": 0.418161 + }, + { + "accuracy": 0.338235, + "f1": 0.346232, + "f1_weighted": 0.344202 + }, + { + "accuracy": 0.372549, + "f1": 0.367894, + "f1_weighted": 0.370422 + }, + { + "accuracy": 0.406863, + "f1": 0.378888, + "f1_weighted": 0.413218 + }, + { + "accuracy": 0.426471, + "f1": 0.394131, + "f1_weighted": 0.418972 + }, + { + "accuracy": 0.455882, + "f1": 0.455724, + "f1_weighted": 0.469717 + } + ], + "main_score": 0.395588, + "hf_subset": "aka_Latn", + "languages": [ + "aka-Latn" + ] + }, + { + "accuracy": 0.352941, + "f1": 0.341107, + "f1_weighted": 0.351628, + "scores_per_experiment": [ + { + "accuracy": 0.387255, + "f1": 0.382561, + "f1_weighted": 0.394049 + }, + { + "accuracy": 0.264706, + "f1": 0.24717, + "f1_weighted": 0.252301 + }, + { + "accuracy": 0.421569, + "f1": 0.415098, + "f1_weighted": 0.431945 + }, + { + "accuracy": 0.377451, + "f1": 0.368989, + "f1_weighted": 0.378925 + }, + { + "accuracy": 0.279412, + "f1": 0.280149, + "f1_weighted": 0.282029 + }, + { + "accuracy": 0.333333, + "f1": 0.313821, + "f1_weighted": 0.333759 + }, + { + "accuracy": 0.377451, + "f1": 0.366973, + "f1_weighted": 0.369026 + }, + { + "accuracy": 0.382353, + "f1": 0.385304, + "f1_weighted": 0.380765 + }, + { + "accuracy": 0.352941, + "f1": 0.309578, + "f1_weighted": 0.339006 + }, + { + "accuracy": 0.352941, + "f1": 0.34143, + "f1_weighted": 0.354477 + } + ], + "main_score": 0.352941, + "hf_subset": "als_Latn", + "languages": [ + "als-Latn" + ] + }, + { + "accuracy": 0.182843, + "f1": 0.110462, + "f1_weighted": 0.125249, + "scores_per_experiment": [ + { + "accuracy": 0.235294, + "f1": 0.128615, + "f1_weighted": 0.164593 + }, + { + "accuracy": 0.151961, + "f1": 0.094939, + "f1_weighted": 0.096663 + }, + { + "accuracy": 0.112745, + "f1": 0.095741, + "f1_weighted": 0.087678 + }, + { + "accuracy": 0.205882, + "f1": 0.102267, + "f1_weighted": 0.120685 + }, + { + "accuracy": 0.171569, + "f1": 0.111835, + "f1_weighted": 0.124865 + }, + { + "accuracy": 0.27451, + "f1": 0.149409, + "f1_weighted": 0.179098 + }, + { + "accuracy": 0.166667, + "f1": 0.106962, + "f1_weighted": 0.109565 + }, + { + "accuracy": 0.235294, + "f1": 0.14669, + "f1_weighted": 0.189721 + }, + { + "accuracy": 0.132353, + "f1": 0.075685, + "f1_weighted": 0.073971 + }, + { + "accuracy": 0.142157, + "f1": 0.092471, + "f1_weighted": 0.105653 + } + ], + "main_score": 0.182843, + "hf_subset": "amh_Ethi", + "languages": [ + "amh-Ethi" + ] + }, + { + "accuracy": 0.166667, + "f1": 0.116588, + "f1_weighted": 0.114551, + "scores_per_experiment": [ + { + "accuracy": 0.205882, + "f1": 0.107154, + "f1_weighted": 0.122547 + }, + { + "accuracy": 0.215686, + "f1": 0.127125, + "f1_weighted": 0.142449 + }, + { + "accuracy": 0.132353, + "f1": 0.099432, + "f1_weighted": 0.079403 + }, + { + "accuracy": 0.215686, + "f1": 0.137795, + "f1_weighted": 0.131964 + }, + { + "accuracy": 0.147059, + "f1": 0.117903, + "f1_weighted": 0.111926 + }, + { + "accuracy": 0.161765, + "f1": 0.149758, + "f1_weighted": 0.1417 + }, + { + "accuracy": 0.147059, + "f1": 0.112249, + "f1_weighted": 0.11235 + }, + { + "accuracy": 0.147059, + "f1": 0.116202, + "f1_weighted": 0.105494 + }, + { + "accuracy": 0.147059, + "f1": 0.095691, + "f1_weighted": 0.097982 + }, + { + "accuracy": 0.147059, + "f1": 0.102574, + "f1_weighted": 0.099692 + } + ], + "main_score": 0.166667, + "hf_subset": "apc_Arab", + "languages": [ + "apc-Arab" + ] + }, + { + "accuracy": 0.268627, + "f1": 0.253859, + "f1_weighted": 0.266143, + "scores_per_experiment": [ + { + "accuracy": 0.333333, + "f1": 0.314867, + "f1_weighted": 0.33765 + }, + { + "accuracy": 0.259804, + "f1": 0.248258, + "f1_weighted": 0.263838 + }, + { + "accuracy": 0.254902, + "f1": 0.252594, + "f1_weighted": 0.266651 + }, + { + "accuracy": 0.313725, + "f1": 0.309231, + "f1_weighted": 0.317072 + }, + { + "accuracy": 0.215686, + "f1": 0.190914, + "f1_weighted": 0.21832 + }, + { + "accuracy": 0.264706, + "f1": 0.246135, + "f1_weighted": 0.264151 + }, + { + "accuracy": 0.25, + "f1": 0.208372, + "f1_weighted": 0.227684 + }, + { + "accuracy": 0.29902, + "f1": 0.295196, + "f1_weighted": 0.292992 + }, + { + "accuracy": 0.235294, + "f1": 0.204987, + "f1_weighted": 0.215616 + }, + { + "accuracy": 0.259804, + "f1": 0.268036, + "f1_weighted": 0.257455 + } + ], + "main_score": 0.268627, + "hf_subset": "arb_Latn", + "languages": [ + "arb-Latn" + ] + }, + { + "accuracy": 0.183333, + "f1": 0.123014, + "f1_weighted": 0.128005, + "scores_per_experiment": [ + { + "accuracy": 0.235294, + "f1": 0.142464, + "f1_weighted": 0.157782 + }, + { + "accuracy": 0.176471, + "f1": 0.11177, + "f1_weighted": 0.119512 + }, + { + "accuracy": 0.132353, + "f1": 0.107243, + "f1_weighted": 0.085955 + }, + { + "accuracy": 0.230392, + "f1": 0.135611, + "f1_weighted": 0.146515 + }, + { + "accuracy": 0.196078, + "f1": 0.1356, + "f1_weighted": 0.147774 + }, + { + "accuracy": 0.132353, + "f1": 0.104668, + "f1_weighted": 0.100861 + }, + { + "accuracy": 0.215686, + "f1": 0.119396, + "f1_weighted": 0.136431 + }, + { + "accuracy": 0.171569, + "f1": 0.128395, + "f1_weighted": 0.136629 + }, + { + "accuracy": 0.171569, + "f1": 0.125404, + "f1_weighted": 0.126739 + }, + { + "accuracy": 0.171569, + "f1": 0.119588, + "f1_weighted": 0.121851 + } + ], + "main_score": 0.183333, + "hf_subset": "ars_Arab", + "languages": [ + "ars-Arab" + ] + }, + { + "accuracy": 0.184804, + "f1": 0.128298, + "f1_weighted": 0.133703, + "scores_per_experiment": [ + { + "accuracy": 0.220588, + "f1": 0.136169, + "f1_weighted": 0.147226 + }, + { + "accuracy": 0.151961, + "f1": 0.099738, + "f1_weighted": 0.084418 + }, + { + "accuracy": 0.151961, + "f1": 0.118659, + "f1_weighted": 0.114395 + }, + { + "accuracy": 0.235294, + "f1": 0.16476, + "f1_weighted": 0.172623 + }, + { + "accuracy": 0.142157, + "f1": 0.120311, + "f1_weighted": 0.119763 + }, + { + "accuracy": 0.171569, + "f1": 0.155998, + "f1_weighted": 0.16043 + }, + { + "accuracy": 0.22549, + "f1": 0.147214, + "f1_weighted": 0.165657 + }, + { + "accuracy": 0.230392, + "f1": 0.138467, + "f1_weighted": 0.155234 + }, + { + "accuracy": 0.151961, + "f1": 0.095284, + "f1_weighted": 0.10555 + }, + { + "accuracy": 0.166667, + "f1": 0.106379, + "f1_weighted": 0.111735 + } + ], + "main_score": 0.184804, + "hf_subset": "ary_Arab", + "languages": [ + "ary-Arab" + ] + }, + { + "accuracy": 0.172059, + "f1": 0.118391, + "f1_weighted": 0.123539, + "scores_per_experiment": [ + { + "accuracy": 0.215686, + "f1": 0.11194, + "f1_weighted": 0.138704 + }, + { + "accuracy": 0.205882, + "f1": 0.127576, + "f1_weighted": 0.134827 + }, + { + "accuracy": 0.127451, + "f1": 0.10543, + "f1_weighted": 0.092572 + }, + { + "accuracy": 0.20098, + "f1": 0.086373, + "f1_weighted": 0.092782 + }, + { + "accuracy": 0.166667, + "f1": 0.150274, + "f1_weighted": 0.164115 + }, + { + "accuracy": 0.137255, + "f1": 0.114464, + "f1_weighted": 0.091243 + }, + { + "accuracy": 0.186275, + "f1": 0.142882, + "f1_weighted": 0.146848 + }, + { + "accuracy": 0.151961, + "f1": 0.114556, + "f1_weighted": 0.115975 + }, + { + "accuracy": 0.166667, + "f1": 0.119877, + "f1_weighted": 0.140631 + }, + { + "accuracy": 0.161765, + "f1": 0.110539, + "f1_weighted": 0.117696 + } + ], + "main_score": 0.172059, + "hf_subset": "arz_Arab", + "languages": [ + "arz-Arab" + ] + }, + { + "accuracy": 0.160784, + "f1": 0.127171, + "f1_weighted": 0.133393, + "scores_per_experiment": [ + { + "accuracy": 0.161765, + "f1": 0.126346, + "f1_weighted": 0.12986 + }, + { + "accuracy": 0.151961, + "f1": 0.144929, + "f1_weighted": 0.147964 + }, + { + "accuracy": 0.137255, + "f1": 0.108957, + "f1_weighted": 0.088504 + }, + { + "accuracy": 0.122549, + "f1": 0.074588, + "f1_weighted": 0.071407 + }, + { + "accuracy": 0.147059, + "f1": 0.129732, + "f1_weighted": 0.124235 + }, + { + "accuracy": 0.191176, + "f1": 0.156167, + "f1_weighted": 0.170338 + }, + { + "accuracy": 0.181373, + "f1": 0.147294, + "f1_weighted": 0.146231 + }, + { + "accuracy": 0.240196, + "f1": 0.158731, + "f1_weighted": 0.203684 + }, + { + "accuracy": 0.147059, + "f1": 0.118831, + "f1_weighted": 0.139753 + }, + { + "accuracy": 0.127451, + "f1": 0.106136, + "f1_weighted": 0.111953 + } + ], + "main_score": 0.160784, + "hf_subset": "asm_Beng", + "languages": [ + "asm-Beng" + ] + }, + { + "accuracy": 0.488725, + "f1": 0.469715, + "f1_weighted": 0.486222, + "scores_per_experiment": [ + { + "accuracy": 0.529412, + "f1": 0.512066, + "f1_weighted": 0.52852 + }, + { + "accuracy": 0.514706, + "f1": 0.488602, + "f1_weighted": 0.517467 + }, + { + "accuracy": 0.495098, + "f1": 0.478038, + "f1_weighted": 0.488519 + }, + { + "accuracy": 0.490196, + "f1": 0.476465, + "f1_weighted": 0.497422 + }, + { + "accuracy": 0.514706, + "f1": 0.490918, + "f1_weighted": 0.514433 + }, + { + "accuracy": 0.504902, + "f1": 0.489131, + "f1_weighted": 0.504822 + }, + { + "accuracy": 0.509804, + "f1": 0.476781, + "f1_weighted": 0.498387 + }, + { + "accuracy": 0.45098, + "f1": 0.437195, + "f1_weighted": 0.447011 + }, + { + "accuracy": 0.446078, + "f1": 0.426041, + "f1_weighted": 0.4262 + }, + { + "accuracy": 0.431373, + "f1": 0.421914, + "f1_weighted": 0.439439 + } + ], + "main_score": 0.488725, + "hf_subset": "ast_Latn", + "languages": [ + "ast-Latn" + ] + }, + { + "accuracy": 0.17549, + "f1": 0.132511, + "f1_weighted": 0.13385, + "scores_per_experiment": [ + { + "accuracy": 0.186275, + "f1": 0.13272, + "f1_weighted": 0.140686 + }, + { + "accuracy": 0.196078, + "f1": 0.137154, + "f1_weighted": 0.144199 + }, + { + "accuracy": 0.147059, + "f1": 0.138672, + "f1_weighted": 0.121691 + }, + { + "accuracy": 0.156863, + "f1": 0.11037, + "f1_weighted": 0.105138 + }, + { + "accuracy": 0.156863, + "f1": 0.126048, + "f1_weighted": 0.130746 + }, + { + "accuracy": 0.176471, + "f1": 0.159883, + "f1_weighted": 0.152165 + }, + { + "accuracy": 0.210784, + "f1": 0.129725, + "f1_weighted": 0.142457 + }, + { + "accuracy": 0.215686, + "f1": 0.181155, + "f1_weighted": 0.191925 + }, + { + "accuracy": 0.142157, + "f1": 0.103924, + "f1_weighted": 0.096292 + }, + { + "accuracy": 0.166667, + "f1": 0.105465, + "f1_weighted": 0.113204 + } + ], + "main_score": 0.17549, + "hf_subset": "awa_Deva", + "languages": [ + "awa-Deva" + ] + }, + { + "accuracy": 0.276471, + "f1": 0.266339, + "f1_weighted": 0.277134, + "scores_per_experiment": [ + { + "accuracy": 0.308824, + "f1": 0.31243, + "f1_weighted": 0.323424 + }, + { + "accuracy": 0.289216, + "f1": 0.280127, + "f1_weighted": 0.298025 + }, + { + "accuracy": 0.254902, + "f1": 0.229502, + "f1_weighted": 0.244908 + }, + { + "accuracy": 0.308824, + "f1": 0.302367, + "f1_weighted": 0.309749 + }, + { + "accuracy": 0.264706, + "f1": 0.266021, + "f1_weighted": 0.268834 + }, + { + "accuracy": 0.25, + "f1": 0.239047, + "f1_weighted": 0.252891 + }, + { + "accuracy": 0.25, + "f1": 0.234419, + "f1_weighted": 0.245305 + }, + { + "accuracy": 0.254902, + "f1": 0.254834, + "f1_weighted": 0.25092 + }, + { + "accuracy": 0.308824, + "f1": 0.268947, + "f1_weighted": 0.29469 + }, + { + "accuracy": 0.27451, + "f1": 0.275693, + "f1_weighted": 0.282595 + } + ], + "main_score": 0.276471, + "hf_subset": "ayr_Latn", + "languages": [ + "ayr-Latn" + ] + }, + { + "accuracy": 0.235784, + "f1": 0.146195, + "f1_weighted": 0.176216, + "scores_per_experiment": [ + { + "accuracy": 0.27451, + "f1": 0.162923, + "f1_weighted": 0.223872 + }, + { + "accuracy": 0.25, + "f1": 0.113768, + "f1_weighted": 0.156937 + }, + { + "accuracy": 0.147059, + "f1": 0.152392, + "f1_weighted": 0.141288 + }, + { + "accuracy": 0.235294, + "f1": 0.119146, + "f1_weighted": 0.144099 + }, + { + "accuracy": 0.269608, + "f1": 0.191993, + "f1_weighted": 0.229213 + }, + { + "accuracy": 0.220588, + "f1": 0.160091, + "f1_weighted": 0.204423 + }, + { + "accuracy": 0.259804, + "f1": 0.170329, + "f1_weighted": 0.17772 + }, + { + "accuracy": 0.25, + "f1": 0.153496, + "f1_weighted": 0.201893 + }, + { + "accuracy": 0.240196, + "f1": 0.117374, + "f1_weighted": 0.154365 + }, + { + "accuracy": 0.210784, + "f1": 0.12044, + "f1_weighted": 0.128353 + } + ], + "main_score": 0.235784, + "hf_subset": "azb_Arab", + "languages": [ + "azb-Arab" + ] + }, + { + "accuracy": 0.309314, + "f1": 0.295219, + "f1_weighted": 0.313506, + "scores_per_experiment": [ + { + "accuracy": 0.343137, + "f1": 0.333759, + "f1_weighted": 0.347619 + }, + { + "accuracy": 0.333333, + "f1": 0.330623, + "f1_weighted": 0.330472 + }, + { + "accuracy": 0.333333, + "f1": 0.32152, + "f1_weighted": 0.344925 + }, + { + "accuracy": 0.29902, + "f1": 0.285842, + "f1_weighted": 0.301035 + }, + { + "accuracy": 0.259804, + "f1": 0.234837, + "f1_weighted": 0.253402 + }, + { + "accuracy": 0.27451, + "f1": 0.261478, + "f1_weighted": 0.287388 + }, + { + "accuracy": 0.29902, + "f1": 0.283172, + "f1_weighted": 0.298595 + }, + { + "accuracy": 0.333333, + "f1": 0.334972, + "f1_weighted": 0.351058 + }, + { + "accuracy": 0.284314, + "f1": 0.264009, + "f1_weighted": 0.288173 + }, + { + "accuracy": 0.333333, + "f1": 0.301978, + "f1_weighted": 0.332393 + } + ], + "main_score": 0.309314, + "hf_subset": "azj_Latn", + "languages": [ + "azj-Latn" + ] + }, + { + "accuracy": 0.204902, + "f1": 0.17917, + "f1_weighted": 0.194464, + "scores_per_experiment": [ + { + "accuracy": 0.279412, + "f1": 0.21603, + "f1_weighted": 0.258237 + }, + { + "accuracy": 0.210784, + "f1": 0.18613, + "f1_weighted": 0.190744 + }, + { + "accuracy": 0.171569, + "f1": 0.159328, + "f1_weighted": 0.172229 + }, + { + "accuracy": 0.220588, + "f1": 0.206476, + "f1_weighted": 0.214822 + }, + { + "accuracy": 0.176471, + "f1": 0.17414, + "f1_weighted": 0.170205 + }, + { + "accuracy": 0.254902, + "f1": 0.244921, + "f1_weighted": 0.258398 + }, + { + "accuracy": 0.161765, + "f1": 0.142023, + "f1_weighted": 0.141537 + }, + { + "accuracy": 0.220588, + "f1": 0.158502, + "f1_weighted": 0.194181 + }, + { + "accuracy": 0.166667, + "f1": 0.136266, + "f1_weighted": 0.148188 + }, + { + "accuracy": 0.186275, + "f1": 0.167881, + "f1_weighted": 0.196098 + } + ], + "main_score": 0.204902, + "hf_subset": "bak_Cyrl", + "languages": [ + "bak-Cyrl" + ] + }, + { + "accuracy": 0.304902, + "f1": 0.293692, + "f1_weighted": 0.303794, + "scores_per_experiment": [ + { + "accuracy": 0.29902, + "f1": 0.294101, + "f1_weighted": 0.304777 + }, + { + "accuracy": 0.343137, + "f1": 0.33252, + "f1_weighted": 0.340267 + }, + { + "accuracy": 0.289216, + "f1": 0.271346, + "f1_weighted": 0.287877 + }, + { + "accuracy": 0.279412, + "f1": 0.271158, + "f1_weighted": 0.26975 + }, + { + "accuracy": 0.313725, + "f1": 0.307856, + "f1_weighted": 0.307489 + }, + { + "accuracy": 0.318627, + "f1": 0.304599, + "f1_weighted": 0.327308 + }, + { + "accuracy": 0.357843, + "f1": 0.319951, + "f1_weighted": 0.358823 + }, + { + "accuracy": 0.269608, + "f1": 0.269588, + "f1_weighted": 0.259293 + }, + { + "accuracy": 0.294118, + "f1": 0.281227, + "f1_weighted": 0.286724 + }, + { + "accuracy": 0.284314, + "f1": 0.284571, + "f1_weighted": 0.295635 + } + ], + "main_score": 0.304902, + "hf_subset": "bam_Latn", + "languages": [ + "bam-Latn" + ] + }, + { + "accuracy": 0.341176, + "f1": 0.327238, + "f1_weighted": 0.345057, + "scores_per_experiment": [ + { + "accuracy": 0.387255, + "f1": 0.383842, + "f1_weighted": 0.394516 + }, + { + "accuracy": 0.352941, + "f1": 0.338089, + "f1_weighted": 0.35291 + }, + { + "accuracy": 0.289216, + "f1": 0.279077, + "f1_weighted": 0.280718 + }, + { + "accuracy": 0.313725, + "f1": 0.29259, + "f1_weighted": 0.317409 + }, + { + "accuracy": 0.367647, + "f1": 0.360934, + "f1_weighted": 0.373192 + }, + { + "accuracy": 0.313725, + "f1": 0.273763, + "f1_weighted": 0.323524 + }, + { + "accuracy": 0.333333, + "f1": 0.327383, + "f1_weighted": 0.336228 + }, + { + "accuracy": 0.318627, + "f1": 0.314949, + "f1_weighted": 0.327811 + }, + { + "accuracy": 0.352941, + "f1": 0.336176, + "f1_weighted": 0.361517 + }, + { + "accuracy": 0.382353, + "f1": 0.365577, + "f1_weighted": 0.382746 + } + ], + "main_score": 0.341176, + "hf_subset": "ban_Latn", + "languages": [ + "ban-Latn" + ] + }, + { + "accuracy": 0.185294, + "f1": 0.165843, + "f1_weighted": 0.173125, + "scores_per_experiment": [ + { + "accuracy": 0.230392, + "f1": 0.165587, + "f1_weighted": 0.198556 + }, + { + "accuracy": 0.205882, + "f1": 0.162355, + "f1_weighted": 0.159287 + }, + { + "accuracy": 0.147059, + "f1": 0.155579, + "f1_weighted": 0.139519 + }, + { + "accuracy": 0.196078, + "f1": 0.188115, + "f1_weighted": 0.201795 + }, + { + "accuracy": 0.137255, + "f1": 0.133889, + "f1_weighted": 0.121319 + }, + { + "accuracy": 0.186275, + "f1": 0.181862, + "f1_weighted": 0.189179 + }, + { + "accuracy": 0.196078, + "f1": 0.164183, + "f1_weighted": 0.173502 + }, + { + "accuracy": 0.22549, + "f1": 0.210232, + "f1_weighted": 0.223419 + }, + { + "accuracy": 0.171569, + "f1": 0.167398, + "f1_weighted": 0.171861 + }, + { + "accuracy": 0.156863, + "f1": 0.129227, + "f1_weighted": 0.152807 + } + ], + "main_score": 0.185294, + "hf_subset": "bel_Cyrl", + "languages": [ + "bel-Cyrl" + ] + }, + { + "accuracy": 0.338725, + "f1": 0.323578, + "f1_weighted": 0.335165, + "scores_per_experiment": [ + { + "accuracy": 0.367647, + "f1": 0.327935, + "f1_weighted": 0.361148 + }, + { + "accuracy": 0.29902, + "f1": 0.299808, + "f1_weighted": 0.28513 + }, + { + "accuracy": 0.313725, + "f1": 0.294605, + "f1_weighted": 0.302977 + }, + { + "accuracy": 0.318627, + "f1": 0.329692, + "f1_weighted": 0.327204 + }, + { + "accuracy": 0.323529, + "f1": 0.305759, + "f1_weighted": 0.307863 + }, + { + "accuracy": 0.323529, + "f1": 0.304052, + "f1_weighted": 0.316382 + }, + { + "accuracy": 0.362745, + "f1": 0.328726, + "f1_weighted": 0.357872 + }, + { + "accuracy": 0.392157, + "f1": 0.384687, + "f1_weighted": 0.406134 + }, + { + "accuracy": 0.348039, + "f1": 0.323629, + "f1_weighted": 0.353066 + }, + { + "accuracy": 0.338235, + "f1": 0.336889, + "f1_weighted": 0.333877 + } + ], + "main_score": 0.338725, + "hf_subset": "bem_Latn", + "languages": [ + "bem-Latn" + ] + }, + { + "accuracy": 0.159804, + "f1": 0.12335, + "f1_weighted": 0.133274, + "scores_per_experiment": [ + { + "accuracy": 0.166667, + "f1": 0.124608, + "f1_weighted": 0.143849 + }, + { + "accuracy": 0.137255, + "f1": 0.103742, + "f1_weighted": 0.113021 + }, + { + "accuracy": 0.215686, + "f1": 0.174129, + "f1_weighted": 0.174838 + }, + { + "accuracy": 0.156863, + "f1": 0.122306, + "f1_weighted": 0.121218 + }, + { + "accuracy": 0.137255, + "f1": 0.098069, + "f1_weighted": 0.12675 + }, + { + "accuracy": 0.196078, + "f1": 0.145356, + "f1_weighted": 0.16208 + }, + { + "accuracy": 0.147059, + "f1": 0.14175, + "f1_weighted": 0.147347 + }, + { + "accuracy": 0.147059, + "f1": 0.109003, + "f1_weighted": 0.122141 + }, + { + "accuracy": 0.127451, + "f1": 0.109707, + "f1_weighted": 0.111487 + }, + { + "accuracy": 0.166667, + "f1": 0.104826, + "f1_weighted": 0.110009 + } + ], + "main_score": 0.159804, + "hf_subset": "ben_Beng", + "languages": [ + "ben-Beng" + ] + }, + { + "accuracy": 0.221569, + "f1": 0.177037, + "f1_weighted": 0.197095, + "scores_per_experiment": [ + { + "accuracy": 0.240196, + "f1": 0.187758, + "f1_weighted": 0.208039 + }, + { + "accuracy": 0.259804, + "f1": 0.211697, + "f1_weighted": 0.24086 + }, + { + "accuracy": 0.171569, + "f1": 0.15974, + "f1_weighted": 0.150068 + }, + { + "accuracy": 0.235294, + "f1": 0.194915, + "f1_weighted": 0.220065 + }, + { + "accuracy": 0.166667, + "f1": 0.147472, + "f1_weighted": 0.150925 + }, + { + "accuracy": 0.205882, + "f1": 0.199119, + "f1_weighted": 0.189803 + }, + { + "accuracy": 0.220588, + "f1": 0.179717, + "f1_weighted": 0.200703 + }, + { + "accuracy": 0.289216, + "f1": 0.17905, + "f1_weighted": 0.2365 + }, + { + "accuracy": 0.240196, + "f1": 0.170922, + "f1_weighted": 0.216992 + }, + { + "accuracy": 0.186275, + "f1": 0.139978, + "f1_weighted": 0.156995 + } + ], + "main_score": 0.221569, + "hf_subset": "bho_Deva", + "languages": [ + "bho-Deva" + ] + }, + { + "accuracy": 0.343627, + "f1": 0.33431, + "f1_weighted": 0.34828, + "scores_per_experiment": [ + { + "accuracy": 0.367647, + "f1": 0.357785, + "f1_weighted": 0.37162 + }, + { + "accuracy": 0.313725, + "f1": 0.304982, + "f1_weighted": 0.318862 + }, + { + "accuracy": 0.338235, + "f1": 0.331262, + "f1_weighted": 0.336428 + }, + { + "accuracy": 0.343137, + "f1": 0.339062, + "f1_weighted": 0.337804 + }, + { + "accuracy": 0.323529, + "f1": 0.316408, + "f1_weighted": 0.329512 + }, + { + "accuracy": 0.313725, + "f1": 0.305749, + "f1_weighted": 0.314995 + }, + { + "accuracy": 0.367647, + "f1": 0.351434, + "f1_weighted": 0.369778 + }, + { + "accuracy": 0.397059, + "f1": 0.382792, + "f1_weighted": 0.412289 + }, + { + "accuracy": 0.328431, + "f1": 0.319806, + "f1_weighted": 0.340045 + }, + { + "accuracy": 0.343137, + "f1": 0.333824, + "f1_weighted": 0.351463 + } + ], + "main_score": 0.343627, + "hf_subset": "bjn_Latn", + "languages": [ + "bjn-Latn" + ] + }, + { + "accuracy": 0.183824, + "f1": 0.127714, + "f1_weighted": 0.142979, + "scores_per_experiment": [ + { + "accuracy": 0.186275, + "f1": 0.121027, + "f1_weighted": 0.140498 + }, + { + "accuracy": 0.20098, + "f1": 0.121672, + "f1_weighted": 0.130459 + }, + { + "accuracy": 0.166667, + "f1": 0.136786, + "f1_weighted": 0.13377 + }, + { + "accuracy": 0.156863, + "f1": 0.098957, + "f1_weighted": 0.109189 + }, + { + "accuracy": 0.181373, + "f1": 0.117543, + "f1_weighted": 0.152625 + }, + { + "accuracy": 0.147059, + "f1": 0.096151, + "f1_weighted": 0.092851 + }, + { + "accuracy": 0.230392, + "f1": 0.178074, + "f1_weighted": 0.203643 + }, + { + "accuracy": 0.191176, + "f1": 0.157953, + "f1_weighted": 0.162782 + }, + { + "accuracy": 0.230392, + "f1": 0.143563, + "f1_weighted": 0.186028 + }, + { + "accuracy": 0.147059, + "f1": 0.10541, + "f1_weighted": 0.11794 + } + ], + "main_score": 0.183824, + "hf_subset": "bod_Tibt", + "languages": [ + "bod-Tibt" + ] + }, + { + "accuracy": 0.343627, + "f1": 0.328531, + "f1_weighted": 0.345041, + "scores_per_experiment": [ + { + "accuracy": 0.357843, + "f1": 0.351622, + "f1_weighted": 0.354214 + }, + { + "accuracy": 0.323529, + "f1": 0.322407, + "f1_weighted": 0.329403 + }, + { + "accuracy": 0.323529, + "f1": 0.301848, + "f1_weighted": 0.329553 + }, + { + "accuracy": 0.318627, + "f1": 0.301159, + "f1_weighted": 0.319975 + }, + { + "accuracy": 0.362745, + "f1": 0.351184, + "f1_weighted": 0.37128 + }, + { + "accuracy": 0.338235, + "f1": 0.319674, + "f1_weighted": 0.335962 + }, + { + "accuracy": 0.372549, + "f1": 0.357261, + "f1_weighted": 0.363491 + }, + { + "accuracy": 0.446078, + "f1": 0.424833, + "f1_weighted": 0.445839 + }, + { + "accuracy": 0.303922, + "f1": 0.276718, + "f1_weighted": 0.300075 + }, + { + "accuracy": 0.289216, + "f1": 0.278608, + "f1_weighted": 0.300618 + } + ], + "main_score": 0.343627, + "hf_subset": "bos_Latn", + "languages": [ + "bos-Latn" + ] + }, + { + "accuracy": 0.318627, + "f1": 0.305051, + "f1_weighted": 0.324572, + "scores_per_experiment": [ + { + "accuracy": 0.348039, + "f1": 0.354149, + "f1_weighted": 0.3545 + }, + { + "accuracy": 0.303922, + "f1": 0.285717, + "f1_weighted": 0.31963 + }, + { + "accuracy": 0.284314, + "f1": 0.275536, + "f1_weighted": 0.29282 + }, + { + "accuracy": 0.294118, + "f1": 0.266332, + "f1_weighted": 0.288265 + }, + { + "accuracy": 0.303922, + "f1": 0.28667, + "f1_weighted": 0.309666 + }, + { + "accuracy": 0.294118, + "f1": 0.276627, + "f1_weighted": 0.301252 + }, + { + "accuracy": 0.343137, + "f1": 0.31777, + "f1_weighted": 0.337797 + }, + { + "accuracy": 0.362745, + "f1": 0.367049, + "f1_weighted": 0.374156 + }, + { + "accuracy": 0.328431, + "f1": 0.309224, + "f1_weighted": 0.337593 + }, + { + "accuracy": 0.323529, + "f1": 0.311431, + "f1_weighted": 0.330043 + } + ], + "main_score": 0.318627, + "hf_subset": "bug_Latn", + "languages": [ + "bug-Latn" + ] + }, + { + "accuracy": 0.204902, + "f1": 0.188911, + "f1_weighted": 0.200519, + "scores_per_experiment": [ + { + "accuracy": 0.284314, + "f1": 0.228383, + "f1_weighted": 0.267557 + }, + { + "accuracy": 0.20098, + "f1": 0.18271, + "f1_weighted": 0.19229 + }, + { + "accuracy": 0.181373, + "f1": 0.177041, + "f1_weighted": 0.179929 + }, + { + "accuracy": 0.191176, + "f1": 0.186529, + "f1_weighted": 0.194298 + }, + { + "accuracy": 0.166667, + "f1": 0.16088, + "f1_weighted": 0.174156 + }, + { + "accuracy": 0.196078, + "f1": 0.190863, + "f1_weighted": 0.196068 + }, + { + "accuracy": 0.220588, + "f1": 0.207973, + "f1_weighted": 0.206612 + }, + { + "accuracy": 0.215686, + "f1": 0.181469, + "f1_weighted": 0.205008 + }, + { + "accuracy": 0.196078, + "f1": 0.186196, + "f1_weighted": 0.195032 + }, + { + "accuracy": 0.196078, + "f1": 0.187062, + "f1_weighted": 0.194244 + } + ], + "main_score": 0.204902, + "hf_subset": "bul_Cyrl", + "languages": [ + "bul-Cyrl" + ] + }, + { + "accuracy": 0.510294, + "f1": 0.501681, + "f1_weighted": 0.512812, + "scores_per_experiment": [ + { + "accuracy": 0.553922, + "f1": 0.540145, + "f1_weighted": 0.554735 + }, + { + "accuracy": 0.495098, + "f1": 0.501058, + "f1_weighted": 0.503141 + }, + { + "accuracy": 0.5, + "f1": 0.509761, + "f1_weighted": 0.49251 + }, + { + "accuracy": 0.5, + "f1": 0.480701, + "f1_weighted": 0.502202 + }, + { + "accuracy": 0.509804, + "f1": 0.494488, + "f1_weighted": 0.521558 + }, + { + "accuracy": 0.446078, + "f1": 0.432767, + "f1_weighted": 0.44606 + }, + { + "accuracy": 0.539216, + "f1": 0.519483, + "f1_weighted": 0.536643 + }, + { + "accuracy": 0.514706, + "f1": 0.518252, + "f1_weighted": 0.525121 + }, + { + "accuracy": 0.465686, + "f1": 0.447899, + "f1_weighted": 0.462383 + }, + { + "accuracy": 0.578431, + "f1": 0.572256, + "f1_weighted": 0.583771 + } + ], + "main_score": 0.510294, + "hf_subset": "cat_Latn", + "languages": [ + "cat-Latn" + ] + }, + { + "accuracy": 0.452451, + "f1": 0.43345, + "f1_weighted": 0.458903, + "scores_per_experiment": [ + { + "accuracy": 0.5, + "f1": 0.473586, + "f1_weighted": 0.502889 + }, + { + "accuracy": 0.455882, + "f1": 0.436972, + "f1_weighted": 0.463956 + }, + { + "accuracy": 0.421569, + "f1": 0.389027, + "f1_weighted": 0.422849 + }, + { + "accuracy": 0.45098, + "f1": 0.422963, + "f1_weighted": 0.452035 + }, + { + "accuracy": 0.406863, + "f1": 0.4003, + "f1_weighted": 0.406567 + }, + { + "accuracy": 0.45098, + "f1": 0.419014, + "f1_weighted": 0.453927 + }, + { + "accuracy": 0.460784, + "f1": 0.449867, + "f1_weighted": 0.474449 + }, + { + "accuracy": 0.436275, + "f1": 0.425387, + "f1_weighted": 0.444759 + }, + { + "accuracy": 0.436275, + "f1": 0.417122, + "f1_weighted": 0.44745 + }, + { + "accuracy": 0.504902, + "f1": 0.500258, + "f1_weighted": 0.520145 + } + ], + "main_score": 0.452451, + "hf_subset": "ceb_Latn", + "languages": [ + "ceb-Latn" + ] + }, + { + "accuracy": 0.351961, + "f1": 0.334535, + "f1_weighted": 0.349752, + "scores_per_experiment": [ + { + "accuracy": 0.436275, + "f1": 0.403335, + "f1_weighted": 0.44401 + }, + { + "accuracy": 0.323529, + "f1": 0.303204, + "f1_weighted": 0.327219 + }, + { + "accuracy": 0.269608, + "f1": 0.258459, + "f1_weighted": 0.264761 + }, + { + "accuracy": 0.323529, + "f1": 0.314746, + "f1_weighted": 0.330582 + }, + { + "accuracy": 0.367647, + "f1": 0.353565, + "f1_weighted": 0.366732 + }, + { + "accuracy": 0.308824, + "f1": 0.29608, + "f1_weighted": 0.305581 + }, + { + "accuracy": 0.372549, + "f1": 0.347301, + "f1_weighted": 0.362076 + }, + { + "accuracy": 0.416667, + "f1": 0.394116, + "f1_weighted": 0.415347 + }, + { + "accuracy": 0.352941, + "f1": 0.333286, + "f1_weighted": 0.342681 + }, + { + "accuracy": 0.348039, + "f1": 0.341254, + "f1_weighted": 0.338534 + } + ], + "main_score": 0.351961, + "hf_subset": "ces_Latn", + "languages": [ + "ces-Latn" + ] + }, + { + "accuracy": 0.271569, + "f1": 0.256463, + "f1_weighted": 0.274668, + "scores_per_experiment": [ + { + "accuracy": 0.303922, + "f1": 0.258045, + "f1_weighted": 0.299338 + }, + { + "accuracy": 0.240196, + "f1": 0.214741, + "f1_weighted": 0.238734 + }, + { + "accuracy": 0.20098, + "f1": 0.180402, + "f1_weighted": 0.202932 + }, + { + "accuracy": 0.254902, + "f1": 0.246014, + "f1_weighted": 0.262449 + }, + { + "accuracy": 0.264706, + "f1": 0.269926, + "f1_weighted": 0.264685 + }, + { + "accuracy": 0.289216, + "f1": 0.267437, + "f1_weighted": 0.297403 + }, + { + "accuracy": 0.294118, + "f1": 0.28618, + "f1_weighted": 0.305847 + }, + { + "accuracy": 0.264706, + "f1": 0.256561, + "f1_weighted": 0.258875 + }, + { + "accuracy": 0.348039, + "f1": 0.33153, + "f1_weighted": 0.364387 + }, + { + "accuracy": 0.254902, + "f1": 0.253797, + "f1_weighted": 0.25203 + } + ], + "main_score": 0.271569, + "hf_subset": "cjk_Latn", + "languages": [ + "cjk-Latn" + ] + }, + { + "accuracy": 0.153922, + "f1": 0.11924, + "f1_weighted": 0.124442, + "scores_per_experiment": [ + { + "accuracy": 0.220588, + "f1": 0.158928, + "f1_weighted": 0.183037 + }, + { + "accuracy": 0.137255, + "f1": 0.113798, + "f1_weighted": 0.101546 + }, + { + "accuracy": 0.112745, + "f1": 0.094441, + "f1_weighted": 0.082375 + }, + { + "accuracy": 0.156863, + "f1": 0.135788, + "f1_weighted": 0.15086 + }, + { + "accuracy": 0.156863, + "f1": 0.132481, + "f1_weighted": 0.143834 + }, + { + "accuracy": 0.161765, + "f1": 0.133798, + "f1_weighted": 0.124699 + }, + { + "accuracy": 0.151961, + "f1": 0.103002, + "f1_weighted": 0.121198 + }, + { + "accuracy": 0.171569, + "f1": 0.111873, + "f1_weighted": 0.1385 + }, + { + "accuracy": 0.142157, + "f1": 0.131798, + "f1_weighted": 0.114112 + }, + { + "accuracy": 0.127451, + "f1": 0.076491, + "f1_weighted": 0.08426 + } + ], + "main_score": 0.153922, + "hf_subset": "ckb_Arab", + "languages": [ + "ckb-Arab" + ] + }, + { + "accuracy": 0.347059, + "f1": 0.339926, + "f1_weighted": 0.350115, + "scores_per_experiment": [ + { + "accuracy": 0.348039, + "f1": 0.338396, + "f1_weighted": 0.354353 + }, + { + "accuracy": 0.338235, + "f1": 0.329663, + "f1_weighted": 0.337941 + }, + { + "accuracy": 0.318627, + "f1": 0.315699, + "f1_weighted": 0.306343 + }, + { + "accuracy": 0.392157, + "f1": 0.366403, + "f1_weighted": 0.387622 + }, + { + "accuracy": 0.328431, + "f1": 0.330706, + "f1_weighted": 0.320267 + }, + { + "accuracy": 0.348039, + "f1": 0.351295, + "f1_weighted": 0.356963 + }, + { + "accuracy": 0.362745, + "f1": 0.35427, + "f1_weighted": 0.36959 + }, + { + "accuracy": 0.382353, + "f1": 0.376145, + "f1_weighted": 0.40017 + }, + { + "accuracy": 0.29902, + "f1": 0.286873, + "f1_weighted": 0.309587 + }, + { + "accuracy": 0.352941, + "f1": 0.349809, + "f1_weighted": 0.35832 + } + ], + "main_score": 0.347059, + "hf_subset": "crh_Latn", + "languages": [ + "crh-Latn" + ] + }, + { + "accuracy": 0.27598, + "f1": 0.265795, + "f1_weighted": 0.276882, + "scores_per_experiment": [ + { + "accuracy": 0.323529, + "f1": 0.301238, + "f1_weighted": 0.325381 + }, + { + "accuracy": 0.235294, + "f1": 0.225668, + "f1_weighted": 0.235748 + }, + { + "accuracy": 0.240196, + "f1": 0.236816, + "f1_weighted": 0.231305 + }, + { + "accuracy": 0.259804, + "f1": 0.247089, + "f1_weighted": 0.261479 + }, + { + "accuracy": 0.289216, + "f1": 0.285872, + "f1_weighted": 0.292294 + }, + { + "accuracy": 0.220588, + "f1": 0.219421, + "f1_weighted": 0.217258 + }, + { + "accuracy": 0.343137, + "f1": 0.325323, + "f1_weighted": 0.346282 + }, + { + "accuracy": 0.284314, + "f1": 0.296059, + "f1_weighted": 0.295714 + }, + { + "accuracy": 0.254902, + "f1": 0.235265, + "f1_weighted": 0.247976 + }, + { + "accuracy": 0.308824, + "f1": 0.285197, + "f1_weighted": 0.315386 + } + ], + "main_score": 0.27598, + "hf_subset": "cym_Latn", + "languages": [ + "cym-Latn" + ] + }, + { + "accuracy": 0.45, + "f1": 0.439341, + "f1_weighted": 0.451854, + "scores_per_experiment": [ + { + "accuracy": 0.534314, + "f1": 0.516902, + "f1_weighted": 0.537791 + }, + { + "accuracy": 0.436275, + "f1": 0.406446, + "f1_weighted": 0.438852 + }, + { + "accuracy": 0.421569, + "f1": 0.424592, + "f1_weighted": 0.419325 + }, + { + "accuracy": 0.45098, + "f1": 0.446837, + "f1_weighted": 0.447452 + }, + { + "accuracy": 0.47549, + "f1": 0.456271, + "f1_weighted": 0.478708 + }, + { + "accuracy": 0.431373, + "f1": 0.405342, + "f1_weighted": 0.431123 + }, + { + "accuracy": 0.431373, + "f1": 0.418316, + "f1_weighted": 0.438085 + }, + { + "accuracy": 0.441176, + "f1": 0.443638, + "f1_weighted": 0.448788 + }, + { + "accuracy": 0.411765, + "f1": 0.400991, + "f1_weighted": 0.40851 + }, + { + "accuracy": 0.465686, + "f1": 0.47408, + "f1_weighted": 0.469903 + } + ], + "main_score": 0.45, + "hf_subset": "dan_Latn", + "languages": [ + "dan-Latn" + ] + }, + { + "accuracy": 0.655882, + "f1": 0.63741, + "f1_weighted": 0.651325, + "scores_per_experiment": [ + { + "accuracy": 0.656863, + "f1": 0.639134, + "f1_weighted": 0.657373 + }, + { + "accuracy": 0.627451, + "f1": 0.625329, + "f1_weighted": 0.624253 + }, + { + "accuracy": 0.642157, + "f1": 0.642813, + "f1_weighted": 0.626025 + }, + { + "accuracy": 0.647059, + "f1": 0.629458, + "f1_weighted": 0.64025 + }, + { + "accuracy": 0.651961, + "f1": 0.647214, + "f1_weighted": 0.652785 + }, + { + "accuracy": 0.661765, + "f1": 0.633514, + "f1_weighted": 0.65526 + }, + { + "accuracy": 0.671569, + "f1": 0.642013, + "f1_weighted": 0.673924 + }, + { + "accuracy": 0.666667, + "f1": 0.65437, + "f1_weighted": 0.668571 + }, + { + "accuracy": 0.656863, + "f1": 0.600364, + "f1_weighted": 0.642879 + }, + { + "accuracy": 0.676471, + "f1": 0.659887, + "f1_weighted": 0.671936 + } + ], + "main_score": 0.655882, + "hf_subset": "deu_Latn", + "languages": [ + "deu-Latn" + ] + }, + { + "accuracy": 0.340196, + "f1": 0.322116, + "f1_weighted": 0.345368, + "scores_per_experiment": [ + { + "accuracy": 0.318627, + "f1": 0.309331, + "f1_weighted": 0.330312 + }, + { + "accuracy": 0.284314, + "f1": 0.257876, + "f1_weighted": 0.275418 + }, + { + "accuracy": 0.323529, + "f1": 0.311018, + "f1_weighted": 0.330877 + }, + { + "accuracy": 0.392157, + "f1": 0.375027, + "f1_weighted": 0.398281 + }, + { + "accuracy": 0.382353, + "f1": 0.347765, + "f1_weighted": 0.388331 + }, + { + "accuracy": 0.313725, + "f1": 0.31015, + "f1_weighted": 0.315514 + }, + { + "accuracy": 0.29902, + "f1": 0.269753, + "f1_weighted": 0.304397 + }, + { + "accuracy": 0.392157, + "f1": 0.384505, + "f1_weighted": 0.40162 + }, + { + "accuracy": 0.343137, + "f1": 0.31493, + "f1_weighted": 0.349314 + }, + { + "accuracy": 0.352941, + "f1": 0.340803, + "f1_weighted": 0.359618 + } + ], + "main_score": 0.340196, + "hf_subset": "dik_Latn", + "languages": [ + "dik-Latn" + ] + }, + { + "accuracy": 0.285784, + "f1": 0.27147, + "f1_weighted": 0.28285, + "scores_per_experiment": [ + { + "accuracy": 0.29902, + "f1": 0.286069, + "f1_weighted": 0.304647 + }, + { + "accuracy": 0.284314, + "f1": 0.257651, + "f1_weighted": 0.273566 + }, + { + "accuracy": 0.294118, + "f1": 0.275109, + "f1_weighted": 0.284481 + }, + { + "accuracy": 0.264706, + "f1": 0.256616, + "f1_weighted": 0.255055 + }, + { + "accuracy": 0.259804, + "f1": 0.242652, + "f1_weighted": 0.263721 + }, + { + "accuracy": 0.269608, + "f1": 0.259781, + "f1_weighted": 0.268039 + }, + { + "accuracy": 0.29902, + "f1": 0.282051, + "f1_weighted": 0.300432 + }, + { + "accuracy": 0.294118, + "f1": 0.290285, + "f1_weighted": 0.282398 + }, + { + "accuracy": 0.27451, + "f1": 0.250869, + "f1_weighted": 0.270913 + }, + { + "accuracy": 0.318627, + "f1": 0.313615, + "f1_weighted": 0.325249 + } + ], + "main_score": 0.285784, + "hf_subset": "dyu_Latn", + "languages": [ + "dyu-Latn" + ] + }, + { + "accuracy": 0.160294, + "f1": 0.099251, + "f1_weighted": 0.114542, + "scores_per_experiment": [ + { + "accuracy": 0.176471, + "f1": 0.122577, + "f1_weighted": 0.133718 + }, + { + "accuracy": 0.230392, + "f1": 0.091551, + "f1_weighted": 0.143364 + }, + { + "accuracy": 0.112745, + "f1": 0.083265, + "f1_weighted": 0.077529 + }, + { + "accuracy": 0.122549, + "f1": 0.089752, + "f1_weighted": 0.081241 + }, + { + "accuracy": 0.186275, + "f1": 0.128012, + "f1_weighted": 0.166355 + }, + { + "accuracy": 0.151961, + "f1": 0.082895, + "f1_weighted": 0.086809 + }, + { + "accuracy": 0.181373, + "f1": 0.122749, + "f1_weighted": 0.124246 + }, + { + "accuracy": 0.142157, + "f1": 0.092134, + "f1_weighted": 0.111163 + }, + { + "accuracy": 0.137255, + "f1": 0.086699, + "f1_weighted": 0.092883 + }, + { + "accuracy": 0.161765, + "f1": 0.092873, + "f1_weighted": 0.128106 + } + ], + "main_score": 0.160294, + "hf_subset": "dzo_Tibt", + "languages": [ + "dzo-Tibt" + ] + }, + { + "accuracy": 0.175, + "f1": 0.137596, + "f1_weighted": 0.146243, + "scores_per_experiment": [ + { + "accuracy": 0.205882, + "f1": 0.138933, + "f1_weighted": 0.1546 + }, + { + "accuracy": 0.186275, + "f1": 0.162216, + "f1_weighted": 0.17163 + }, + { + "accuracy": 0.102941, + "f1": 0.096787, + "f1_weighted": 0.07991 + }, + { + "accuracy": 0.22549, + "f1": 0.163686, + "f1_weighted": 0.193663 + }, + { + "accuracy": 0.102941, + "f1": 0.088146, + "f1_weighted": 0.080225 + }, + { + "accuracy": 0.142157, + "f1": 0.142785, + "f1_weighted": 0.132545 + }, + { + "accuracy": 0.240196, + "f1": 0.155398, + "f1_weighted": 0.176108 + }, + { + "accuracy": 0.20098, + "f1": 0.168011, + "f1_weighted": 0.194397 + }, + { + "accuracy": 0.20098, + "f1": 0.155858, + "f1_weighted": 0.167823 + }, + { + "accuracy": 0.142157, + "f1": 0.104139, + "f1_weighted": 0.111529 + } + ], + "main_score": 0.175, + "hf_subset": "ell_Grek", + "languages": [ + "ell-Grek" + ] + }, + { + "accuracy": 0.705882, + "f1": 0.686505, + "f1_weighted": 0.704251, + "scores_per_experiment": [ + { + "accuracy": 0.735294, + "f1": 0.703927, + "f1_weighted": 0.731149 + }, + { + "accuracy": 0.671569, + "f1": 0.651334, + "f1_weighted": 0.667102 + }, + { + "accuracy": 0.661765, + "f1": 0.641696, + "f1_weighted": 0.656635 + }, + { + "accuracy": 0.735294, + "f1": 0.714097, + "f1_weighted": 0.736829 + }, + { + "accuracy": 0.754902, + "f1": 0.756863, + "f1_weighted": 0.757728 + }, + { + "accuracy": 0.715686, + "f1": 0.699442, + "f1_weighted": 0.716266 + }, + { + "accuracy": 0.710784, + "f1": 0.692922, + "f1_weighted": 0.719224 + }, + { + "accuracy": 0.691176, + "f1": 0.676052, + "f1_weighted": 0.69075 + }, + { + "accuracy": 0.661765, + "f1": 0.636673, + "f1_weighted": 0.652441 + }, + { + "accuracy": 0.720588, + "f1": 0.692045, + "f1_weighted": 0.714383 + } + ], + "main_score": 0.705882, + "hf_subset": "eng_Latn", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.42549, + "f1": 0.416767, + "f1_weighted": 0.423339, + "scores_per_experiment": [ + { + "accuracy": 0.485294, + "f1": 0.462691, + "f1_weighted": 0.483024 + }, + { + "accuracy": 0.441176, + "f1": 0.429508, + "f1_weighted": 0.440011 + }, + { + "accuracy": 0.397059, + "f1": 0.395072, + "f1_weighted": 0.399013 + }, + { + "accuracy": 0.436275, + "f1": 0.422371, + "f1_weighted": 0.446948 + }, + { + "accuracy": 0.421569, + "f1": 0.417714, + "f1_weighted": 0.414529 + }, + { + "accuracy": 0.421569, + "f1": 0.413013, + "f1_weighted": 0.419599 + }, + { + "accuracy": 0.411765, + "f1": 0.409663, + "f1_weighted": 0.393856 + }, + { + "accuracy": 0.436275, + "f1": 0.436807, + "f1_weighted": 0.436555 + }, + { + "accuracy": 0.401961, + "f1": 0.382922, + "f1_weighted": 0.397271 + }, + { + "accuracy": 0.401961, + "f1": 0.397905, + "f1_weighted": 0.402587 + } + ], + "main_score": 0.42549, + "hf_subset": "epo_Latn", + "languages": [ + "epo-Latn" + ] + }, + { + "accuracy": 0.336275, + "f1": 0.327827, + "f1_weighted": 0.333755, + "scores_per_experiment": [ + { + "accuracy": 0.377451, + "f1": 0.346992, + "f1_weighted": 0.378382 + }, + { + "accuracy": 0.245098, + "f1": 0.237931, + "f1_weighted": 0.239513 + }, + { + "accuracy": 0.377451, + "f1": 0.367464, + "f1_weighted": 0.368704 + }, + { + "accuracy": 0.328431, + "f1": 0.340019, + "f1_weighted": 0.326613 + }, + { + "accuracy": 0.328431, + "f1": 0.331228, + "f1_weighted": 0.338437 + }, + { + "accuracy": 0.308824, + "f1": 0.298305, + "f1_weighted": 0.302547 + }, + { + "accuracy": 0.377451, + "f1": 0.359702, + "f1_weighted": 0.377306 + }, + { + "accuracy": 0.333333, + "f1": 0.33723, + "f1_weighted": 0.341892 + }, + { + "accuracy": 0.303922, + "f1": 0.287642, + "f1_weighted": 0.27914 + }, + { + "accuracy": 0.382353, + "f1": 0.371752, + "f1_weighted": 0.385014 + } + ], + "main_score": 0.336275, + "hf_subset": "est_Latn", + "languages": [ + "est-Latn" + ] + }, + { + "accuracy": 0.341176, + "f1": 0.329153, + "f1_weighted": 0.343871, + "scores_per_experiment": [ + { + "accuracy": 0.352941, + "f1": 0.326955, + "f1_weighted": 0.356481 + }, + { + "accuracy": 0.382353, + "f1": 0.363191, + "f1_weighted": 0.382229 + }, + { + "accuracy": 0.29902, + "f1": 0.297859, + "f1_weighted": 0.299489 + }, + { + "accuracy": 0.406863, + "f1": 0.399199, + "f1_weighted": 0.399533 + }, + { + "accuracy": 0.303922, + "f1": 0.301384, + "f1_weighted": 0.30199 + }, + { + "accuracy": 0.367647, + "f1": 0.356292, + "f1_weighted": 0.377712 + }, + { + "accuracy": 0.313725, + "f1": 0.291067, + "f1_weighted": 0.318206 + }, + { + "accuracy": 0.318627, + "f1": 0.305268, + "f1_weighted": 0.334655 + }, + { + "accuracy": 0.323529, + "f1": 0.300786, + "f1_weighted": 0.315519 + }, + { + "accuracy": 0.343137, + "f1": 0.349529, + "f1_weighted": 0.352899 + } + ], + "main_score": 0.341176, + "hf_subset": "eus_Latn", + "languages": [ + "eus-Latn" + ] + }, + { + "accuracy": 0.309314, + "f1": 0.302439, + "f1_weighted": 0.309538, + "scores_per_experiment": [ + { + "accuracy": 0.308824, + "f1": 0.297869, + "f1_weighted": 0.314635 + }, + { + "accuracy": 0.328431, + "f1": 0.328369, + "f1_weighted": 0.316704 + }, + { + "accuracy": 0.230392, + "f1": 0.227498, + "f1_weighted": 0.221492 + }, + { + "accuracy": 0.318627, + "f1": 0.313973, + "f1_weighted": 0.333001 + }, + { + "accuracy": 0.269608, + "f1": 0.268682, + "f1_weighted": 0.273747 + }, + { + "accuracy": 0.29902, + "f1": 0.276944, + "f1_weighted": 0.298024 + }, + { + "accuracy": 0.333333, + "f1": 0.32243, + "f1_weighted": 0.32941 + }, + { + "accuracy": 0.338235, + "f1": 0.326599, + "f1_weighted": 0.332825 + }, + { + "accuracy": 0.308824, + "f1": 0.295752, + "f1_weighted": 0.311739 + }, + { + "accuracy": 0.357843, + "f1": 0.366275, + "f1_weighted": 0.363797 + } + ], + "main_score": 0.309314, + "hf_subset": "ewe_Latn", + "languages": [ + "ewe-Latn" + ] + }, + { + "accuracy": 0.343627, + "f1": 0.322546, + "f1_weighted": 0.34141, + "scores_per_experiment": [ + { + "accuracy": 0.372549, + "f1": 0.340092, + "f1_weighted": 0.366377 + }, + { + "accuracy": 0.279412, + "f1": 0.266294, + "f1_weighted": 0.28594 + }, + { + "accuracy": 0.308824, + "f1": 0.295587, + "f1_weighted": 0.288019 + }, + { + "accuracy": 0.362745, + "f1": 0.338283, + "f1_weighted": 0.358614 + }, + { + "accuracy": 0.348039, + "f1": 0.330724, + "f1_weighted": 0.351298 + }, + { + "accuracy": 0.348039, + "f1": 0.327565, + "f1_weighted": 0.347377 + }, + { + "accuracy": 0.362745, + "f1": 0.319601, + "f1_weighted": 0.356119 + }, + { + "accuracy": 0.431373, + "f1": 0.410983, + "f1_weighted": 0.428454 + }, + { + "accuracy": 0.294118, + "f1": 0.285372, + "f1_weighted": 0.294371 + }, + { + "accuracy": 0.328431, + "f1": 0.310958, + "f1_weighted": 0.337533 + } + ], + "main_score": 0.343627, + "hf_subset": "fao_Latn", + "languages": [ + "fao-Latn" + ] + }, + { + "accuracy": 0.362255, + "f1": 0.34335, + "f1_weighted": 0.360501, + "scores_per_experiment": [ + { + "accuracy": 0.416667, + "f1": 0.380736, + "f1_weighted": 0.406121 + }, + { + "accuracy": 0.308824, + "f1": 0.297685, + "f1_weighted": 0.310446 + }, + { + "accuracy": 0.308824, + "f1": 0.295438, + "f1_weighted": 0.298911 + }, + { + "accuracy": 0.323529, + "f1": 0.294361, + "f1_weighted": 0.328702 + }, + { + "accuracy": 0.392157, + "f1": 0.379269, + "f1_weighted": 0.387096 + }, + { + "accuracy": 0.372549, + "f1": 0.359628, + "f1_weighted": 0.378853 + }, + { + "accuracy": 0.362745, + "f1": 0.33587, + "f1_weighted": 0.364889 + }, + { + "accuracy": 0.352941, + "f1": 0.329163, + "f1_weighted": 0.34143 + }, + { + "accuracy": 0.382353, + "f1": 0.374357, + "f1_weighted": 0.38658 + }, + { + "accuracy": 0.401961, + "f1": 0.386991, + "f1_weighted": 0.401983 + } + ], + "main_score": 0.362255, + "hf_subset": "fij_Latn", + "languages": [ + "fij-Latn" + ] + }, + { + "accuracy": 0.35098, + "f1": 0.337698, + "f1_weighted": 0.353372, + "scores_per_experiment": [ + { + "accuracy": 0.362745, + "f1": 0.343845, + "f1_weighted": 0.365794 + }, + { + "accuracy": 0.377451, + "f1": 0.358866, + "f1_weighted": 0.38011 + }, + { + "accuracy": 0.343137, + "f1": 0.332374, + "f1_weighted": 0.340852 + }, + { + "accuracy": 0.382353, + "f1": 0.375117, + "f1_weighted": 0.386589 + }, + { + "accuracy": 0.338235, + "f1": 0.33639, + "f1_weighted": 0.358688 + }, + { + "accuracy": 0.284314, + "f1": 0.28504, + "f1_weighted": 0.27335 + }, + { + "accuracy": 0.397059, + "f1": 0.376043, + "f1_weighted": 0.402381 + }, + { + "accuracy": 0.343137, + "f1": 0.339624, + "f1_weighted": 0.349812 + }, + { + "accuracy": 0.313725, + "f1": 0.290489, + "f1_weighted": 0.309348 + }, + { + "accuracy": 0.367647, + "f1": 0.33919, + "f1_weighted": 0.366799 + } + ], + "main_score": 0.35098, + "hf_subset": "fin_Latn", + "languages": [ + "fin-Latn" + ] + }, + { + "accuracy": 0.317157, + "f1": 0.293943, + "f1_weighted": 0.316157, + "scores_per_experiment": [ + { + "accuracy": 0.303922, + "f1": 0.283094, + "f1_weighted": 0.299925 + }, + { + "accuracy": 0.318627, + "f1": 0.30542, + "f1_weighted": 0.317045 + }, + { + "accuracy": 0.289216, + "f1": 0.257614, + "f1_weighted": 0.28754 + }, + { + "accuracy": 0.328431, + "f1": 0.299323, + "f1_weighted": 0.326168 + }, + { + "accuracy": 0.318627, + "f1": 0.299075, + "f1_weighted": 0.317166 + }, + { + "accuracy": 0.313725, + "f1": 0.274438, + "f1_weighted": 0.309514 + }, + { + "accuracy": 0.318627, + "f1": 0.300796, + "f1_weighted": 0.326321 + }, + { + "accuracy": 0.382353, + "f1": 0.360485, + "f1_weighted": 0.382349 + }, + { + "accuracy": 0.294118, + "f1": 0.29006, + "f1_weighted": 0.29342 + }, + { + "accuracy": 0.303922, + "f1": 0.269125, + "f1_weighted": 0.302121 + } + ], + "main_score": 0.317157, + "hf_subset": "fon_Latn", + "languages": [ + "fon-Latn" + ] + }, + { + "accuracy": 0.678431, + "f1": 0.663358, + "f1_weighted": 0.67482, + "scores_per_experiment": [ + { + "accuracy": 0.671569, + "f1": 0.65067, + "f1_weighted": 0.667429 + }, + { + "accuracy": 0.656863, + "f1": 0.639052, + "f1_weighted": 0.65623 + }, + { + "accuracy": 0.612745, + "f1": 0.602771, + "f1_weighted": 0.594861 + }, + { + "accuracy": 0.696078, + "f1": 0.674671, + "f1_weighted": 0.693385 + }, + { + "accuracy": 0.676471, + "f1": 0.673819, + "f1_weighted": 0.681541 + }, + { + "accuracy": 0.696078, + "f1": 0.674478, + "f1_weighted": 0.689786 + }, + { + "accuracy": 0.710784, + "f1": 0.70323, + "f1_weighted": 0.711901 + }, + { + "accuracy": 0.70098, + "f1": 0.692425, + "f1_weighted": 0.700387 + }, + { + "accuracy": 0.666667, + "f1": 0.62745, + "f1_weighted": 0.657249 + }, + { + "accuracy": 0.696078, + "f1": 0.695018, + "f1_weighted": 0.695432 + } + ], + "main_score": 0.678431, + "hf_subset": "fra_Latn", + "languages": [ + "fra-Latn" + ] + }, + { + "accuracy": 0.471078, + "f1": 0.450966, + "f1_weighted": 0.472379, + "scores_per_experiment": [ + { + "accuracy": 0.509804, + "f1": 0.484466, + "f1_weighted": 0.507536 + }, + { + "accuracy": 0.5, + "f1": 0.465737, + "f1_weighted": 0.495628 + }, + { + "accuracy": 0.431373, + "f1": 0.427069, + "f1_weighted": 0.427988 + }, + { + "accuracy": 0.455882, + "f1": 0.44182, + "f1_weighted": 0.46923 + }, + { + "accuracy": 0.509804, + "f1": 0.472864, + "f1_weighted": 0.513439 + }, + { + "accuracy": 0.495098, + "f1": 0.476186, + "f1_weighted": 0.491185 + }, + { + "accuracy": 0.490196, + "f1": 0.475432, + "f1_weighted": 0.490154 + }, + { + "accuracy": 0.470588, + "f1": 0.466386, + "f1_weighted": 0.476443 + }, + { + "accuracy": 0.441176, + "f1": 0.407147, + "f1_weighted": 0.430564 + }, + { + "accuracy": 0.406863, + "f1": 0.392556, + "f1_weighted": 0.421621 + } + ], + "main_score": 0.471078, + "hf_subset": "fur_Latn", + "languages": [ + "fur-Latn" + ] + }, + { + "accuracy": 0.298039, + "f1": 0.28801, + "f1_weighted": 0.296068, + "scores_per_experiment": [ + { + "accuracy": 0.259804, + "f1": 0.250617, + "f1_weighted": 0.239701 + }, + { + "accuracy": 0.27451, + "f1": 0.276613, + "f1_weighted": 0.269798 + }, + { + "accuracy": 0.294118, + "f1": 0.284638, + "f1_weighted": 0.299215 + }, + { + "accuracy": 0.259804, + "f1": 0.25062, + "f1_weighted": 0.26389 + }, + { + "accuracy": 0.348039, + "f1": 0.333378, + "f1_weighted": 0.340732 + }, + { + "accuracy": 0.29902, + "f1": 0.298586, + "f1_weighted": 0.306794 + }, + { + "accuracy": 0.264706, + "f1": 0.253865, + "f1_weighted": 0.255204 + }, + { + "accuracy": 0.397059, + "f1": 0.380285, + "f1_weighted": 0.396638 + }, + { + "accuracy": 0.25, + "f1": 0.223734, + "f1_weighted": 0.243481 + }, + { + "accuracy": 0.333333, + "f1": 0.327768, + "f1_weighted": 0.345221 + } + ], + "main_score": 0.298039, + "hf_subset": "fuv_Latn", + "languages": [ + "fuv-Latn" + ] + }, + { + "accuracy": 0.233824, + "f1": 0.221715, + "f1_weighted": 0.234826, + "scores_per_experiment": [ + { + "accuracy": 0.245098, + "f1": 0.228483, + "f1_weighted": 0.233937 + }, + { + "accuracy": 0.186275, + "f1": 0.178973, + "f1_weighted": 0.189205 + }, + { + "accuracy": 0.235294, + "f1": 0.221134, + "f1_weighted": 0.24117 + }, + { + "accuracy": 0.29902, + "f1": 0.29463, + "f1_weighted": 0.301298 + }, + { + "accuracy": 0.220588, + "f1": 0.196389, + "f1_weighted": 0.211176 + }, + { + "accuracy": 0.210784, + "f1": 0.198465, + "f1_weighted": 0.219902 + }, + { + "accuracy": 0.259804, + "f1": 0.240975, + "f1_weighted": 0.26281 + }, + { + "accuracy": 0.191176, + "f1": 0.187906, + "f1_weighted": 0.199892 + }, + { + "accuracy": 0.220588, + "f1": 0.214407, + "f1_weighted": 0.222676 + }, + { + "accuracy": 0.269608, + "f1": 0.255785, + "f1_weighted": 0.266197 + } + ], + "main_score": 0.233824, + "hf_subset": "gaz_Latn", + "languages": [ + "gaz-Latn" + ] + }, + { + "accuracy": 0.259804, + "f1": 0.255995, + "f1_weighted": 0.256457, + "scores_per_experiment": [ + { + "accuracy": 0.323529, + "f1": 0.305252, + "f1_weighted": 0.327089 + }, + { + "accuracy": 0.254902, + "f1": 0.253072, + "f1_weighted": 0.248229 + }, + { + "accuracy": 0.245098, + "f1": 0.24307, + "f1_weighted": 0.230759 + }, + { + "accuracy": 0.215686, + "f1": 0.218378, + "f1_weighted": 0.208686 + }, + { + "accuracy": 0.220588, + "f1": 0.226731, + "f1_weighted": 0.223358 + }, + { + "accuracy": 0.25, + "f1": 0.248902, + "f1_weighted": 0.247431 + }, + { + "accuracy": 0.269608, + "f1": 0.259476, + "f1_weighted": 0.254313 + }, + { + "accuracy": 0.269608, + "f1": 0.259354, + "f1_weighted": 0.267036 + }, + { + "accuracy": 0.308824, + "f1": 0.303035, + "f1_weighted": 0.312245 + }, + { + "accuracy": 0.240196, + "f1": 0.242683, + "f1_weighted": 0.245423 + } + ], + "main_score": 0.259804, + "hf_subset": "gla_Latn", + "languages": [ + "gla-Latn" + ] + }, + { + "accuracy": 0.27451, + "f1": 0.267856, + "f1_weighted": 0.272999, + "scores_per_experiment": [ + { + "accuracy": 0.352941, + "f1": 0.346567, + "f1_weighted": 0.346931 + }, + { + "accuracy": 0.259804, + "f1": 0.254717, + "f1_weighted": 0.262404 + }, + { + "accuracy": 0.220588, + "f1": 0.212443, + "f1_weighted": 0.21149 + }, + { + "accuracy": 0.289216, + "f1": 0.289452, + "f1_weighted": 0.280728 + }, + { + "accuracy": 0.29902, + "f1": 0.302829, + "f1_weighted": 0.315685 + }, + { + "accuracy": 0.259804, + "f1": 0.247094, + "f1_weighted": 0.250843 + }, + { + "accuracy": 0.245098, + "f1": 0.239299, + "f1_weighted": 0.243469 + }, + { + "accuracy": 0.279412, + "f1": 0.265768, + "f1_weighted": 0.278936 + }, + { + "accuracy": 0.289216, + "f1": 0.268133, + "f1_weighted": 0.287573 + }, + { + "accuracy": 0.25, + "f1": 0.252254, + "f1_weighted": 0.251926 + } + ], + "main_score": 0.27451, + "hf_subset": "gle_Latn", + "languages": [ + "gle-Latn" + ] + }, + { + "accuracy": 0.545098, + "f1": 0.521692, + "f1_weighted": 0.545235, + "scores_per_experiment": [ + { + "accuracy": 0.553922, + "f1": 0.516344, + "f1_weighted": 0.553484 + }, + { + "accuracy": 0.529412, + "f1": 0.493675, + "f1_weighted": 0.524022 + }, + { + "accuracy": 0.509804, + "f1": 0.507847, + "f1_weighted": 0.505334 + }, + { + "accuracy": 0.54902, + "f1": 0.516179, + "f1_weighted": 0.542483 + }, + { + "accuracy": 0.588235, + "f1": 0.565183, + "f1_weighted": 0.599455 + }, + { + "accuracy": 0.563725, + "f1": 0.55057, + "f1_weighted": 0.570843 + }, + { + "accuracy": 0.583333, + "f1": 0.559018, + "f1_weighted": 0.581443 + }, + { + "accuracy": 0.495098, + "f1": 0.48376, + "f1_weighted": 0.504257 + }, + { + "accuracy": 0.514706, + "f1": 0.47074, + "f1_weighted": 0.501734 + }, + { + "accuracy": 0.563725, + "f1": 0.553607, + "f1_weighted": 0.569295 + } + ], + "main_score": 0.545098, + "hf_subset": "glg_Latn", + "languages": [ + "glg-Latn" + ] + }, + { + "accuracy": 0.4, + "f1": 0.381089, + "f1_weighted": 0.403299, + "scores_per_experiment": [ + { + "accuracy": 0.416667, + "f1": 0.389413, + "f1_weighted": 0.420502 + }, + { + "accuracy": 0.377451, + "f1": 0.359273, + "f1_weighted": 0.383197 + }, + { + "accuracy": 0.392157, + "f1": 0.382319, + "f1_weighted": 0.398918 + }, + { + "accuracy": 0.397059, + "f1": 0.402276, + "f1_weighted": 0.392885 + }, + { + "accuracy": 0.47549, + "f1": 0.444688, + "f1_weighted": 0.478751 + }, + { + "accuracy": 0.426471, + "f1": 0.397419, + "f1_weighted": 0.442566 + }, + { + "accuracy": 0.343137, + "f1": 0.322792, + "f1_weighted": 0.34391 + }, + { + "accuracy": 0.406863, + "f1": 0.388984, + "f1_weighted": 0.415435 + }, + { + "accuracy": 0.406863, + "f1": 0.372629, + "f1_weighted": 0.400646 + }, + { + "accuracy": 0.357843, + "f1": 0.351099, + "f1_weighted": 0.356185 + } + ], + "main_score": 0.4, + "hf_subset": "grn_Latn", + "languages": [ + "grn-Latn" + ] + }, + { + "accuracy": 0.167647, + "f1": 0.135589, + "f1_weighted": 0.133208, + "scores_per_experiment": [ + { + "accuracy": 0.171569, + "f1": 0.121215, + "f1_weighted": 0.127847 + }, + { + "accuracy": 0.161765, + "f1": 0.139909, + "f1_weighted": 0.134975 + }, + { + "accuracy": 0.142157, + "f1": 0.135313, + "f1_weighted": 0.119484 + }, + { + "accuracy": 0.137255, + "f1": 0.091265, + "f1_weighted": 0.08341 + }, + { + "accuracy": 0.166667, + "f1": 0.154544, + "f1_weighted": 0.151194 + }, + { + "accuracy": 0.142157, + "f1": 0.13478, + "f1_weighted": 0.112186 + }, + { + "accuracy": 0.161765, + "f1": 0.143044, + "f1_weighted": 0.128761 + }, + { + "accuracy": 0.235294, + "f1": 0.180932, + "f1_weighted": 0.192619 + }, + { + "accuracy": 0.25, + "f1": 0.167124, + "f1_weighted": 0.195669 + }, + { + "accuracy": 0.107843, + "f1": 0.087761, + "f1_weighted": 0.085934 + } + ], + "main_score": 0.167647, + "hf_subset": "guj_Gujr", + "languages": [ + "guj-Gujr" + ] + }, + { + "accuracy": 0.319118, + "f1": 0.305989, + "f1_weighted": 0.321289, + "scores_per_experiment": [ + { + "accuracy": 0.446078, + "f1": 0.434973, + "f1_weighted": 0.44586 + }, + { + "accuracy": 0.279412, + "f1": 0.268727, + "f1_weighted": 0.286454 + }, + { + "accuracy": 0.259804, + "f1": 0.254246, + "f1_weighted": 0.259719 + }, + { + "accuracy": 0.303922, + "f1": 0.291705, + "f1_weighted": 0.307962 + }, + { + "accuracy": 0.269608, + "f1": 0.264648, + "f1_weighted": 0.268159 + }, + { + "accuracy": 0.318627, + "f1": 0.304108, + "f1_weighted": 0.311133 + }, + { + "accuracy": 0.343137, + "f1": 0.332519, + "f1_weighted": 0.342616 + }, + { + "accuracy": 0.289216, + "f1": 0.271757, + "f1_weighted": 0.3009 + }, + { + "accuracy": 0.323529, + "f1": 0.295511, + "f1_weighted": 0.319839 + }, + { + "accuracy": 0.357843, + "f1": 0.341692, + "f1_weighted": 0.370249 + } + ], + "main_score": 0.319118, + "hf_subset": "hat_Latn", + "languages": [ + "hat-Latn" + ] + }, + { + "accuracy": 0.29902, + "f1": 0.287585, + "f1_weighted": 0.298785, + "scores_per_experiment": [ + { + "accuracy": 0.357843, + "f1": 0.34865, + "f1_weighted": 0.36302 + }, + { + "accuracy": 0.259804, + "f1": 0.259209, + "f1_weighted": 0.260119 + }, + { + "accuracy": 0.269608, + "f1": 0.259888, + "f1_weighted": 0.259747 + }, + { + "accuracy": 0.313725, + "f1": 0.293029, + "f1_weighted": 0.308581 + }, + { + "accuracy": 0.245098, + "f1": 0.23664, + "f1_weighted": 0.237729 + }, + { + "accuracy": 0.279412, + "f1": 0.265641, + "f1_weighted": 0.285438 + }, + { + "accuracy": 0.303922, + "f1": 0.295888, + "f1_weighted": 0.304256 + }, + { + "accuracy": 0.323529, + "f1": 0.308342, + "f1_weighted": 0.326866 + }, + { + "accuracy": 0.279412, + "f1": 0.268379, + "f1_weighted": 0.278055 + }, + { + "accuracy": 0.357843, + "f1": 0.34018, + "f1_weighted": 0.364035 + } + ], + "main_score": 0.29902, + "hf_subset": "hau_Latn", + "languages": [ + "hau-Latn" + ] + }, + { + "accuracy": 0.184314, + "f1": 0.133228, + "f1_weighted": 0.142989, + "scores_per_experiment": [ + { + "accuracy": 0.205882, + "f1": 0.099329, + "f1_weighted": 0.117637 + }, + { + "accuracy": 0.245098, + "f1": 0.190723, + "f1_weighted": 0.219307 + }, + { + "accuracy": 0.147059, + "f1": 0.13224, + "f1_weighted": 0.132096 + }, + { + "accuracy": 0.161765, + "f1": 0.11133, + "f1_weighted": 0.113453 + }, + { + "accuracy": 0.210784, + "f1": 0.134352, + "f1_weighted": 0.141979 + }, + { + "accuracy": 0.181373, + "f1": 0.169005, + "f1_weighted": 0.147158 + }, + { + "accuracy": 0.205882, + "f1": 0.135869, + "f1_weighted": 0.158275 + }, + { + "accuracy": 0.20098, + "f1": 0.175297, + "f1_weighted": 0.192216 + }, + { + "accuracy": 0.117647, + "f1": 0.082981, + "f1_weighted": 0.097135 + }, + { + "accuracy": 0.166667, + "f1": 0.101157, + "f1_weighted": 0.110637 + } + ], + "main_score": 0.184314, + "hf_subset": "heb_Hebr", + "languages": [ + "heb-Hebr" + ] + }, + { + "accuracy": 0.179412, + "f1": 0.151093, + "f1_weighted": 0.157956, + "scores_per_experiment": [ + { + "accuracy": 0.191176, + "f1": 0.141851, + "f1_weighted": 0.152854 + }, + { + "accuracy": 0.205882, + "f1": 0.176142, + "f1_weighted": 0.161868 + }, + { + "accuracy": 0.191176, + "f1": 0.162998, + "f1_weighted": 0.167524 + }, + { + "accuracy": 0.151961, + "f1": 0.121165, + "f1_weighted": 0.114891 + }, + { + "accuracy": 0.171569, + "f1": 0.154251, + "f1_weighted": 0.143717 + }, + { + "accuracy": 0.186275, + "f1": 0.180246, + "f1_weighted": 0.173898 + }, + { + "accuracy": 0.186275, + "f1": 0.142315, + "f1_weighted": 0.172307 + }, + { + "accuracy": 0.142157, + "f1": 0.1439, + "f1_weighted": 0.138856 + }, + { + "accuracy": 0.20098, + "f1": 0.155608, + "f1_weighted": 0.191841 + }, + { + "accuracy": 0.166667, + "f1": 0.132455, + "f1_weighted": 0.161799 + } + ], + "main_score": 0.179412, + "hf_subset": "hin_Deva", + "languages": [ + "hin-Deva" + ] + }, + { + "accuracy": 0.187255, + "f1": 0.139939, + "f1_weighted": 0.147841, + "scores_per_experiment": [ + { + "accuracy": 0.142157, + "f1": 0.112711, + "f1_weighted": 0.113479 + }, + { + "accuracy": 0.171569, + "f1": 0.132025, + "f1_weighted": 0.131825 + }, + { + "accuracy": 0.156863, + "f1": 0.15062, + "f1_weighted": 0.137505 + }, + { + "accuracy": 0.137255, + "f1": 0.10936, + "f1_weighted": 0.099711 + }, + { + "accuracy": 0.240196, + "f1": 0.178645, + "f1_weighted": 0.194435 + }, + { + "accuracy": 0.161765, + "f1": 0.124771, + "f1_weighted": 0.137352 + }, + { + "accuracy": 0.235294, + "f1": 0.154581, + "f1_weighted": 0.177151 + }, + { + "accuracy": 0.215686, + "f1": 0.145236, + "f1_weighted": 0.16094 + }, + { + "accuracy": 0.215686, + "f1": 0.168413, + "f1_weighted": 0.18688 + }, + { + "accuracy": 0.196078, + "f1": 0.123025, + "f1_weighted": 0.139129 + } + ], + "main_score": 0.187255, + "hf_subset": "hne_Deva", + "languages": [ + "hne-Deva" + ] + }, + { + "accuracy": 0.355882, + "f1": 0.340623, + "f1_weighted": 0.357545, + "scores_per_experiment": [ + { + "accuracy": 0.367647, + "f1": 0.346475, + "f1_weighted": 0.373946 + }, + { + "accuracy": 0.338235, + "f1": 0.333521, + "f1_weighted": 0.346232 + }, + { + "accuracy": 0.343137, + "f1": 0.323509, + "f1_weighted": 0.34668 + }, + { + "accuracy": 0.382353, + "f1": 0.354079, + "f1_weighted": 0.382146 + }, + { + "accuracy": 0.318627, + "f1": 0.306798, + "f1_weighted": 0.32158 + }, + { + "accuracy": 0.367647, + "f1": 0.357608, + "f1_weighted": 0.370843 + }, + { + "accuracy": 0.357843, + "f1": 0.347347, + "f1_weighted": 0.350151 + }, + { + "accuracy": 0.397059, + "f1": 0.382162, + "f1_weighted": 0.40062 + }, + { + "accuracy": 0.313725, + "f1": 0.289533, + "f1_weighted": 0.301373 + }, + { + "accuracy": 0.372549, + "f1": 0.365197, + "f1_weighted": 0.381876 + } + ], + "main_score": 0.355882, + "hf_subset": "hrv_Latn", + "languages": [ + "hrv-Latn" + ] + }, + { + "accuracy": 0.285784, + "f1": 0.275103, + "f1_weighted": 0.286249, + "scores_per_experiment": [ + { + "accuracy": 0.289216, + "f1": 0.270773, + "f1_weighted": 0.29698 + }, + { + "accuracy": 0.269608, + "f1": 0.269082, + "f1_weighted": 0.266475 + }, + { + "accuracy": 0.22549, + "f1": 0.197764, + "f1_weighted": 0.216456 + }, + { + "accuracy": 0.29902, + "f1": 0.28608, + "f1_weighted": 0.292943 + }, + { + "accuracy": 0.318627, + "f1": 0.300538, + "f1_weighted": 0.325683 + }, + { + "accuracy": 0.313725, + "f1": 0.30989, + "f1_weighted": 0.306148 + }, + { + "accuracy": 0.269608, + "f1": 0.267852, + "f1_weighted": 0.279584 + }, + { + "accuracy": 0.29902, + "f1": 0.296376, + "f1_weighted": 0.313768 + }, + { + "accuracy": 0.303922, + "f1": 0.287404, + "f1_weighted": 0.290898 + }, + { + "accuracy": 0.269608, + "f1": 0.265271, + "f1_weighted": 0.273553 + } + ], + "main_score": 0.285784, + "hf_subset": "hun_Latn", + "languages": [ + "hun-Latn" + ] + }, + { + "accuracy": 0.183824, + "f1": 0.148438, + "f1_weighted": 0.15415, + "scores_per_experiment": [ + { + "accuracy": 0.22549, + "f1": 0.180689, + "f1_weighted": 0.186329 + }, + { + "accuracy": 0.171569, + "f1": 0.135318, + "f1_weighted": 0.122286 + }, + { + "accuracy": 0.181373, + "f1": 0.151518, + "f1_weighted": 0.163381 + }, + { + "accuracy": 0.20098, + "f1": 0.162261, + "f1_weighted": 0.190541 + }, + { + "accuracy": 0.171569, + "f1": 0.137401, + "f1_weighted": 0.147877 + }, + { + "accuracy": 0.210784, + "f1": 0.19452, + "f1_weighted": 0.178146 + }, + { + "accuracy": 0.186275, + "f1": 0.146634, + "f1_weighted": 0.140963 + }, + { + "accuracy": 0.176471, + "f1": 0.152444, + "f1_weighted": 0.169529 + }, + { + "accuracy": 0.161765, + "f1": 0.120844, + "f1_weighted": 0.134896 + }, + { + "accuracy": 0.151961, + "f1": 0.102754, + "f1_weighted": 0.107552 + } + ], + "main_score": 0.183824, + "hf_subset": "hye_Armn", + "languages": [ + "hye-Armn" + ] + }, + { + "accuracy": 0.291176, + "f1": 0.280565, + "f1_weighted": 0.29426, + "scores_per_experiment": [ + { + "accuracy": 0.289216, + "f1": 0.27799, + "f1_weighted": 0.289154 + }, + { + "accuracy": 0.269608, + "f1": 0.269265, + "f1_weighted": 0.277933 + }, + { + "accuracy": 0.308824, + "f1": 0.295303, + "f1_weighted": 0.311727 + }, + { + "accuracy": 0.259804, + "f1": 0.241567, + "f1_weighted": 0.264039 + }, + { + "accuracy": 0.294118, + "f1": 0.28001, + "f1_weighted": 0.283565 + }, + { + "accuracy": 0.294118, + "f1": 0.294066, + "f1_weighted": 0.292326 + }, + { + "accuracy": 0.279412, + "f1": 0.256568, + "f1_weighted": 0.283806 + }, + { + "accuracy": 0.303922, + "f1": 0.295247, + "f1_weighted": 0.309354 + }, + { + "accuracy": 0.279412, + "f1": 0.270739, + "f1_weighted": 0.292244 + }, + { + "accuracy": 0.333333, + "f1": 0.324891, + "f1_weighted": 0.338456 + } + ], + "main_score": 0.291176, + "hf_subset": "ibo_Latn", + "languages": [ + "ibo-Latn" + ] + }, + { + "accuracy": 0.42451, + "f1": 0.403933, + "f1_weighted": 0.429075, + "scores_per_experiment": [ + { + "accuracy": 0.5, + "f1": 0.48143, + "f1_weighted": 0.503174 + }, + { + "accuracy": 0.480392, + "f1": 0.459921, + "f1_weighted": 0.481292 + }, + { + "accuracy": 0.392157, + "f1": 0.382183, + "f1_weighted": 0.39144 + }, + { + "accuracy": 0.416667, + "f1": 0.407769, + "f1_weighted": 0.423415 + }, + { + "accuracy": 0.397059, + "f1": 0.373342, + "f1_weighted": 0.40583 + }, + { + "accuracy": 0.406863, + "f1": 0.385063, + "f1_weighted": 0.409792 + }, + { + "accuracy": 0.406863, + "f1": 0.360089, + "f1_weighted": 0.414163 + }, + { + "accuracy": 0.460784, + "f1": 0.456237, + "f1_weighted": 0.473472 + }, + { + "accuracy": 0.377451, + "f1": 0.346181, + "f1_weighted": 0.385789 + }, + { + "accuracy": 0.406863, + "f1": 0.387117, + "f1_weighted": 0.402382 + } + ], + "main_score": 0.42451, + "hf_subset": "ilo_Latn", + "languages": [ + "ilo-Latn" + ] + }, + { + "accuracy": 0.42402, + "f1": 0.408372, + "f1_weighted": 0.427381, + "scores_per_experiment": [ + { + "accuracy": 0.470588, + "f1": 0.459585, + "f1_weighted": 0.475359 + }, + { + "accuracy": 0.446078, + "f1": 0.414442, + "f1_weighted": 0.445895 + }, + { + "accuracy": 0.431373, + "f1": 0.419327, + "f1_weighted": 0.441 + }, + { + "accuracy": 0.465686, + "f1": 0.451744, + "f1_weighted": 0.473901 + }, + { + "accuracy": 0.333333, + "f1": 0.325018, + "f1_weighted": 0.33181 + }, + { + "accuracy": 0.416667, + "f1": 0.401483, + "f1_weighted": 0.415961 + }, + { + "accuracy": 0.416667, + "f1": 0.37705, + "f1_weighted": 0.410325 + }, + { + "accuracy": 0.392157, + "f1": 0.399056, + "f1_weighted": 0.407191 + }, + { + "accuracy": 0.455882, + "f1": 0.418474, + "f1_weighted": 0.447767 + }, + { + "accuracy": 0.411765, + "f1": 0.417546, + "f1_weighted": 0.424597 + } + ], + "main_score": 0.42402, + "hf_subset": "ind_Latn", + "languages": [ + "ind-Latn" + ] + }, + { + "accuracy": 0.284314, + "f1": 0.261894, + "f1_weighted": 0.280643, + "scores_per_experiment": [ + { + "accuracy": 0.357843, + "f1": 0.313359, + "f1_weighted": 0.346192 + }, + { + "accuracy": 0.264706, + "f1": 0.259811, + "f1_weighted": 0.270808 + }, + { + "accuracy": 0.235294, + "f1": 0.213654, + "f1_weighted": 0.237452 + }, + { + "accuracy": 0.323529, + "f1": 0.299123, + "f1_weighted": 0.317115 + }, + { + "accuracy": 0.240196, + "f1": 0.213079, + "f1_weighted": 0.230792 + }, + { + "accuracy": 0.289216, + "f1": 0.278204, + "f1_weighted": 0.292363 + }, + { + "accuracy": 0.269608, + "f1": 0.218163, + "f1_weighted": 0.266591 + }, + { + "accuracy": 0.308824, + "f1": 0.295396, + "f1_weighted": 0.301616 + }, + { + "accuracy": 0.259804, + "f1": 0.243699, + "f1_weighted": 0.251042 + }, + { + "accuracy": 0.294118, + "f1": 0.284452, + "f1_weighted": 0.292462 + } + ], + "main_score": 0.284314, + "hf_subset": "isl_Latn", + "languages": [ + "isl-Latn" + ] + }, + { + "accuracy": 0.587255, + "f1": 0.561819, + "f1_weighted": 0.583977, + "scores_per_experiment": [ + { + "accuracy": 0.568627, + "f1": 0.555555, + "f1_weighted": 0.574264 + }, + { + "accuracy": 0.583333, + "f1": 0.548304, + "f1_weighted": 0.57965 + }, + { + "accuracy": 0.54902, + "f1": 0.546876, + "f1_weighted": 0.540395 + }, + { + "accuracy": 0.642157, + "f1": 0.603575, + "f1_weighted": 0.639307 + }, + { + "accuracy": 0.627451, + "f1": 0.606745, + "f1_weighted": 0.635905 + }, + { + "accuracy": 0.607843, + "f1": 0.579353, + "f1_weighted": 0.599788 + }, + { + "accuracy": 0.583333, + "f1": 0.558146, + "f1_weighted": 0.575229 + }, + { + "accuracy": 0.578431, + "f1": 0.573759, + "f1_weighted": 0.577919 + }, + { + "accuracy": 0.534314, + "f1": 0.492024, + "f1_weighted": 0.52366 + }, + { + "accuracy": 0.598039, + "f1": 0.553854, + "f1_weighted": 0.593656 + } + ], + "main_score": 0.587255, + "hf_subset": "ita_Latn", + "languages": [ + "ita-Latn" + ] + }, + { + "accuracy": 0.29951, + "f1": 0.286789, + "f1_weighted": 0.304012, + "scores_per_experiment": [ + { + "accuracy": 0.328431, + "f1": 0.314865, + "f1_weighted": 0.324018 + }, + { + "accuracy": 0.289216, + "f1": 0.287856, + "f1_weighted": 0.288138 + }, + { + "accuracy": 0.27451, + "f1": 0.272468, + "f1_weighted": 0.279142 + }, + { + "accuracy": 0.29902, + "f1": 0.284829, + "f1_weighted": 0.310389 + }, + { + "accuracy": 0.29902, + "f1": 0.295188, + "f1_weighted": 0.302306 + }, + { + "accuracy": 0.230392, + "f1": 0.205499, + "f1_weighted": 0.221171 + }, + { + "accuracy": 0.294118, + "f1": 0.283642, + "f1_weighted": 0.314125 + }, + { + "accuracy": 0.392157, + "f1": 0.359038, + "f1_weighted": 0.392027 + }, + { + "accuracy": 0.254902, + "f1": 0.245748, + "f1_weighted": 0.268859 + }, + { + "accuracy": 0.333333, + "f1": 0.318753, + "f1_weighted": 0.339945 + } + ], + "main_score": 0.29951, + "hf_subset": "jav_Latn", + "languages": [ + "jav-Latn" + ] + }, + { + "accuracy": 0.205882, + "f1": 0.131852, + "f1_weighted": 0.148065, + "scores_per_experiment": [ + { + "accuracy": 0.264706, + "f1": 0.158798, + "f1_weighted": 0.177917 + }, + { + "accuracy": 0.137255, + "f1": 0.085591, + "f1_weighted": 0.075657 + }, + { + "accuracy": 0.122549, + "f1": 0.103728, + "f1_weighted": 0.105078 + }, + { + "accuracy": 0.22549, + "f1": 0.146906, + "f1_weighted": 0.166709 + }, + { + "accuracy": 0.230392, + "f1": 0.131672, + "f1_weighted": 0.165604 + }, + { + "accuracy": 0.230392, + "f1": 0.145987, + "f1_weighted": 0.169901 + }, + { + "accuracy": 0.245098, + "f1": 0.134948, + "f1_weighted": 0.170754 + }, + { + "accuracy": 0.25, + "f1": 0.166808, + "f1_weighted": 0.182911 + }, + { + "accuracy": 0.196078, + "f1": 0.140808, + "f1_weighted": 0.164746 + }, + { + "accuracy": 0.156863, + "f1": 0.103281, + "f1_weighted": 0.10137 + } + ], + "main_score": 0.205882, + "hf_subset": "jpn_Jpan", + "languages": [ + "jpn-Jpan" + ] + }, + { + "accuracy": 0.240196, + "f1": 0.229376, + "f1_weighted": 0.242206, + "scores_per_experiment": [ + { + "accuracy": 0.279412, + "f1": 0.25504, + "f1_weighted": 0.282879 + }, + { + "accuracy": 0.20098, + "f1": 0.20428, + "f1_weighted": 0.204146 + }, + { + "accuracy": 0.181373, + "f1": 0.164614, + "f1_weighted": 0.198021 + }, + { + "accuracy": 0.254902, + "f1": 0.241177, + "f1_weighted": 0.255877 + }, + { + "accuracy": 0.29902, + "f1": 0.287505, + "f1_weighted": 0.293669 + }, + { + "accuracy": 0.196078, + "f1": 0.197053, + "f1_weighted": 0.189518 + }, + { + "accuracy": 0.27451, + "f1": 0.264895, + "f1_weighted": 0.27963 + }, + { + "accuracy": 0.279412, + "f1": 0.281574, + "f1_weighted": 0.287952 + }, + { + "accuracy": 0.210784, + "f1": 0.17701, + "f1_weighted": 0.204032 + }, + { + "accuracy": 0.22549, + "f1": 0.22061, + "f1_weighted": 0.226332 + } + ], + "main_score": 0.240196, + "hf_subset": "kab_Latn", + "languages": [ + "kab-Latn" + ] + }, + { + "accuracy": 0.370098, + "f1": 0.352428, + "f1_weighted": 0.372977, + "scores_per_experiment": [ + { + "accuracy": 0.352941, + "f1": 0.325366, + "f1_weighted": 0.35828 + }, + { + "accuracy": 0.397059, + "f1": 0.382738, + "f1_weighted": 0.39589 + }, + { + "accuracy": 0.416667, + "f1": 0.394161, + "f1_weighted": 0.415422 + }, + { + "accuracy": 0.279412, + "f1": 0.264346, + "f1_weighted": 0.28381 + }, + { + "accuracy": 0.377451, + "f1": 0.368746, + "f1_weighted": 0.379301 + }, + { + "accuracy": 0.318627, + "f1": 0.314361, + "f1_weighted": 0.321809 + }, + { + "accuracy": 0.431373, + "f1": 0.398892, + "f1_weighted": 0.442425 + }, + { + "accuracy": 0.352941, + "f1": 0.361896, + "f1_weighted": 0.358565 + }, + { + "accuracy": 0.392157, + "f1": 0.364164, + "f1_weighted": 0.395994 + }, + { + "accuracy": 0.382353, + "f1": 0.34961, + "f1_weighted": 0.378273 + } + ], + "main_score": 0.370098, + "hf_subset": "kac_Latn", + "languages": [ + "kac-Latn" + ] + }, + { + "accuracy": 0.304902, + "f1": 0.295183, + "f1_weighted": 0.302964, + "scores_per_experiment": [ + { + "accuracy": 0.372549, + "f1": 0.346308, + "f1_weighted": 0.374037 + }, + { + "accuracy": 0.279412, + "f1": 0.274565, + "f1_weighted": 0.271829 + }, + { + "accuracy": 0.29902, + "f1": 0.28814, + "f1_weighted": 0.298612 + }, + { + "accuracy": 0.294118, + "f1": 0.290507, + "f1_weighted": 0.2949 + }, + { + "accuracy": 0.323529, + "f1": 0.308609, + "f1_weighted": 0.324252 + }, + { + "accuracy": 0.240196, + "f1": 0.236331, + "f1_weighted": 0.235393 + }, + { + "accuracy": 0.254902, + "f1": 0.241911, + "f1_weighted": 0.244966 + }, + { + "accuracy": 0.318627, + "f1": 0.312212, + "f1_weighted": 0.317965 + }, + { + "accuracy": 0.323529, + "f1": 0.311094, + "f1_weighted": 0.324952 + }, + { + "accuracy": 0.343137, + "f1": 0.342149, + "f1_weighted": 0.342734 + } + ], + "main_score": 0.304902, + "hf_subset": "kam_Latn", + "languages": [ + "kam-Latn" + ] + }, + { + "accuracy": 0.177451, + "f1": 0.137776, + "f1_weighted": 0.143359, + "scores_per_experiment": [ + { + "accuracy": 0.240196, + "f1": 0.173457, + "f1_weighted": 0.189787 + }, + { + "accuracy": 0.147059, + "f1": 0.110159, + "f1_weighted": 0.110616 + }, + { + "accuracy": 0.156863, + "f1": 0.149206, + "f1_weighted": 0.114531 + }, + { + "accuracy": 0.151961, + "f1": 0.115291, + "f1_weighted": 0.101828 + }, + { + "accuracy": 0.127451, + "f1": 0.099774, + "f1_weighted": 0.091002 + }, + { + "accuracy": 0.240196, + "f1": 0.205218, + "f1_weighted": 0.223397 + }, + { + "accuracy": 0.22549, + "f1": 0.145173, + "f1_weighted": 0.175861 + }, + { + "accuracy": 0.176471, + "f1": 0.141817, + "f1_weighted": 0.164328 + }, + { + "accuracy": 0.166667, + "f1": 0.137167, + "f1_weighted": 0.150186 + }, + { + "accuracy": 0.142157, + "f1": 0.100495, + "f1_weighted": 0.112059 + } + ], + "main_score": 0.177451, + "hf_subset": "kan_Knda", + "languages": [ + "kan-Knda" + ] + }, + { + "accuracy": 0.209804, + "f1": 0.170613, + "f1_weighted": 0.185362, + "scores_per_experiment": [ + { + "accuracy": 0.196078, + "f1": 0.174885, + "f1_weighted": 0.18181 + }, + { + "accuracy": 0.156863, + "f1": 0.131447, + "f1_weighted": 0.116689 + }, + { + "accuracy": 0.186275, + "f1": 0.16236, + "f1_weighted": 0.155303 + }, + { + "accuracy": 0.205882, + "f1": 0.15491, + "f1_weighted": 0.181352 + }, + { + "accuracy": 0.156863, + "f1": 0.156489, + "f1_weighted": 0.14165 + }, + { + "accuracy": 0.240196, + "f1": 0.199765, + "f1_weighted": 0.228154 + }, + { + "accuracy": 0.264706, + "f1": 0.178226, + "f1_weighted": 0.21667 + }, + { + "accuracy": 0.254902, + "f1": 0.218157, + "f1_weighted": 0.247638 + }, + { + "accuracy": 0.22549, + "f1": 0.143391, + "f1_weighted": 0.159253 + }, + { + "accuracy": 0.210784, + "f1": 0.1865, + "f1_weighted": 0.225105 + } + ], + "main_score": 0.209804, + "hf_subset": "kas_Deva", + "languages": [ + "kas-Deva" + ] + }, + { + "accuracy": 0.170098, + "f1": 0.125885, + "f1_weighted": 0.133544, + "scores_per_experiment": [ + { + "accuracy": 0.215686, + "f1": 0.116491, + "f1_weighted": 0.134449 + }, + { + "accuracy": 0.147059, + "f1": 0.101668, + "f1_weighted": 0.087125 + }, + { + "accuracy": 0.122549, + "f1": 0.087347, + "f1_weighted": 0.076489 + }, + { + "accuracy": 0.22549, + "f1": 0.164158, + "f1_weighted": 0.192618 + }, + { + "accuracy": 0.132353, + "f1": 0.127606, + "f1_weighted": 0.094085 + }, + { + "accuracy": 0.171569, + "f1": 0.157658, + "f1_weighted": 0.163323 + }, + { + "accuracy": 0.205882, + "f1": 0.153375, + "f1_weighted": 0.194054 + }, + { + "accuracy": 0.196078, + "f1": 0.152196, + "f1_weighted": 0.177598 + }, + { + "accuracy": 0.161765, + "f1": 0.11829, + "f1_weighted": 0.127852 + }, + { + "accuracy": 0.122549, + "f1": 0.080058, + "f1_weighted": 0.08785 + } + ], + "main_score": 0.170098, + "hf_subset": "kat_Geor", + "languages": [ + "kat-Geor" + ] + }, + { + "accuracy": 0.215196, + "f1": 0.196359, + "f1_weighted": 0.208618, + "scores_per_experiment": [ + { + "accuracy": 0.284314, + "f1": 0.191729, + "f1_weighted": 0.237442 + }, + { + "accuracy": 0.171569, + "f1": 0.15992, + "f1_weighted": 0.16216 + }, + { + "accuracy": 0.215686, + "f1": 0.220602, + "f1_weighted": 0.210513 + }, + { + "accuracy": 0.220588, + "f1": 0.203936, + "f1_weighted": 0.215213 + }, + { + "accuracy": 0.205882, + "f1": 0.206734, + "f1_weighted": 0.206682 + }, + { + "accuracy": 0.176471, + "f1": 0.171341, + "f1_weighted": 0.169686 + }, + { + "accuracy": 0.171569, + "f1": 0.178315, + "f1_weighted": 0.174244 + }, + { + "accuracy": 0.230392, + "f1": 0.20185, + "f1_weighted": 0.228448 + }, + { + "accuracy": 0.254902, + "f1": 0.245199, + "f1_weighted": 0.268286 + }, + { + "accuracy": 0.220588, + "f1": 0.183969, + "f1_weighted": 0.21351 + } + ], + "main_score": 0.215196, + "hf_subset": "kaz_Cyrl", + "languages": [ + "kaz-Cyrl" + ] + }, + { + "accuracy": 0.323529, + "f1": 0.310954, + "f1_weighted": 0.322769, + "scores_per_experiment": [ + { + "accuracy": 0.313725, + "f1": 0.299059, + "f1_weighted": 0.325673 + }, + { + "accuracy": 0.254902, + "f1": 0.253852, + "f1_weighted": 0.250907 + }, + { + "accuracy": 0.303922, + "f1": 0.297046, + "f1_weighted": 0.292759 + }, + { + "accuracy": 0.338235, + "f1": 0.335091, + "f1_weighted": 0.342433 + }, + { + "accuracy": 0.357843, + "f1": 0.341086, + "f1_weighted": 0.361382 + }, + { + "accuracy": 0.318627, + "f1": 0.315579, + "f1_weighted": 0.321243 + }, + { + "accuracy": 0.352941, + "f1": 0.32809, + "f1_weighted": 0.350526 + }, + { + "accuracy": 0.338235, + "f1": 0.32526, + "f1_weighted": 0.342074 + }, + { + "accuracy": 0.333333, + "f1": 0.303548, + "f1_weighted": 0.325598 + }, + { + "accuracy": 0.323529, + "f1": 0.310926, + "f1_weighted": 0.315094 + } + ], + "main_score": 0.323529, + "hf_subset": "kbp_Latn", + "languages": [ + "kbp-Latn" + ] + }, + { + "accuracy": 0.416667, + "f1": 0.397397, + "f1_weighted": 0.417653, + "scores_per_experiment": [ + { + "accuracy": 0.421569, + "f1": 0.402643, + "f1_weighted": 0.423135 + }, + { + "accuracy": 0.455882, + "f1": 0.445229, + "f1_weighted": 0.450746 + }, + { + "accuracy": 0.416667, + "f1": 0.399394, + "f1_weighted": 0.413163 + }, + { + "accuracy": 0.431373, + "f1": 0.409755, + "f1_weighted": 0.43191 + }, + { + "accuracy": 0.377451, + "f1": 0.363113, + "f1_weighted": 0.386059 + }, + { + "accuracy": 0.411765, + "f1": 0.379187, + "f1_weighted": 0.403856 + }, + { + "accuracy": 0.397059, + "f1": 0.37551, + "f1_weighted": 0.401376 + }, + { + "accuracy": 0.401961, + "f1": 0.380445, + "f1_weighted": 0.413863 + }, + { + "accuracy": 0.436275, + "f1": 0.407758, + "f1_weighted": 0.436522 + }, + { + "accuracy": 0.416667, + "f1": 0.410932, + "f1_weighted": 0.415897 + } + ], + "main_score": 0.416667, + "hf_subset": "kea_Latn", + "languages": [ + "kea-Latn" + ] + }, + { + "accuracy": 0.207843, + "f1": 0.183989, + "f1_weighted": 0.195953, + "scores_per_experiment": [ + { + "accuracy": 0.254902, + "f1": 0.198739, + "f1_weighted": 0.228438 + }, + { + "accuracy": 0.181373, + "f1": 0.155539, + "f1_weighted": 0.15876 + }, + { + "accuracy": 0.166667, + "f1": 0.171371, + "f1_weighted": 0.163699 + }, + { + "accuracy": 0.191176, + "f1": 0.165413, + "f1_weighted": 0.176114 + }, + { + "accuracy": 0.196078, + "f1": 0.189734, + "f1_weighted": 0.181233 + }, + { + "accuracy": 0.22549, + "f1": 0.209658, + "f1_weighted": 0.221051 + }, + { + "accuracy": 0.235294, + "f1": 0.196443, + "f1_weighted": 0.209758 + }, + { + "accuracy": 0.25, + "f1": 0.228479, + "f1_weighted": 0.249779 + }, + { + "accuracy": 0.210784, + "f1": 0.18351, + "f1_weighted": 0.206633 + }, + { + "accuracy": 0.166667, + "f1": 0.141003, + "f1_weighted": 0.164063 + } + ], + "main_score": 0.207843, + "hf_subset": "khk_Cyrl", + "languages": [ + "khk-Cyrl" + ] + }, + { + "accuracy": 0.253431, + "f1": 0.216476, + "f1_weighted": 0.22905, + "scores_per_experiment": [ + { + "accuracy": 0.289216, + "f1": 0.252604, + "f1_weighted": 0.258354 + }, + { + "accuracy": 0.264706, + "f1": 0.2369, + "f1_weighted": 0.251196 + }, + { + "accuracy": 0.22549, + "f1": 0.18541, + "f1_weighted": 0.21061 + }, + { + "accuracy": 0.210784, + "f1": 0.176879, + "f1_weighted": 0.171252 + }, + { + "accuracy": 0.294118, + "f1": 0.250669, + "f1_weighted": 0.271966 + }, + { + "accuracy": 0.22549, + "f1": 0.163649, + "f1_weighted": 0.174806 + }, + { + "accuracy": 0.289216, + "f1": 0.247207, + "f1_weighted": 0.266427 + }, + { + "accuracy": 0.289216, + "f1": 0.267556, + "f1_weighted": 0.277712 + }, + { + "accuracy": 0.245098, + "f1": 0.199789, + "f1_weighted": 0.219976 + }, + { + "accuracy": 0.20098, + "f1": 0.184101, + "f1_weighted": 0.188201 + } + ], + "main_score": 0.253431, + "hf_subset": "khm_Khmr", + "languages": [ + "khm-Khmr" + ] + }, + { + "accuracy": 0.384804, + "f1": 0.3681, + "f1_weighted": 0.393176, + "scores_per_experiment": [ + { + "accuracy": 0.411765, + "f1": 0.392535, + "f1_weighted": 0.420349 + }, + { + "accuracy": 0.401961, + "f1": 0.392078, + "f1_weighted": 0.399942 + }, + { + "accuracy": 0.411765, + "f1": 0.377122, + "f1_weighted": 0.408417 + }, + { + "accuracy": 0.343137, + "f1": 0.324593, + "f1_weighted": 0.354641 + }, + { + "accuracy": 0.382353, + "f1": 0.373724, + "f1_weighted": 0.393485 + }, + { + "accuracy": 0.357843, + "f1": 0.34147, + "f1_weighted": 0.372735 + }, + { + "accuracy": 0.372549, + "f1": 0.361972, + "f1_weighted": 0.378503 + }, + { + "accuracy": 0.401961, + "f1": 0.377194, + "f1_weighted": 0.41341 + }, + { + "accuracy": 0.362745, + "f1": 0.331955, + "f1_weighted": 0.370192 + }, + { + "accuracy": 0.401961, + "f1": 0.408357, + "f1_weighted": 0.420085 + } + ], + "main_score": 0.384804, + "hf_subset": "kik_Latn", + "languages": [ + "kik-Latn" + ] + }, + { + "accuracy": 0.313725, + "f1": 0.3, + "f1_weighted": 0.315026, + "scores_per_experiment": [ + { + "accuracy": 0.323529, + "f1": 0.298167, + "f1_weighted": 0.326498 + }, + { + "accuracy": 0.284314, + "f1": 0.27978, + "f1_weighted": 0.264989 + }, + { + "accuracy": 0.289216, + "f1": 0.28268, + "f1_weighted": 0.294325 + }, + { + "accuracy": 0.338235, + "f1": 0.30685, + "f1_weighted": 0.330933 + }, + { + "accuracy": 0.29902, + "f1": 0.308325, + "f1_weighted": 0.311864 + }, + { + "accuracy": 0.289216, + "f1": 0.283385, + "f1_weighted": 0.285061 + }, + { + "accuracy": 0.357843, + "f1": 0.341464, + "f1_weighted": 0.361322 + }, + { + "accuracy": 0.318627, + "f1": 0.315934, + "f1_weighted": 0.3252 + }, + { + "accuracy": 0.328431, + "f1": 0.292208, + "f1_weighted": 0.323667 + }, + { + "accuracy": 0.308824, + "f1": 0.291207, + "f1_weighted": 0.326402 + } + ], + "main_score": 0.313725, + "hf_subset": "kin_Latn", + "languages": [ + "kin-Latn" + ] + }, + { + "accuracy": 0.209314, + "f1": 0.184005, + "f1_weighted": 0.202407, + "scores_per_experiment": [ + { + "accuracy": 0.254902, + "f1": 0.1845, + "f1_weighted": 0.22106 + }, + { + "accuracy": 0.205882, + "f1": 0.18962, + "f1_weighted": 0.203757 + }, + { + "accuracy": 0.176471, + "f1": 0.162885, + "f1_weighted": 0.179726 + }, + { + "accuracy": 0.245098, + "f1": 0.212464, + "f1_weighted": 0.24857 + }, + { + "accuracy": 0.196078, + "f1": 0.185278, + "f1_weighted": 0.178538 + }, + { + "accuracy": 0.215686, + "f1": 0.205974, + "f1_weighted": 0.212605 + }, + { + "accuracy": 0.186275, + "f1": 0.191095, + "f1_weighted": 0.184439 + }, + { + "accuracy": 0.240196, + "f1": 0.202033, + "f1_weighted": 0.237964 + }, + { + "accuracy": 0.176471, + "f1": 0.149775, + "f1_weighted": 0.177019 + }, + { + "accuracy": 0.196078, + "f1": 0.156423, + "f1_weighted": 0.18039 + } + ], + "main_score": 0.209314, + "hf_subset": "kir_Cyrl", + "languages": [ + "kir-Cyrl" + ] + }, + { + "accuracy": 0.295098, + "f1": 0.2813, + "f1_weighted": 0.292837, + "scores_per_experiment": [ + { + "accuracy": 0.245098, + "f1": 0.239239, + "f1_weighted": 0.253648 + }, + { + "accuracy": 0.289216, + "f1": 0.269683, + "f1_weighted": 0.283364 + }, + { + "accuracy": 0.294118, + "f1": 0.285197, + "f1_weighted": 0.277904 + }, + { + "accuracy": 0.303922, + "f1": 0.301416, + "f1_weighted": 0.309922 + }, + { + "accuracy": 0.27451, + "f1": 0.250919, + "f1_weighted": 0.268919 + }, + { + "accuracy": 0.352941, + "f1": 0.336769, + "f1_weighted": 0.352062 + }, + { + "accuracy": 0.333333, + "f1": 0.315902, + "f1_weighted": 0.331065 + }, + { + "accuracy": 0.328431, + "f1": 0.311581, + "f1_weighted": 0.32775 + }, + { + "accuracy": 0.240196, + "f1": 0.220902, + "f1_weighted": 0.241704 + }, + { + "accuracy": 0.289216, + "f1": 0.281395, + "f1_weighted": 0.28203 + } + ], + "main_score": 0.295098, + "hf_subset": "kmb_Latn", + "languages": [ + "kmb-Latn" + ] + }, + { + "accuracy": 0.268137, + "f1": 0.250493, + "f1_weighted": 0.26855, + "scores_per_experiment": [ + { + "accuracy": 0.284314, + "f1": 0.255937, + "f1_weighted": 0.291449 + }, + { + "accuracy": 0.220588, + "f1": 0.211601, + "f1_weighted": 0.207102 + }, + { + "accuracy": 0.254902, + "f1": 0.23244, + "f1_weighted": 0.254157 + }, + { + "accuracy": 0.284314, + "f1": 0.267336, + "f1_weighted": 0.28331 + }, + { + "accuracy": 0.245098, + "f1": 0.229452, + "f1_weighted": 0.252014 + }, + { + "accuracy": 0.25, + "f1": 0.236026, + "f1_weighted": 0.252008 + }, + { + "accuracy": 0.230392, + "f1": 0.226031, + "f1_weighted": 0.231062 + }, + { + "accuracy": 0.328431, + "f1": 0.296312, + "f1_weighted": 0.320783 + }, + { + "accuracy": 0.308824, + "f1": 0.2973, + "f1_weighted": 0.308133 + }, + { + "accuracy": 0.27451, + "f1": 0.252494, + "f1_weighted": 0.285479 + } + ], + "main_score": 0.268137, + "hf_subset": "kmr_Latn", + "languages": [ + "kmr-Latn" + ] + }, + { + "accuracy": 0.363725, + "f1": 0.352144, + "f1_weighted": 0.368489, + "scores_per_experiment": [ + { + "accuracy": 0.372549, + "f1": 0.354566, + "f1_weighted": 0.377197 + }, + { + "accuracy": 0.343137, + "f1": 0.32241, + "f1_weighted": 0.342385 + }, + { + "accuracy": 0.377451, + "f1": 0.381287, + "f1_weighted": 0.387153 + }, + { + "accuracy": 0.397059, + "f1": 0.386315, + "f1_weighted": 0.405401 + }, + { + "accuracy": 0.352941, + "f1": 0.349243, + "f1_weighted": 0.358785 + }, + { + "accuracy": 0.367647, + "f1": 0.351368, + "f1_weighted": 0.373148 + }, + { + "accuracy": 0.343137, + "f1": 0.332332, + "f1_weighted": 0.346921 + }, + { + "accuracy": 0.352941, + "f1": 0.350091, + "f1_weighted": 0.358163 + }, + { + "accuracy": 0.357843, + "f1": 0.334233, + "f1_weighted": 0.358467 + }, + { + "accuracy": 0.372549, + "f1": 0.359596, + "f1_weighted": 0.377269 + } + ], + "main_score": 0.363725, + "hf_subset": "knc_Latn", + "languages": [ + "knc-Latn" + ] + }, + { + "accuracy": 0.383824, + "f1": 0.36617, + "f1_weighted": 0.384696, + "scores_per_experiment": [ + { + "accuracy": 0.431373, + "f1": 0.420731, + "f1_weighted": 0.43559 + }, + { + "accuracy": 0.377451, + "f1": 0.371552, + "f1_weighted": 0.373193 + }, + { + "accuracy": 0.401961, + "f1": 0.384009, + "f1_weighted": 0.395 + }, + { + "accuracy": 0.387255, + "f1": 0.366994, + "f1_weighted": 0.385925 + }, + { + "accuracy": 0.362745, + "f1": 0.352749, + "f1_weighted": 0.365689 + }, + { + "accuracy": 0.362745, + "f1": 0.336798, + "f1_weighted": 0.361017 + }, + { + "accuracy": 0.401961, + "f1": 0.380103, + "f1_weighted": 0.410325 + }, + { + "accuracy": 0.348039, + "f1": 0.339942, + "f1_weighted": 0.346401 + }, + { + "accuracy": 0.338235, + "f1": 0.300817, + "f1_weighted": 0.342271 + }, + { + "accuracy": 0.426471, + "f1": 0.408003, + "f1_weighted": 0.431545 + } + ], + "main_score": 0.383824, + "hf_subset": "kon_Latn", + "languages": [ + "kon-Latn" + ] + }, + { + "accuracy": 0.204902, + "f1": 0.160686, + "f1_weighted": 0.179053, + "scores_per_experiment": [ + { + "accuracy": 0.220588, + "f1": 0.160191, + "f1_weighted": 0.180397 + }, + { + "accuracy": 0.196078, + "f1": 0.163939, + "f1_weighted": 0.160014 + }, + { + "accuracy": 0.181373, + "f1": 0.183571, + "f1_weighted": 0.179462 + }, + { + "accuracy": 0.230392, + "f1": 0.205613, + "f1_weighted": 0.214656 + }, + { + "accuracy": 0.186275, + "f1": 0.153515, + "f1_weighted": 0.176684 + }, + { + "accuracy": 0.20098, + "f1": 0.139445, + "f1_weighted": 0.158969 + }, + { + "accuracy": 0.235294, + "f1": 0.175808, + "f1_weighted": 0.209745 + }, + { + "accuracy": 0.220588, + "f1": 0.186815, + "f1_weighted": 0.212296 + }, + { + "accuracy": 0.245098, + "f1": 0.142387, + "f1_weighted": 0.179193 + }, + { + "accuracy": 0.132353, + "f1": 0.095576, + "f1_weighted": 0.119111 + } + ], + "main_score": 0.204902, + "hf_subset": "kor_Hang", + "languages": [ + "kor-Hang" + ] + }, + { + "accuracy": 0.260294, + "f1": 0.222062, + "f1_weighted": 0.2377, + "scores_per_experiment": [ + { + "accuracy": 0.303922, + "f1": 0.26915, + "f1_weighted": 0.277019 + }, + { + "accuracy": 0.27451, + "f1": 0.24403, + "f1_weighted": 0.269879 + }, + { + "accuracy": 0.230392, + "f1": 0.184805, + "f1_weighted": 0.206889 + }, + { + "accuracy": 0.210784, + "f1": 0.201507, + "f1_weighted": 0.200724 + }, + { + "accuracy": 0.294118, + "f1": 0.213087, + "f1_weighted": 0.258275 + }, + { + "accuracy": 0.259804, + "f1": 0.193095, + "f1_weighted": 0.203219 + }, + { + "accuracy": 0.259804, + "f1": 0.184994, + "f1_weighted": 0.207096 + }, + { + "accuracy": 0.308824, + "f1": 0.308928, + "f1_weighted": 0.301124 + }, + { + "accuracy": 0.215686, + "f1": 0.203345, + "f1_weighted": 0.215379 + }, + { + "accuracy": 0.245098, + "f1": 0.217677, + "f1_weighted": 0.2374 + } + ], + "main_score": 0.260294, + "hf_subset": "lao_Laoo", + "languages": [ + "lao-Laoo" + ] + }, + { + "accuracy": 0.421569, + "f1": 0.396368, + "f1_weighted": 0.422588, + "scores_per_experiment": [ + { + "accuracy": 0.426471, + "f1": 0.390709, + "f1_weighted": 0.423975 + }, + { + "accuracy": 0.426471, + "f1": 0.374169, + "f1_weighted": 0.422445 + }, + { + "accuracy": 0.377451, + "f1": 0.36246, + "f1_weighted": 0.37873 + }, + { + "accuracy": 0.480392, + "f1": 0.445114, + "f1_weighted": 0.46747 + }, + { + "accuracy": 0.465686, + "f1": 0.435802, + "f1_weighted": 0.48018 + }, + { + "accuracy": 0.470588, + "f1": 0.453162, + "f1_weighted": 0.472759 + }, + { + "accuracy": 0.372549, + "f1": 0.361145, + "f1_weighted": 0.366309 + }, + { + "accuracy": 0.416667, + "f1": 0.406677, + "f1_weighted": 0.414434 + }, + { + "accuracy": 0.348039, + "f1": 0.326047, + "f1_weighted": 0.360699 + }, + { + "accuracy": 0.431373, + "f1": 0.408397, + "f1_weighted": 0.438874 + } + ], + "main_score": 0.421569, + "hf_subset": "lij_Latn", + "languages": [ + "lij-Latn" + ] + }, + { + "accuracy": 0.423039, + "f1": 0.414236, + "f1_weighted": 0.42539, + "scores_per_experiment": [ + { + "accuracy": 0.460784, + "f1": 0.446428, + "f1_weighted": 0.462036 + }, + { + "accuracy": 0.382353, + "f1": 0.377275, + "f1_weighted": 0.379203 + }, + { + "accuracy": 0.426471, + "f1": 0.420309, + "f1_weighted": 0.421434 + }, + { + "accuracy": 0.446078, + "f1": 0.447905, + "f1_weighted": 0.447038 + }, + { + "accuracy": 0.446078, + "f1": 0.452045, + "f1_weighted": 0.45831 + }, + { + "accuracy": 0.480392, + "f1": 0.458688, + "f1_weighted": 0.480877 + }, + { + "accuracy": 0.367647, + "f1": 0.361656, + "f1_weighted": 0.375964 + }, + { + "accuracy": 0.397059, + "f1": 0.397502, + "f1_weighted": 0.403225 + }, + { + "accuracy": 0.382353, + "f1": 0.351921, + "f1_weighted": 0.3764 + }, + { + "accuracy": 0.441176, + "f1": 0.428629, + "f1_weighted": 0.449417 + } + ], + "main_score": 0.423039, + "hf_subset": "lim_Latn", + "languages": [ + "lim-Latn" + ] + }, + { + "accuracy": 0.383333, + "f1": 0.364866, + "f1_weighted": 0.380064, + "scores_per_experiment": [ + { + "accuracy": 0.357843, + "f1": 0.350227, + "f1_weighted": 0.360938 + }, + { + "accuracy": 0.397059, + "f1": 0.382424, + "f1_weighted": 0.403135 + }, + { + "accuracy": 0.387255, + "f1": 0.362686, + "f1_weighted": 0.374826 + }, + { + "accuracy": 0.446078, + "f1": 0.437714, + "f1_weighted": 0.448606 + }, + { + "accuracy": 0.431373, + "f1": 0.40474, + "f1_weighted": 0.43626 + }, + { + "accuracy": 0.333333, + "f1": 0.332569, + "f1_weighted": 0.329338 + }, + { + "accuracy": 0.362745, + "f1": 0.324521, + "f1_weighted": 0.332671 + }, + { + "accuracy": 0.343137, + "f1": 0.321365, + "f1_weighted": 0.338294 + }, + { + "accuracy": 0.367647, + "f1": 0.339828, + "f1_weighted": 0.368916 + }, + { + "accuracy": 0.406863, + "f1": 0.39259, + "f1_weighted": 0.407655 + } + ], + "main_score": 0.383333, + "hf_subset": "lin_Latn", + "languages": [ + "lin-Latn" + ] + }, + { + "accuracy": 0.325, + "f1": 0.310025, + "f1_weighted": 0.324884, + "scores_per_experiment": [ + { + "accuracy": 0.338235, + "f1": 0.340384, + "f1_weighted": 0.341226 + }, + { + "accuracy": 0.289216, + "f1": 0.266085, + "f1_weighted": 0.28486 + }, + { + "accuracy": 0.352941, + "f1": 0.335628, + "f1_weighted": 0.350956 + }, + { + "accuracy": 0.357843, + "f1": 0.329633, + "f1_weighted": 0.356442 + }, + { + "accuracy": 0.328431, + "f1": 0.303252, + "f1_weighted": 0.330638 + }, + { + "accuracy": 0.313725, + "f1": 0.308768, + "f1_weighted": 0.312854 + }, + { + "accuracy": 0.27451, + "f1": 0.258676, + "f1_weighted": 0.271065 + }, + { + "accuracy": 0.362745, + "f1": 0.358937, + "f1_weighted": 0.369377 + }, + { + "accuracy": 0.338235, + "f1": 0.304448, + "f1_weighted": 0.331379 + }, + { + "accuracy": 0.294118, + "f1": 0.294437, + "f1_weighted": 0.300044 + } + ], + "main_score": 0.325, + "hf_subset": "lit_Latn", + "languages": [ + "lit-Latn" + ] + }, + { + "accuracy": 0.42451, + "f1": 0.406267, + "f1_weighted": 0.420053, + "scores_per_experiment": [ + { + "accuracy": 0.5, + "f1": 0.491175, + "f1_weighted": 0.499005 + }, + { + "accuracy": 0.406863, + "f1": 0.37412, + "f1_weighted": 0.398581 + }, + { + "accuracy": 0.416667, + "f1": 0.398267, + "f1_weighted": 0.405079 + }, + { + "accuracy": 0.431373, + "f1": 0.405056, + "f1_weighted": 0.425502 + }, + { + "accuracy": 0.406863, + "f1": 0.381414, + "f1_weighted": 0.393116 + }, + { + "accuracy": 0.45098, + "f1": 0.439249, + "f1_weighted": 0.447917 + }, + { + "accuracy": 0.455882, + "f1": 0.433805, + "f1_weighted": 0.450082 + }, + { + "accuracy": 0.377451, + "f1": 0.369708, + "f1_weighted": 0.380112 + }, + { + "accuracy": 0.397059, + "f1": 0.378686, + "f1_weighted": 0.399519 + }, + { + "accuracy": 0.401961, + "f1": 0.39119, + "f1_weighted": 0.401615 + } + ], + "main_score": 0.42451, + "hf_subset": "lmo_Latn", + "languages": [ + "lmo-Latn" + ] + }, + { + "accuracy": 0.323039, + "f1": 0.308869, + "f1_weighted": 0.32558, + "scores_per_experiment": [ + { + "accuracy": 0.382353, + "f1": 0.363607, + "f1_weighted": 0.385509 + }, + { + "accuracy": 0.328431, + "f1": 0.305332, + "f1_weighted": 0.322632 + }, + { + "accuracy": 0.348039, + "f1": 0.33825, + "f1_weighted": 0.356531 + }, + { + "accuracy": 0.279412, + "f1": 0.274978, + "f1_weighted": 0.277148 + }, + { + "accuracy": 0.323529, + "f1": 0.318059, + "f1_weighted": 0.329356 + }, + { + "accuracy": 0.352941, + "f1": 0.345197, + "f1_weighted": 0.351595 + }, + { + "accuracy": 0.348039, + "f1": 0.313911, + "f1_weighted": 0.358111 + }, + { + "accuracy": 0.333333, + "f1": 0.320855, + "f1_weighted": 0.338386 + }, + { + "accuracy": 0.22549, + "f1": 0.212494, + "f1_weighted": 0.228014 + }, + { + "accuracy": 0.308824, + "f1": 0.296002, + "f1_weighted": 0.308514 + } + ], + "main_score": 0.323039, + "hf_subset": "ltg_Latn", + "languages": [ + "ltg-Latn" + ] + }, + { + "accuracy": 0.455882, + "f1": 0.448279, + "f1_weighted": 0.457569, + "scores_per_experiment": [ + { + "accuracy": 0.504902, + "f1": 0.483719, + "f1_weighted": 0.498898 + }, + { + "accuracy": 0.421569, + "f1": 0.426107, + "f1_weighted": 0.417746 + }, + { + "accuracy": 0.441176, + "f1": 0.422904, + "f1_weighted": 0.442994 + }, + { + "accuracy": 0.495098, + "f1": 0.487359, + "f1_weighted": 0.506497 + }, + { + "accuracy": 0.436275, + "f1": 0.429113, + "f1_weighted": 0.441909 + }, + { + "accuracy": 0.470588, + "f1": 0.452056, + "f1_weighted": 0.465257 + }, + { + "accuracy": 0.436275, + "f1": 0.432757, + "f1_weighted": 0.443691 + }, + { + "accuracy": 0.421569, + "f1": 0.432298, + "f1_weighted": 0.428386 + }, + { + "accuracy": 0.460784, + "f1": 0.445297, + "f1_weighted": 0.452624 + }, + { + "accuracy": 0.470588, + "f1": 0.471182, + "f1_weighted": 0.477685 + } + ], + "main_score": 0.455882, + "hf_subset": "ltz_Latn", + "languages": [ + "ltz-Latn" + ] + }, + { + "accuracy": 0.341667, + "f1": 0.326007, + "f1_weighted": 0.343853, + "scores_per_experiment": [ + { + "accuracy": 0.367647, + "f1": 0.333773, + "f1_weighted": 0.377992 + }, + { + "accuracy": 0.348039, + "f1": 0.338235, + "f1_weighted": 0.343175 + }, + { + "accuracy": 0.29902, + "f1": 0.289456, + "f1_weighted": 0.301761 + }, + { + "accuracy": 0.357843, + "f1": 0.338387, + "f1_weighted": 0.365819 + }, + { + "accuracy": 0.362745, + "f1": 0.338421, + "f1_weighted": 0.362125 + }, + { + "accuracy": 0.338235, + "f1": 0.328454, + "f1_weighted": 0.338349 + }, + { + "accuracy": 0.357843, + "f1": 0.361489, + "f1_weighted": 0.372059 + }, + { + "accuracy": 0.348039, + "f1": 0.338511, + "f1_weighted": 0.352207 + }, + { + "accuracy": 0.303922, + "f1": 0.271098, + "f1_weighted": 0.298542 + }, + { + "accuracy": 0.333333, + "f1": 0.322243, + "f1_weighted": 0.326502 + } + ], + "main_score": 0.341667, + "hf_subset": "lua_Latn", + "languages": [ + "lua-Latn" + ] + }, + { + "accuracy": 0.280392, + "f1": 0.265995, + "f1_weighted": 0.282346, + "scores_per_experiment": [ + { + "accuracy": 0.264706, + "f1": 0.24995, + "f1_weighted": 0.271538 + }, + { + "accuracy": 0.289216, + "f1": 0.265926, + "f1_weighted": 0.291649 + }, + { + "accuracy": 0.230392, + "f1": 0.217525, + "f1_weighted": 0.227022 + }, + { + "accuracy": 0.279412, + "f1": 0.271261, + "f1_weighted": 0.277639 + }, + { + "accuracy": 0.308824, + "f1": 0.289317, + "f1_weighted": 0.306692 + }, + { + "accuracy": 0.25, + "f1": 0.243784, + "f1_weighted": 0.260789 + }, + { + "accuracy": 0.289216, + "f1": 0.257443, + "f1_weighted": 0.28519 + }, + { + "accuracy": 0.29902, + "f1": 0.284667, + "f1_weighted": 0.313455 + }, + { + "accuracy": 0.289216, + "f1": 0.275804, + "f1_weighted": 0.282869 + }, + { + "accuracy": 0.303922, + "f1": 0.30427, + "f1_weighted": 0.30662 + } + ], + "main_score": 0.280392, + "hf_subset": "lug_Latn", + "languages": [ + "lug-Latn" + ] + }, + { + "accuracy": 0.326471, + "f1": 0.313979, + "f1_weighted": 0.329523, + "scores_per_experiment": [ + { + "accuracy": 0.382353, + "f1": 0.339598, + "f1_weighted": 0.370803 + }, + { + "accuracy": 0.333333, + "f1": 0.330722, + "f1_weighted": 0.337347 + }, + { + "accuracy": 0.357843, + "f1": 0.357097, + "f1_weighted": 0.348929 + }, + { + "accuracy": 0.259804, + "f1": 0.247847, + "f1_weighted": 0.2716 + }, + { + "accuracy": 0.313725, + "f1": 0.297706, + "f1_weighted": 0.316912 + }, + { + "accuracy": 0.328431, + "f1": 0.313268, + "f1_weighted": 0.332308 + }, + { + "accuracy": 0.348039, + "f1": 0.308608, + "f1_weighted": 0.346934 + }, + { + "accuracy": 0.328431, + "f1": 0.322716, + "f1_weighted": 0.339336 + }, + { + "accuracy": 0.269608, + "f1": 0.263747, + "f1_weighted": 0.275842 + }, + { + "accuracy": 0.343137, + "f1": 0.358486, + "f1_weighted": 0.355213 + } + ], + "main_score": 0.326471, + "hf_subset": "luo_Latn", + "languages": [ + "luo-Latn" + ] + }, + { + "accuracy": 0.426961, + "f1": 0.405505, + "f1_weighted": 0.437271, + "scores_per_experiment": [ + { + "accuracy": 0.426471, + "f1": 0.375063, + "f1_weighted": 0.425382 + }, + { + "accuracy": 0.421569, + "f1": 0.415055, + "f1_weighted": 0.430248 + }, + { + "accuracy": 0.382353, + "f1": 0.355701, + "f1_weighted": 0.389382 + }, + { + "accuracy": 0.431373, + "f1": 0.408405, + "f1_weighted": 0.432297 + }, + { + "accuracy": 0.5, + "f1": 0.495731, + "f1_weighted": 0.509834 + }, + { + "accuracy": 0.397059, + "f1": 0.360723, + "f1_weighted": 0.417666 + }, + { + "accuracy": 0.460784, + "f1": 0.443974, + "f1_weighted": 0.475263 + }, + { + "accuracy": 0.431373, + "f1": 0.429963, + "f1_weighted": 0.450899 + }, + { + "accuracy": 0.367647, + "f1": 0.342313, + "f1_weighted": 0.391901 + }, + { + "accuracy": 0.45098, + "f1": 0.428121, + "f1_weighted": 0.449838 + } + ], + "main_score": 0.426961, + "hf_subset": "lus_Latn", + "languages": [ + "lus-Latn" + ] + }, + { + "accuracy": 0.326961, + "f1": 0.316236, + "f1_weighted": 0.328267, + "scores_per_experiment": [ + { + "accuracy": 0.392157, + "f1": 0.370826, + "f1_weighted": 0.394996 + }, + { + "accuracy": 0.372549, + "f1": 0.359776, + "f1_weighted": 0.358463 + }, + { + "accuracy": 0.338235, + "f1": 0.330389, + "f1_weighted": 0.341791 + }, + { + "accuracy": 0.308824, + "f1": 0.313472, + "f1_weighted": 0.313824 + }, + { + "accuracy": 0.323529, + "f1": 0.308756, + "f1_weighted": 0.334987 + }, + { + "accuracy": 0.323529, + "f1": 0.307657, + "f1_weighted": 0.329756 + }, + { + "accuracy": 0.230392, + "f1": 0.223361, + "f1_weighted": 0.233526 + }, + { + "accuracy": 0.377451, + "f1": 0.36551, + "f1_weighted": 0.382895 + }, + { + "accuracy": 0.27451, + "f1": 0.270887, + "f1_weighted": 0.264969 + }, + { + "accuracy": 0.328431, + "f1": 0.311727, + "f1_weighted": 0.327466 + } + ], + "main_score": 0.326961, + "hf_subset": "lvs_Latn", + "languages": [ + "lvs-Latn" + ] + }, + { + "accuracy": 0.169608, + "f1": 0.133336, + "f1_weighted": 0.136475, + "scores_per_experiment": [ + { + "accuracy": 0.151961, + "f1": 0.12758, + "f1_weighted": 0.125219 + }, + { + "accuracy": 0.176471, + "f1": 0.140848, + "f1_weighted": 0.13889 + }, + { + "accuracy": 0.112745, + "f1": 0.112257, + "f1_weighted": 0.085096 + }, + { + "accuracy": 0.137255, + "f1": 0.101119, + "f1_weighted": 0.088072 + }, + { + "accuracy": 0.171569, + "f1": 0.166752, + "f1_weighted": 0.159651 + }, + { + "accuracy": 0.196078, + "f1": 0.16301, + "f1_weighted": 0.176244 + }, + { + "accuracy": 0.176471, + "f1": 0.141107, + "f1_weighted": 0.145726 + }, + { + "accuracy": 0.196078, + "f1": 0.1176, + "f1_weighted": 0.131967 + }, + { + "accuracy": 0.20098, + "f1": 0.140319, + "f1_weighted": 0.167825 + }, + { + "accuracy": 0.176471, + "f1": 0.122771, + "f1_weighted": 0.146064 + } + ], + "main_score": 0.169608, + "hf_subset": "mag_Deva", + "languages": [ + "mag-Deva" + ] + }, + { + "accuracy": 0.194608, + "f1": 0.149845, + "f1_weighted": 0.163025, + "scores_per_experiment": [ + { + "accuracy": 0.196078, + "f1": 0.139937, + "f1_weighted": 0.168474 + }, + { + "accuracy": 0.20098, + "f1": 0.149735, + "f1_weighted": 0.158102 + }, + { + "accuracy": 0.196078, + "f1": 0.152378, + "f1_weighted": 0.161482 + }, + { + "accuracy": 0.147059, + "f1": 0.103652, + "f1_weighted": 0.09671 + }, + { + "accuracy": 0.142157, + "f1": 0.130482, + "f1_weighted": 0.125966 + }, + { + "accuracy": 0.220588, + "f1": 0.215202, + "f1_weighted": 0.211428 + }, + { + "accuracy": 0.210784, + "f1": 0.170534, + "f1_weighted": 0.184382 + }, + { + "accuracy": 0.279412, + "f1": 0.194485, + "f1_weighted": 0.240365 + }, + { + "accuracy": 0.196078, + "f1": 0.142165, + "f1_weighted": 0.160848 + }, + { + "accuracy": 0.156863, + "f1": 0.099884, + "f1_weighted": 0.122489 + } + ], + "main_score": 0.194608, + "hf_subset": "mai_Deva", + "languages": [ + "mai-Deva" + ] + }, + { + "accuracy": 0.180392, + "f1": 0.12712, + "f1_weighted": 0.139678, + "scores_per_experiment": [ + { + "accuracy": 0.181373, + "f1": 0.127057, + "f1_weighted": 0.143498 + }, + { + "accuracy": 0.176471, + "f1": 0.098018, + "f1_weighted": 0.085283 + }, + { + "accuracy": 0.161765, + "f1": 0.128246, + "f1_weighted": 0.112552 + }, + { + "accuracy": 0.161765, + "f1": 0.13539, + "f1_weighted": 0.126133 + }, + { + "accuracy": 0.151961, + "f1": 0.123445, + "f1_weighted": 0.119419 + }, + { + "accuracy": 0.220588, + "f1": 0.161995, + "f1_weighted": 0.191523 + }, + { + "accuracy": 0.186275, + "f1": 0.125778, + "f1_weighted": 0.154012 + }, + { + "accuracy": 0.230392, + "f1": 0.134165, + "f1_weighted": 0.183859 + }, + { + "accuracy": 0.20098, + "f1": 0.140988, + "f1_weighted": 0.166321 + }, + { + "accuracy": 0.132353, + "f1": 0.096117, + "f1_weighted": 0.114181 + } + ], + "main_score": 0.180392, + "hf_subset": "mal_Mlym", + "languages": [ + "mal-Mlym" + ] + }, + { + "accuracy": 0.165196, + "f1": 0.132872, + "f1_weighted": 0.143357, + "scores_per_experiment": [ + { + "accuracy": 0.151961, + "f1": 0.104205, + "f1_weighted": 0.127009 + }, + { + "accuracy": 0.166667, + "f1": 0.12176, + "f1_weighted": 0.148157 + }, + { + "accuracy": 0.171569, + "f1": 0.157136, + "f1_weighted": 0.153201 + }, + { + "accuracy": 0.156863, + "f1": 0.124634, + "f1_weighted": 0.131966 + }, + { + "accuracy": 0.142157, + "f1": 0.127463, + "f1_weighted": 0.098766 + }, + { + "accuracy": 0.166667, + "f1": 0.131807, + "f1_weighted": 0.14717 + }, + { + "accuracy": 0.151961, + "f1": 0.134805, + "f1_weighted": 0.133495 + }, + { + "accuracy": 0.191176, + "f1": 0.151998, + "f1_weighted": 0.177174 + }, + { + "accuracy": 0.205882, + "f1": 0.175474, + "f1_weighted": 0.208898 + }, + { + "accuracy": 0.147059, + "f1": 0.099438, + "f1_weighted": 0.107731 + } + ], + "main_score": 0.165196, + "hf_subset": "mar_Deva", + "languages": [ + "mar-Deva" + ] + }, + { + "accuracy": 0.342157, + "f1": 0.330438, + "f1_weighted": 0.346743, + "scores_per_experiment": [ + { + "accuracy": 0.357843, + "f1": 0.345069, + "f1_weighted": 0.36336 + }, + { + "accuracy": 0.308824, + "f1": 0.287274, + "f1_weighted": 0.311541 + }, + { + "accuracy": 0.406863, + "f1": 0.404343, + "f1_weighted": 0.406093 + }, + { + "accuracy": 0.284314, + "f1": 0.279636, + "f1_weighted": 0.299619 + }, + { + "accuracy": 0.343137, + "f1": 0.334903, + "f1_weighted": 0.33837 + }, + { + "accuracy": 0.352941, + "f1": 0.33793, + "f1_weighted": 0.350748 + }, + { + "accuracy": 0.333333, + "f1": 0.316726, + "f1_weighted": 0.334042 + }, + { + "accuracy": 0.357843, + "f1": 0.34745, + "f1_weighted": 0.36846 + }, + { + "accuracy": 0.323529, + "f1": 0.314259, + "f1_weighted": 0.339069 + }, + { + "accuracy": 0.352941, + "f1": 0.336788, + "f1_weighted": 0.356129 + } + ], + "main_score": 0.342157, + "hf_subset": "min_Latn", + "languages": [ + "min-Latn" + ] + }, + { + "accuracy": 0.220098, + "f1": 0.198238, + "f1_weighted": 0.211096, + "scores_per_experiment": [ + { + "accuracy": 0.27451, + "f1": 0.207453, + "f1_weighted": 0.245356 + }, + { + "accuracy": 0.215686, + "f1": 0.182469, + "f1_weighted": 0.184553 + }, + { + "accuracy": 0.22549, + "f1": 0.211983, + "f1_weighted": 0.23308 + }, + { + "accuracy": 0.29902, + "f1": 0.277532, + "f1_weighted": 0.306316 + }, + { + "accuracy": 0.166667, + "f1": 0.148504, + "f1_weighted": 0.156059 + }, + { + "accuracy": 0.25, + "f1": 0.23206, + "f1_weighted": 0.234414 + }, + { + "accuracy": 0.176471, + "f1": 0.172544, + "f1_weighted": 0.178309 + }, + { + "accuracy": 0.176471, + "f1": 0.171193, + "f1_weighted": 0.171026 + }, + { + "accuracy": 0.245098, + "f1": 0.209209, + "f1_weighted": 0.234026 + }, + { + "accuracy": 0.171569, + "f1": 0.169435, + "f1_weighted": 0.167823 + } + ], + "main_score": 0.220098, + "hf_subset": "mkd_Cyrl", + "languages": [ + "mkd-Cyrl" + ] + }, + { + "accuracy": 0.340196, + "f1": 0.330103, + "f1_weighted": 0.339344, + "scores_per_experiment": [ + { + "accuracy": 0.372549, + "f1": 0.379468, + "f1_weighted": 0.384522 + }, + { + "accuracy": 0.338235, + "f1": 0.328286, + "f1_weighted": 0.344671 + }, + { + "accuracy": 0.357843, + "f1": 0.367067, + "f1_weighted": 0.347132 + }, + { + "accuracy": 0.348039, + "f1": 0.335087, + "f1_weighted": 0.339002 + }, + { + "accuracy": 0.352941, + "f1": 0.33937, + "f1_weighted": 0.354086 + }, + { + "accuracy": 0.328431, + "f1": 0.314605, + "f1_weighted": 0.32492 + }, + { + "accuracy": 0.323529, + "f1": 0.322828, + "f1_weighted": 0.322262 + }, + { + "accuracy": 0.338235, + "f1": 0.304462, + "f1_weighted": 0.339605 + }, + { + "accuracy": 0.308824, + "f1": 0.285103, + "f1_weighted": 0.301873 + }, + { + "accuracy": 0.333333, + "f1": 0.324751, + "f1_weighted": 0.335364 + } + ], + "main_score": 0.340196, + "hf_subset": "mlt_Latn", + "languages": [ + "mlt-Latn" + ] + }, + { + "accuracy": 0.189216, + "f1": 0.152867, + "f1_weighted": 0.164063, + "scores_per_experiment": [ + { + "accuracy": 0.186275, + "f1": 0.139263, + "f1_weighted": 0.159163 + }, + { + "accuracy": 0.254902, + "f1": 0.193318, + "f1_weighted": 0.226496 + }, + { + "accuracy": 0.137255, + "f1": 0.116089, + "f1_weighted": 0.10574 + }, + { + "accuracy": 0.166667, + "f1": 0.138172, + "f1_weighted": 0.153472 + }, + { + "accuracy": 0.215686, + "f1": 0.189527, + "f1_weighted": 0.196553 + }, + { + "accuracy": 0.191176, + "f1": 0.17716, + "f1_weighted": 0.167607 + }, + { + "accuracy": 0.171569, + "f1": 0.143388, + "f1_weighted": 0.155717 + }, + { + "accuracy": 0.240196, + "f1": 0.173856, + "f1_weighted": 0.208796 + }, + { + "accuracy": 0.171569, + "f1": 0.143032, + "f1_weighted": 0.148681 + }, + { + "accuracy": 0.156863, + "f1": 0.114863, + "f1_weighted": 0.11841 + } + ], + "main_score": 0.189216, + "hf_subset": "mni_Beng", + "languages": [ + "mni-Beng" + ] + }, + { + "accuracy": 0.319118, + "f1": 0.303284, + "f1_weighted": 0.32229, + "scores_per_experiment": [ + { + "accuracy": 0.377451, + "f1": 0.362599, + "f1_weighted": 0.391204 + }, + { + "accuracy": 0.289216, + "f1": 0.276351, + "f1_weighted": 0.285905 + }, + { + "accuracy": 0.303922, + "f1": 0.29106, + "f1_weighted": 0.300304 + }, + { + "accuracy": 0.377451, + "f1": 0.370553, + "f1_weighted": 0.379336 + }, + { + "accuracy": 0.303922, + "f1": 0.289815, + "f1_weighted": 0.31575 + }, + { + "accuracy": 0.313725, + "f1": 0.292438, + "f1_weighted": 0.31977 + }, + { + "accuracy": 0.352941, + "f1": 0.320037, + "f1_weighted": 0.355696 + }, + { + "accuracy": 0.318627, + "f1": 0.298504, + "f1_weighted": 0.317746 + }, + { + "accuracy": 0.259804, + "f1": 0.236699, + "f1_weighted": 0.257776 + }, + { + "accuracy": 0.294118, + "f1": 0.294782, + "f1_weighted": 0.299418 + } + ], + "main_score": 0.319118, + "hf_subset": "mos_Latn", + "languages": [ + "mos-Latn" + ] + }, + { + "accuracy": 0.339706, + "f1": 0.314274, + "f1_weighted": 0.338101, + "scores_per_experiment": [ + { + "accuracy": 0.392157, + "f1": 0.361638, + "f1_weighted": 0.390199 + }, + { + "accuracy": 0.357843, + "f1": 0.311039, + "f1_weighted": 0.338191 + }, + { + "accuracy": 0.338235, + "f1": 0.29603, + "f1_weighted": 0.332036 + }, + { + "accuracy": 0.352941, + "f1": 0.342555, + "f1_weighted": 0.366981 + }, + { + "accuracy": 0.377451, + "f1": 0.337222, + "f1_weighted": 0.369857 + }, + { + "accuracy": 0.259804, + "f1": 0.258493, + "f1_weighted": 0.267524 + }, + { + "accuracy": 0.313725, + "f1": 0.302449, + "f1_weighted": 0.299982 + }, + { + "accuracy": 0.328431, + "f1": 0.308986, + "f1_weighted": 0.327926 + }, + { + "accuracy": 0.289216, + "f1": 0.270751, + "f1_weighted": 0.294448 + }, + { + "accuracy": 0.387255, + "f1": 0.353575, + "f1_weighted": 0.393867 + } + ], + "main_score": 0.339706, + "hf_subset": "mri_Latn", + "languages": [ + "mri-Latn" + ] + }, + { + "accuracy": 0.151961, + "f1": 0.100624, + "f1_weighted": 0.108181, + "scores_per_experiment": [ + { + "accuracy": 0.191176, + "f1": 0.118298, + "f1_weighted": 0.129268 + }, + { + "accuracy": 0.127451, + "f1": 0.07654, + "f1_weighted": 0.064618 + }, + { + "accuracy": 0.137255, + "f1": 0.105264, + "f1_weighted": 0.091455 + }, + { + "accuracy": 0.142157, + "f1": 0.092441, + "f1_weighted": 0.100099 + }, + { + "accuracy": 0.127451, + "f1": 0.077804, + "f1_weighted": 0.080951 + }, + { + "accuracy": 0.117647, + "f1": 0.09591, + "f1_weighted": 0.098877 + }, + { + "accuracy": 0.132353, + "f1": 0.102435, + "f1_weighted": 0.109306 + }, + { + "accuracy": 0.215686, + "f1": 0.093186, + "f1_weighted": 0.120126 + }, + { + "accuracy": 0.181373, + "f1": 0.14215, + "f1_weighted": 0.154615 + }, + { + "accuracy": 0.147059, + "f1": 0.102212, + "f1_weighted": 0.132492 + } + ], + "main_score": 0.151961, + "hf_subset": "mya_Mymr", + "languages": [ + "mya-Mymr" + ] + }, + { + "accuracy": 0.489216, + "f1": 0.471945, + "f1_weighted": 0.489467, + "scores_per_experiment": [ + { + "accuracy": 0.583333, + "f1": 0.56192, + "f1_weighted": 0.584683 + }, + { + "accuracy": 0.485294, + "f1": 0.477671, + "f1_weighted": 0.48826 + }, + { + "accuracy": 0.460784, + "f1": 0.451686, + "f1_weighted": 0.465499 + }, + { + "accuracy": 0.534314, + "f1": 0.512171, + "f1_weighted": 0.532842 + }, + { + "accuracy": 0.455882, + "f1": 0.434534, + "f1_weighted": 0.458888 + }, + { + "accuracy": 0.519608, + "f1": 0.51015, + "f1_weighted": 0.515393 + }, + { + "accuracy": 0.480392, + "f1": 0.460706, + "f1_weighted": 0.474351 + }, + { + "accuracy": 0.480392, + "f1": 0.476693, + "f1_weighted": 0.489128 + }, + { + "accuracy": 0.392157, + "f1": 0.353687, + "f1_weighted": 0.386549 + }, + { + "accuracy": 0.5, + "f1": 0.48023, + "f1_weighted": 0.499081 + } + ], + "main_score": 0.489216, + "hf_subset": "nld_Latn", + "languages": [ + "nld-Latn" + ] + }, + { + "accuracy": 0.448039, + "f1": 0.431677, + "f1_weighted": 0.443204, + "scores_per_experiment": [ + { + "accuracy": 0.504902, + "f1": 0.489599, + "f1_weighted": 0.51042 + }, + { + "accuracy": 0.45098, + "f1": 0.435584, + "f1_weighted": 0.452293 + }, + { + "accuracy": 0.460784, + "f1": 0.449397, + "f1_weighted": 0.432673 + }, + { + "accuracy": 0.426471, + "f1": 0.40417, + "f1_weighted": 0.418304 + }, + { + "accuracy": 0.401961, + "f1": 0.37809, + "f1_weighted": 0.380126 + }, + { + "accuracy": 0.436275, + "f1": 0.424015, + "f1_weighted": 0.444593 + }, + { + "accuracy": 0.426471, + "f1": 0.411636, + "f1_weighted": 0.421845 + }, + { + "accuracy": 0.465686, + "f1": 0.448917, + "f1_weighted": 0.468919 + }, + { + "accuracy": 0.441176, + "f1": 0.415628, + "f1_weighted": 0.4333 + }, + { + "accuracy": 0.465686, + "f1": 0.45973, + "f1_weighted": 0.469569 + } + ], + "main_score": 0.448039, + "hf_subset": "nno_Latn", + "languages": [ + "nno-Latn" + ] + }, + { + "accuracy": 0.473529, + "f1": 0.458574, + "f1_weighted": 0.472489, + "scores_per_experiment": [ + { + "accuracy": 0.553922, + "f1": 0.55187, + "f1_weighted": 0.564117 + }, + { + "accuracy": 0.45098, + "f1": 0.431597, + "f1_weighted": 0.451291 + }, + { + "accuracy": 0.470588, + "f1": 0.451657, + "f1_weighted": 0.461964 + }, + { + "accuracy": 0.465686, + "f1": 0.443093, + "f1_weighted": 0.464551 + }, + { + "accuracy": 0.455882, + "f1": 0.446873, + "f1_weighted": 0.457379 + }, + { + "accuracy": 0.470588, + "f1": 0.453835, + "f1_weighted": 0.464914 + }, + { + "accuracy": 0.45098, + "f1": 0.422514, + "f1_weighted": 0.43821 + }, + { + "accuracy": 0.470588, + "f1": 0.464044, + "f1_weighted": 0.476894 + }, + { + "accuracy": 0.406863, + "f1": 0.382981, + "f1_weighted": 0.400062 + }, + { + "accuracy": 0.539216, + "f1": 0.537275, + "f1_weighted": 0.545505 + } + ], + "main_score": 0.473529, + "hf_subset": "nob_Latn", + "languages": [ + "nob-Latn" + ] + }, + { + "accuracy": 0.179412, + "f1": 0.145853, + "f1_weighted": 0.155143, + "scores_per_experiment": [ + { + "accuracy": 0.196078, + "f1": 0.170073, + "f1_weighted": 0.182706 + }, + { + "accuracy": 0.210784, + "f1": 0.151311, + "f1_weighted": 0.164011 + }, + { + "accuracy": 0.161765, + "f1": 0.140535, + "f1_weighted": 0.1255 + }, + { + "accuracy": 0.151961, + "f1": 0.116624, + "f1_weighted": 0.126449 + }, + { + "accuracy": 0.186275, + "f1": 0.152509, + "f1_weighted": 0.150241 + }, + { + "accuracy": 0.186275, + "f1": 0.184447, + "f1_weighted": 0.166598 + }, + { + "accuracy": 0.191176, + "f1": 0.148437, + "f1_weighted": 0.160712 + }, + { + "accuracy": 0.235294, + "f1": 0.176673, + "f1_weighted": 0.220708 + }, + { + "accuracy": 0.132353, + "f1": 0.105391, + "f1_weighted": 0.117683 + }, + { + "accuracy": 0.142157, + "f1": 0.11253, + "f1_weighted": 0.136824 + } + ], + "main_score": 0.179412, + "hf_subset": "npi_Deva", + "languages": [ + "npi-Deva" + ] + }, + { + "accuracy": 0.168137, + "f1": 0.126009, + "f1_weighted": 0.135628, + "scores_per_experiment": [ + { + "accuracy": 0.191176, + "f1": 0.102176, + "f1_weighted": 0.131385 + }, + { + "accuracy": 0.176471, + "f1": 0.159362, + "f1_weighted": 0.171039 + }, + { + "accuracy": 0.151961, + "f1": 0.119449, + "f1_weighted": 0.117576 + }, + { + "accuracy": 0.142157, + "f1": 0.109372, + "f1_weighted": 0.113602 + }, + { + "accuracy": 0.191176, + "f1": 0.147005, + "f1_weighted": 0.150194 + }, + { + "accuracy": 0.171569, + "f1": 0.155656, + "f1_weighted": 0.145159 + }, + { + "accuracy": 0.176471, + "f1": 0.105614, + "f1_weighted": 0.130701 + }, + { + "accuracy": 0.191176, + "f1": 0.147026, + "f1_weighted": 0.167876 + }, + { + "accuracy": 0.132353, + "f1": 0.100777, + "f1_weighted": 0.088159 + }, + { + "accuracy": 0.156863, + "f1": 0.113658, + "f1_weighted": 0.140585 + } + ], + "main_score": 0.168137, + "hf_subset": "nqo_Nkoo", + "languages": [ + "nqo-Nkoo" + ] + }, + { + "accuracy": 0.340196, + "f1": 0.324961, + "f1_weighted": 0.344687, + "scores_per_experiment": [ + { + "accuracy": 0.397059, + "f1": 0.376758, + "f1_weighted": 0.40499 + }, + { + "accuracy": 0.284314, + "f1": 0.281265, + "f1_weighted": 0.294292 + }, + { + "accuracy": 0.303922, + "f1": 0.292208, + "f1_weighted": 0.305597 + }, + { + "accuracy": 0.313725, + "f1": 0.28212, + "f1_weighted": 0.314625 + }, + { + "accuracy": 0.352941, + "f1": 0.341534, + "f1_weighted": 0.359514 + }, + { + "accuracy": 0.362745, + "f1": 0.342711, + "f1_weighted": 0.358272 + }, + { + "accuracy": 0.367647, + "f1": 0.352054, + "f1_weighted": 0.371886 + }, + { + "accuracy": 0.377451, + "f1": 0.368734, + "f1_weighted": 0.388813 + }, + { + "accuracy": 0.303922, + "f1": 0.285448, + "f1_weighted": 0.297118 + }, + { + "accuracy": 0.338235, + "f1": 0.326781, + "f1_weighted": 0.351762 + } + ], + "main_score": 0.340196, + "hf_subset": "nso_Latn", + "languages": [ + "nso-Latn" + ] + }, + { + "accuracy": 0.244118, + "f1": 0.234952, + "f1_weighted": 0.24351, + "scores_per_experiment": [ + { + "accuracy": 0.279412, + "f1": 0.256757, + "f1_weighted": 0.278958 + }, + { + "accuracy": 0.245098, + "f1": 0.235641, + "f1_weighted": 0.23735 + }, + { + "accuracy": 0.22549, + "f1": 0.225193, + "f1_weighted": 0.219301 + }, + { + "accuracy": 0.25, + "f1": 0.236934, + "f1_weighted": 0.248777 + }, + { + "accuracy": 0.264706, + "f1": 0.272472, + "f1_weighted": 0.264966 + }, + { + "accuracy": 0.279412, + "f1": 0.263205, + "f1_weighted": 0.285829 + }, + { + "accuracy": 0.196078, + "f1": 0.199428, + "f1_weighted": 0.191169 + }, + { + "accuracy": 0.264706, + "f1": 0.253122, + "f1_weighted": 0.266825 + }, + { + "accuracy": 0.220588, + "f1": 0.196088, + "f1_weighted": 0.22613 + }, + { + "accuracy": 0.215686, + "f1": 0.21068, + "f1_weighted": 0.215791 + } + ], + "main_score": 0.244118, + "hf_subset": "nus_Latn", + "languages": [ + "nus-Latn" + ] + }, + { + "accuracy": 0.340196, + "f1": 0.316509, + "f1_weighted": 0.338476, + "scores_per_experiment": [ + { + "accuracy": 0.387255, + "f1": 0.362917, + "f1_weighted": 0.396329 + }, + { + "accuracy": 0.323529, + "f1": 0.308987, + "f1_weighted": 0.327207 + }, + { + "accuracy": 0.348039, + "f1": 0.319629, + "f1_weighted": 0.326764 + }, + { + "accuracy": 0.245098, + "f1": 0.220546, + "f1_weighted": 0.244848 + }, + { + "accuracy": 0.357843, + "f1": 0.332401, + "f1_weighted": 0.355684 + }, + { + "accuracy": 0.328431, + "f1": 0.284787, + "f1_weighted": 0.321876 + }, + { + "accuracy": 0.338235, + "f1": 0.310693, + "f1_weighted": 0.339189 + }, + { + "accuracy": 0.406863, + "f1": 0.387809, + "f1_weighted": 0.411206 + }, + { + "accuracy": 0.289216, + "f1": 0.287157, + "f1_weighted": 0.28641 + }, + { + "accuracy": 0.377451, + "f1": 0.350163, + "f1_weighted": 0.375246 + } + ], + "main_score": 0.340196, + "hf_subset": "nya_Latn", + "languages": [ + "nya-Latn" + ] + }, + { + "accuracy": 0.517647, + "f1": 0.504354, + "f1_weighted": 0.518968, + "scores_per_experiment": [ + { + "accuracy": 0.573529, + "f1": 0.548019, + "f1_weighted": 0.570457 + }, + { + "accuracy": 0.455882, + "f1": 0.447883, + "f1_weighted": 0.458194 + }, + { + "accuracy": 0.45098, + "f1": 0.445393, + "f1_weighted": 0.44997 + }, + { + "accuracy": 0.52451, + "f1": 0.518783, + "f1_weighted": 0.529201 + }, + { + "accuracy": 0.553922, + "f1": 0.549436, + "f1_weighted": 0.56616 + }, + { + "accuracy": 0.553922, + "f1": 0.530879, + "f1_weighted": 0.550777 + }, + { + "accuracy": 0.544118, + "f1": 0.5261, + "f1_weighted": 0.545103 + }, + { + "accuracy": 0.534314, + "f1": 0.520964, + "f1_weighted": 0.536529 + }, + { + "accuracy": 0.460784, + "f1": 0.421051, + "f1_weighted": 0.454409 + }, + { + "accuracy": 0.52451, + "f1": 0.535033, + "f1_weighted": 0.528881 + } + ], + "main_score": 0.517647, + "hf_subset": "oci_Latn", + "languages": [ + "oci-Latn" + ] + }, + { + "accuracy": 0.166667, + "f1": 0.141446, + "f1_weighted": 0.145372, + "scores_per_experiment": [ + { + "accuracy": 0.166667, + "f1": 0.149607, + "f1_weighted": 0.171148 + }, + { + "accuracy": 0.171569, + "f1": 0.15208, + "f1_weighted": 0.141251 + }, + { + "accuracy": 0.137255, + "f1": 0.127892, + "f1_weighted": 0.096911 + }, + { + "accuracy": 0.122549, + "f1": 0.103054, + "f1_weighted": 0.099202 + }, + { + "accuracy": 0.142157, + "f1": 0.140183, + "f1_weighted": 0.117142 + }, + { + "accuracy": 0.25, + "f1": 0.208086, + "f1_weighted": 0.225013 + }, + { + "accuracy": 0.171569, + "f1": 0.14707, + "f1_weighted": 0.163026 + }, + { + "accuracy": 0.196078, + "f1": 0.166288, + "f1_weighted": 0.178403 + }, + { + "accuracy": 0.117647, + "f1": 0.077479, + "f1_weighted": 0.100732 + }, + { + "accuracy": 0.191176, + "f1": 0.142719, + "f1_weighted": 0.160895 + } + ], + "main_score": 0.166667, + "hf_subset": "ory_Orya", + "languages": [ + "ory-Orya" + ] + }, + { + "accuracy": 0.448529, + "f1": 0.421138, + "f1_weighted": 0.458864, + "scores_per_experiment": [ + { + "accuracy": 0.514706, + "f1": 0.481816, + "f1_weighted": 0.523658 + }, + { + "accuracy": 0.446078, + "f1": 0.42089, + "f1_weighted": 0.450554 + }, + { + "accuracy": 0.431373, + "f1": 0.40359, + "f1_weighted": 0.431638 + }, + { + "accuracy": 0.387255, + "f1": 0.364982, + "f1_weighted": 0.39035 + }, + { + "accuracy": 0.470588, + "f1": 0.454835, + "f1_weighted": 0.486044 + }, + { + "accuracy": 0.45098, + "f1": 0.414358, + "f1_weighted": 0.459161 + }, + { + "accuracy": 0.441176, + "f1": 0.403417, + "f1_weighted": 0.459367 + }, + { + "accuracy": 0.465686, + "f1": 0.437084, + "f1_weighted": 0.48374 + }, + { + "accuracy": 0.426471, + "f1": 0.38836, + "f1_weighted": 0.428551 + }, + { + "accuracy": 0.45098, + "f1": 0.442051, + "f1_weighted": 0.475575 + } + ], + "main_score": 0.448529, + "hf_subset": "pag_Latn", + "languages": [ + "pag-Latn" + ] + }, + { + "accuracy": 0.169608, + "f1": 0.127955, + "f1_weighted": 0.133176, + "scores_per_experiment": [ + { + "accuracy": 0.147059, + "f1": 0.115304, + "f1_weighted": 0.126836 + }, + { + "accuracy": 0.151961, + "f1": 0.115132, + "f1_weighted": 0.099008 + }, + { + "accuracy": 0.137255, + "f1": 0.127968, + "f1_weighted": 0.111691 + }, + { + "accuracy": 0.151961, + "f1": 0.119543, + "f1_weighted": 0.116756 + }, + { + "accuracy": 0.171569, + "f1": 0.102884, + "f1_weighted": 0.114096 + }, + { + "accuracy": 0.147059, + "f1": 0.139234, + "f1_weighted": 0.136176 + }, + { + "accuracy": 0.20098, + "f1": 0.143867, + "f1_weighted": 0.161032 + }, + { + "accuracy": 0.205882, + "f1": 0.147935, + "f1_weighted": 0.165352 + }, + { + "accuracy": 0.196078, + "f1": 0.12096, + "f1_weighted": 0.141597 + }, + { + "accuracy": 0.186275, + "f1": 0.146725, + "f1_weighted": 0.159216 + } + ], + "main_score": 0.169608, + "hf_subset": "pan_Guru", + "languages": [ + "pan-Guru" + ] + }, + { + "accuracy": 0.420588, + "f1": 0.404329, + "f1_weighted": 0.420938, + "scores_per_experiment": [ + { + "accuracy": 0.5, + "f1": 0.479816, + "f1_weighted": 0.505922 + }, + { + "accuracy": 0.362745, + "f1": 0.33934, + "f1_weighted": 0.365409 + }, + { + "accuracy": 0.426471, + "f1": 0.414325, + "f1_weighted": 0.421725 + }, + { + "accuracy": 0.387255, + "f1": 0.354754, + "f1_weighted": 0.384334 + }, + { + "accuracy": 0.401961, + "f1": 0.397978, + "f1_weighted": 0.418702 + }, + { + "accuracy": 0.47549, + "f1": 0.431824, + "f1_weighted": 0.467995 + }, + { + "accuracy": 0.480392, + "f1": 0.459639, + "f1_weighted": 0.477863 + }, + { + "accuracy": 0.441176, + "f1": 0.431996, + "f1_weighted": 0.447178 + }, + { + "accuracy": 0.348039, + "f1": 0.339281, + "f1_weighted": 0.342545 + }, + { + "accuracy": 0.382353, + "f1": 0.394333, + "f1_weighted": 0.377708 + } + ], + "main_score": 0.420588, + "hf_subset": "pap_Latn", + "languages": [ + "pap-Latn" + ] + }, + { + "accuracy": 0.226961, + "f1": 0.15726, + "f1_weighted": 0.172463, + "scores_per_experiment": [ + { + "accuracy": 0.220588, + "f1": 0.13793, + "f1_weighted": 0.139323 + }, + { + "accuracy": 0.240196, + "f1": 0.152384, + "f1_weighted": 0.15814 + }, + { + "accuracy": 0.156863, + "f1": 0.126065, + "f1_weighted": 0.134364 + }, + { + "accuracy": 0.245098, + "f1": 0.145942, + "f1_weighted": 0.152263 + }, + { + "accuracy": 0.240196, + "f1": 0.159614, + "f1_weighted": 0.159933 + }, + { + "accuracy": 0.176471, + "f1": 0.17739, + "f1_weighted": 0.180851 + }, + { + "accuracy": 0.230392, + "f1": 0.16171, + "f1_weighted": 0.186698 + }, + { + "accuracy": 0.181373, + "f1": 0.140385, + "f1_weighted": 0.148175 + }, + { + "accuracy": 0.289216, + "f1": 0.183463, + "f1_weighted": 0.223912 + }, + { + "accuracy": 0.289216, + "f1": 0.187722, + "f1_weighted": 0.240971 + } + ], + "main_score": 0.226961, + "hf_subset": "pbt_Arab", + "languages": [ + "pbt-Arab" + ] + }, + { + "accuracy": 0.179902, + "f1": 0.132398, + "f1_weighted": 0.133815, + "scores_per_experiment": [ + { + "accuracy": 0.22549, + "f1": 0.13401, + "f1_weighted": 0.147492 + }, + { + "accuracy": 0.147059, + "f1": 0.090429, + "f1_weighted": 0.076607 + }, + { + "accuracy": 0.132353, + "f1": 0.129765, + "f1_weighted": 0.116647 + }, + { + "accuracy": 0.22549, + "f1": 0.166309, + "f1_weighted": 0.176865 + }, + { + "accuracy": 0.215686, + "f1": 0.142144, + "f1_weighted": 0.134863 + }, + { + "accuracy": 0.181373, + "f1": 0.172578, + "f1_weighted": 0.156939 + }, + { + "accuracy": 0.186275, + "f1": 0.140297, + "f1_weighted": 0.162384 + }, + { + "accuracy": 0.205882, + "f1": 0.12905, + "f1_weighted": 0.156209 + }, + { + "accuracy": 0.132353, + "f1": 0.106735, + "f1_weighted": 0.099249 + }, + { + "accuracy": 0.147059, + "f1": 0.11266, + "f1_weighted": 0.110892 + } + ], + "main_score": 0.179902, + "hf_subset": "pes_Arab", + "languages": [ + "pes-Arab" + ] + }, + { + "accuracy": 0.291667, + "f1": 0.280871, + "f1_weighted": 0.28714, + "scores_per_experiment": [ + { + "accuracy": 0.318627, + "f1": 0.310032, + "f1_weighted": 0.318518 + }, + { + "accuracy": 0.289216, + "f1": 0.292712, + "f1_weighted": 0.281307 + }, + { + "accuracy": 0.289216, + "f1": 0.27238, + "f1_weighted": 0.281816 + }, + { + "accuracy": 0.313725, + "f1": 0.286629, + "f1_weighted": 0.303857 + }, + { + "accuracy": 0.308824, + "f1": 0.292316, + "f1_weighted": 0.318591 + }, + { + "accuracy": 0.205882, + "f1": 0.196456, + "f1_weighted": 0.206807 + }, + { + "accuracy": 0.27451, + "f1": 0.258515, + "f1_weighted": 0.255219 + }, + { + "accuracy": 0.303922, + "f1": 0.297213, + "f1_weighted": 0.294207 + }, + { + "accuracy": 0.294118, + "f1": 0.289382, + "f1_weighted": 0.29362 + }, + { + "accuracy": 0.318627, + "f1": 0.31307, + "f1_weighted": 0.317457 + } + ], + "main_score": 0.291667, + "hf_subset": "plt_Latn", + "languages": [ + "plt-Latn" + ] + }, + { + "accuracy": 0.34902, + "f1": 0.338805, + "f1_weighted": 0.346624, + "scores_per_experiment": [ + { + "accuracy": 0.372549, + "f1": 0.379373, + "f1_weighted": 0.383384 + }, + { + "accuracy": 0.308824, + "f1": 0.294072, + "f1_weighted": 0.305931 + }, + { + "accuracy": 0.29902, + "f1": 0.302121, + "f1_weighted": 0.290576 + }, + { + "accuracy": 0.323529, + "f1": 0.321451, + "f1_weighted": 0.323978 + }, + { + "accuracy": 0.397059, + "f1": 0.368765, + "f1_weighted": 0.386619 + }, + { + "accuracy": 0.348039, + "f1": 0.337284, + "f1_weighted": 0.33938 + }, + { + "accuracy": 0.328431, + "f1": 0.297423, + "f1_weighted": 0.323536 + }, + { + "accuracy": 0.397059, + "f1": 0.39018, + "f1_weighted": 0.396593 + }, + { + "accuracy": 0.318627, + "f1": 0.299238, + "f1_weighted": 0.310147 + }, + { + "accuracy": 0.397059, + "f1": 0.398143, + "f1_weighted": 0.406099 + } + ], + "main_score": 0.34902, + "hf_subset": "pol_Latn", + "languages": [ + "pol-Latn" + ] + }, + { + "accuracy": 0.569118, + "f1": 0.548482, + "f1_weighted": 0.568346, + "scores_per_experiment": [ + { + "accuracy": 0.602941, + "f1": 0.583174, + "f1_weighted": 0.603431 + }, + { + "accuracy": 0.583333, + "f1": 0.565105, + "f1_weighted": 0.583136 + }, + { + "accuracy": 0.578431, + "f1": 0.581696, + "f1_weighted": 0.572721 + }, + { + "accuracy": 0.544118, + "f1": 0.509988, + "f1_weighted": 0.546271 + }, + { + "accuracy": 0.602941, + "f1": 0.564461, + "f1_weighted": 0.604675 + }, + { + "accuracy": 0.558824, + "f1": 0.523102, + "f1_weighted": 0.55418 + }, + { + "accuracy": 0.632353, + "f1": 0.623952, + "f1_weighted": 0.634581 + }, + { + "accuracy": 0.568627, + "f1": 0.570082, + "f1_weighted": 0.574072 + }, + { + "accuracy": 0.504902, + "f1": 0.463725, + "f1_weighted": 0.498265 + }, + { + "accuracy": 0.514706, + "f1": 0.499536, + "f1_weighted": 0.512129 + } + ], + "main_score": 0.569118, + "hf_subset": "por_Latn", + "languages": [ + "por-Latn" + ] + }, + { + "accuracy": 0.172059, + "f1": 0.123561, + "f1_weighted": 0.125868, + "scores_per_experiment": [ + { + "accuracy": 0.215686, + "f1": 0.117141, + "f1_weighted": 0.126308 + }, + { + "accuracy": 0.142157, + "f1": 0.103161, + "f1_weighted": 0.087097 + }, + { + "accuracy": 0.137255, + "f1": 0.127588, + "f1_weighted": 0.124863 + }, + { + "accuracy": 0.215686, + "f1": 0.112373, + "f1_weighted": 0.11543 + }, + { + "accuracy": 0.132353, + "f1": 0.099646, + "f1_weighted": 0.097813 + }, + { + "accuracy": 0.215686, + "f1": 0.181913, + "f1_weighted": 0.170582 + }, + { + "accuracy": 0.191176, + "f1": 0.166342, + "f1_weighted": 0.180934 + }, + { + "accuracy": 0.230392, + "f1": 0.170713, + "f1_weighted": 0.195106 + }, + { + "accuracy": 0.107843, + "f1": 0.063887, + "f1_weighted": 0.056923 + }, + { + "accuracy": 0.132353, + "f1": 0.092851, + "f1_weighted": 0.103626 + } + ], + "main_score": 0.172059, + "hf_subset": "prs_Arab", + "languages": [ + "prs-Arab" + ] + }, + { + "accuracy": 0.327451, + "f1": 0.30674, + "f1_weighted": 0.332469, + "scores_per_experiment": [ + { + "accuracy": 0.406863, + "f1": 0.385056, + "f1_weighted": 0.408252 + }, + { + "accuracy": 0.29902, + "f1": 0.291557, + "f1_weighted": 0.31869 + }, + { + "accuracy": 0.362745, + "f1": 0.332294, + "f1_weighted": 0.364224 + }, + { + "accuracy": 0.269608, + "f1": 0.253821, + "f1_weighted": 0.275509 + }, + { + "accuracy": 0.343137, + "f1": 0.331805, + "f1_weighted": 0.357183 + }, + { + "accuracy": 0.338235, + "f1": 0.317006, + "f1_weighted": 0.340964 + }, + { + "accuracy": 0.318627, + "f1": 0.30102, + "f1_weighted": 0.329597 + }, + { + "accuracy": 0.338235, + "f1": 0.32261, + "f1_weighted": 0.343129 + }, + { + "accuracy": 0.323529, + "f1": 0.279193, + "f1_weighted": 0.327731 + }, + { + "accuracy": 0.27451, + "f1": 0.253034, + "f1_weighted": 0.259407 + } + ], + "main_score": 0.327451, + "hf_subset": "quy_Latn", + "languages": [ + "quy-Latn" + ] + }, + { + "accuracy": 0.596078, + "f1": 0.580278, + "f1_weighted": 0.59849, + "scores_per_experiment": [ + { + "accuracy": 0.632353, + "f1": 0.625922, + "f1_weighted": 0.633578 + }, + { + "accuracy": 0.568627, + "f1": 0.566665, + "f1_weighted": 0.568238 + }, + { + "accuracy": 0.573529, + "f1": 0.570614, + "f1_weighted": 0.576343 + }, + { + "accuracy": 0.656863, + "f1": 0.630171, + "f1_weighted": 0.657878 + }, + { + "accuracy": 0.598039, + "f1": 0.585994, + "f1_weighted": 0.606528 + }, + { + "accuracy": 0.593137, + "f1": 0.574602, + "f1_weighted": 0.593076 + }, + { + "accuracy": 0.583333, + "f1": 0.558521, + "f1_weighted": 0.58732 + }, + { + "accuracy": 0.598039, + "f1": 0.583354, + "f1_weighted": 0.601501 + }, + { + "accuracy": 0.534314, + "f1": 0.493419, + "f1_weighted": 0.523878 + }, + { + "accuracy": 0.622549, + "f1": 0.613514, + "f1_weighted": 0.636556 + } + ], + "main_score": 0.596078, + "hf_subset": "ron_Latn", + "languages": [ + "ron-Latn" + ] + }, + { + "accuracy": 0.287745, + "f1": 0.273461, + "f1_weighted": 0.288872, + "scores_per_experiment": [ + { + "accuracy": 0.338235, + "f1": 0.308181, + "f1_weighted": 0.339908 + }, + { + "accuracy": 0.269608, + "f1": 0.26107, + "f1_weighted": 0.257288 + }, + { + "accuracy": 0.269608, + "f1": 0.252313, + "f1_weighted": 0.264485 + }, + { + "accuracy": 0.269608, + "f1": 0.269068, + "f1_weighted": 0.265783 + }, + { + "accuracy": 0.333333, + "f1": 0.321669, + "f1_weighted": 0.342944 + }, + { + "accuracy": 0.25, + "f1": 0.244559, + "f1_weighted": 0.254513 + }, + { + "accuracy": 0.269608, + "f1": 0.25675, + "f1_weighted": 0.25769 + }, + { + "accuracy": 0.303922, + "f1": 0.293765, + "f1_weighted": 0.314181 + }, + { + "accuracy": 0.27451, + "f1": 0.254634, + "f1_weighted": 0.28109 + }, + { + "accuracy": 0.29902, + "f1": 0.272605, + "f1_weighted": 0.310836 + } + ], + "main_score": 0.287745, + "hf_subset": "run_Latn", + "languages": [ + "run-Latn" + ] + }, + { + "accuracy": 0.22402, + "f1": 0.20378, + "f1_weighted": 0.216233, + "scores_per_experiment": [ + { + "accuracy": 0.264706, + "f1": 0.244968, + "f1_weighted": 0.260432 + }, + { + "accuracy": 0.235294, + "f1": 0.198697, + "f1_weighted": 0.218028 + }, + { + "accuracy": 0.220588, + "f1": 0.204624, + "f1_weighted": 0.21364 + }, + { + "accuracy": 0.20098, + "f1": 0.179339, + "f1_weighted": 0.193518 + }, + { + "accuracy": 0.196078, + "f1": 0.187825, + "f1_weighted": 0.185228 + }, + { + "accuracy": 0.215686, + "f1": 0.20526, + "f1_weighted": 0.214092 + }, + { + "accuracy": 0.220588, + "f1": 0.186662, + "f1_weighted": 0.188279 + }, + { + "accuracy": 0.220588, + "f1": 0.219134, + "f1_weighted": 0.235461 + }, + { + "accuracy": 0.27451, + "f1": 0.252186, + "f1_weighted": 0.267515 + }, + { + "accuracy": 0.191176, + "f1": 0.159106, + "f1_weighted": 0.186136 + } + ], + "main_score": 0.22402, + "hf_subset": "rus_Cyrl", + "languages": [ + "rus-Cyrl" + ] + }, + { + "accuracy": 0.392647, + "f1": 0.36749, + "f1_weighted": 0.393855, + "scores_per_experiment": [ + { + "accuracy": 0.421569, + "f1": 0.395988, + "f1_weighted": 0.427189 + }, + { + "accuracy": 0.401961, + "f1": 0.364071, + "f1_weighted": 0.404084 + }, + { + "accuracy": 0.328431, + "f1": 0.303118, + "f1_weighted": 0.319727 + }, + { + "accuracy": 0.426471, + "f1": 0.398056, + "f1_weighted": 0.424085 + }, + { + "accuracy": 0.367647, + "f1": 0.356058, + "f1_weighted": 0.369594 + }, + { + "accuracy": 0.348039, + "f1": 0.316542, + "f1_weighted": 0.349262 + }, + { + "accuracy": 0.367647, + "f1": 0.353699, + "f1_weighted": 0.379519 + }, + { + "accuracy": 0.406863, + "f1": 0.38385, + "f1_weighted": 0.41084 + }, + { + "accuracy": 0.431373, + "f1": 0.383498, + "f1_weighted": 0.425624 + }, + { + "accuracy": 0.426471, + "f1": 0.420024, + "f1_weighted": 0.428627 + } + ], + "main_score": 0.392647, + "hf_subset": "sag_Latn", + "languages": [ + "sag-Latn" + ] + }, + { + "accuracy": 0.172059, + "f1": 0.134245, + "f1_weighted": 0.145403, + "scores_per_experiment": [ + { + "accuracy": 0.112745, + "f1": 0.085002, + "f1_weighted": 0.084395 + }, + { + "accuracy": 0.20098, + "f1": 0.18228, + "f1_weighted": 0.204715 + }, + { + "accuracy": 0.137255, + "f1": 0.107727, + "f1_weighted": 0.096721 + }, + { + "accuracy": 0.098039, + "f1": 0.074521, + "f1_weighted": 0.064131 + }, + { + "accuracy": 0.215686, + "f1": 0.151533, + "f1_weighted": 0.164319 + }, + { + "accuracy": 0.171569, + "f1": 0.160113, + "f1_weighted": 0.18319 + }, + { + "accuracy": 0.176471, + "f1": 0.124054, + "f1_weighted": 0.14035 + }, + { + "accuracy": 0.25, + "f1": 0.170574, + "f1_weighted": 0.206869 + }, + { + "accuracy": 0.142157, + "f1": 0.136364, + "f1_weighted": 0.144418 + }, + { + "accuracy": 0.215686, + "f1": 0.150279, + "f1_weighted": 0.164917 + } + ], + "main_score": 0.172059, + "hf_subset": "san_Deva", + "languages": [ + "san-Deva" + ] + }, + { + "accuracy": 0.146078, + "f1": 0.098073, + "f1_weighted": 0.104258, + "scores_per_experiment": [ + { + "accuracy": 0.137255, + "f1": 0.077167, + "f1_weighted": 0.081084 + }, + { + "accuracy": 0.122549, + "f1": 0.088082, + "f1_weighted": 0.073615 + }, + { + "accuracy": 0.107843, + "f1": 0.091042, + "f1_weighted": 0.076082 + }, + { + "accuracy": 0.112745, + "f1": 0.081741, + "f1_weighted": 0.082756 + }, + { + "accuracy": 0.122549, + "f1": 0.093181, + "f1_weighted": 0.092901 + }, + { + "accuracy": 0.122549, + "f1": 0.087605, + "f1_weighted": 0.089344 + }, + { + "accuracy": 0.196078, + "f1": 0.111047, + "f1_weighted": 0.150677 + }, + { + "accuracy": 0.181373, + "f1": 0.098718, + "f1_weighted": 0.108239 + }, + { + "accuracy": 0.22549, + "f1": 0.15932, + "f1_weighted": 0.189131 + }, + { + "accuracy": 0.132353, + "f1": 0.092829, + "f1_weighted": 0.098756 + } + ], + "main_score": 0.146078, + "hf_subset": "sat_Olck", + "languages": [ + "sat-Olck" + ] + }, + { + "accuracy": 0.42402, + "f1": 0.399699, + "f1_weighted": 0.421102, + "scores_per_experiment": [ + { + "accuracy": 0.426471, + "f1": 0.401515, + "f1_weighted": 0.430194 + }, + { + "accuracy": 0.416667, + "f1": 0.38955, + "f1_weighted": 0.414798 + }, + { + "accuracy": 0.401961, + "f1": 0.399907, + "f1_weighted": 0.393865 + }, + { + "accuracy": 0.490196, + "f1": 0.443388, + "f1_weighted": 0.470894 + }, + { + "accuracy": 0.426471, + "f1": 0.388887, + "f1_weighted": 0.41944 + }, + { + "accuracy": 0.426471, + "f1": 0.406125, + "f1_weighted": 0.427153 + }, + { + "accuracy": 0.441176, + "f1": 0.398416, + "f1_weighted": 0.436616 + }, + { + "accuracy": 0.367647, + "f1": 0.358977, + "f1_weighted": 0.379146 + }, + { + "accuracy": 0.401961, + "f1": 0.37284, + "f1_weighted": 0.399451 + }, + { + "accuracy": 0.441176, + "f1": 0.437387, + "f1_weighted": 0.439466 + } + ], + "main_score": 0.42402, + "hf_subset": "scn_Latn", + "languages": [ + "scn-Latn" + ] + }, + { + "accuracy": 0.269608, + "f1": 0.242715, + "f1_weighted": 0.257326, + "scores_per_experiment": [ + { + "accuracy": 0.279412, + "f1": 0.288741, + "f1_weighted": 0.288741 + }, + { + "accuracy": 0.210784, + "f1": 0.193261, + "f1_weighted": 0.186183 + }, + { + "accuracy": 0.210784, + "f1": 0.176886, + "f1_weighted": 0.196228 + }, + { + "accuracy": 0.196078, + "f1": 0.196915, + "f1_weighted": 0.184711 + }, + { + "accuracy": 0.303922, + "f1": 0.246112, + "f1_weighted": 0.288923 + }, + { + "accuracy": 0.289216, + "f1": 0.252734, + "f1_weighted": 0.269263 + }, + { + "accuracy": 0.29902, + "f1": 0.247626, + "f1_weighted": 0.275098 + }, + { + "accuracy": 0.348039, + "f1": 0.331186, + "f1_weighted": 0.348292 + }, + { + "accuracy": 0.27451, + "f1": 0.239732, + "f1_weighted": 0.268021 + }, + { + "accuracy": 0.284314, + "f1": 0.253962, + "f1_weighted": 0.267804 + } + ], + "main_score": 0.269608, + "hf_subset": "shn_Mymr", + "languages": [ + "shn-Mymr" + ] + }, + { + "accuracy": 0.187255, + "f1": 0.132005, + "f1_weighted": 0.151523, + "scores_per_experiment": [ + { + "accuracy": 0.166667, + "f1": 0.111168, + "f1_weighted": 0.1326 + }, + { + "accuracy": 0.191176, + "f1": 0.121137, + "f1_weighted": 0.13173 + }, + { + "accuracy": 0.186275, + "f1": 0.151741, + "f1_weighted": 0.150673 + }, + { + "accuracy": 0.205882, + "f1": 0.153026, + "f1_weighted": 0.177339 + }, + { + "accuracy": 0.107843, + "f1": 0.094026, + "f1_weighted": 0.085199 + }, + { + "accuracy": 0.215686, + "f1": 0.150113, + "f1_weighted": 0.186286 + }, + { + "accuracy": 0.205882, + "f1": 0.134049, + "f1_weighted": 0.151362 + }, + { + "accuracy": 0.245098, + "f1": 0.184912, + "f1_weighted": 0.219483 + }, + { + "accuracy": 0.210784, + "f1": 0.111912, + "f1_weighted": 0.148665 + }, + { + "accuracy": 0.137255, + "f1": 0.107968, + "f1_weighted": 0.131894 + } + ], + "main_score": 0.187255, + "hf_subset": "sin_Sinh", + "languages": [ + "sin-Sinh" + ] + }, + { + "accuracy": 0.313725, + "f1": 0.300674, + "f1_weighted": 0.316536, + "scores_per_experiment": [ + { + "accuracy": 0.382353, + "f1": 0.359537, + "f1_weighted": 0.393842 + }, + { + "accuracy": 0.313725, + "f1": 0.307109, + "f1_weighted": 0.319069 + }, + { + "accuracy": 0.245098, + "f1": 0.244135, + "f1_weighted": 0.244543 + }, + { + "accuracy": 0.294118, + "f1": 0.270868, + "f1_weighted": 0.293405 + }, + { + "accuracy": 0.279412, + "f1": 0.278638, + "f1_weighted": 0.280469 + }, + { + "accuracy": 0.308824, + "f1": 0.298091, + "f1_weighted": 0.306634 + }, + { + "accuracy": 0.318627, + "f1": 0.290149, + "f1_weighted": 0.323476 + }, + { + "accuracy": 0.387255, + "f1": 0.370178, + "f1_weighted": 0.393189 + }, + { + "accuracy": 0.318627, + "f1": 0.296083, + "f1_weighted": 0.312551 + }, + { + "accuracy": 0.289216, + "f1": 0.291947, + "f1_weighted": 0.298184 + } + ], + "main_score": 0.313725, + "hf_subset": "slk_Latn", + "languages": [ + "slk-Latn" + ] + }, + { + "accuracy": 0.312745, + "f1": 0.300909, + "f1_weighted": 0.314682, + "scores_per_experiment": [ + { + "accuracy": 0.313725, + "f1": 0.294972, + "f1_weighted": 0.308363 + }, + { + "accuracy": 0.254902, + "f1": 0.24761, + "f1_weighted": 0.260947 + }, + { + "accuracy": 0.318627, + "f1": 0.327481, + "f1_weighted": 0.318691 + }, + { + "accuracy": 0.323529, + "f1": 0.302218, + "f1_weighted": 0.327996 + }, + { + "accuracy": 0.362745, + "f1": 0.348503, + "f1_weighted": 0.368015 + }, + { + "accuracy": 0.338235, + "f1": 0.339781, + "f1_weighted": 0.332461 + }, + { + "accuracy": 0.264706, + "f1": 0.256124, + "f1_weighted": 0.276031 + }, + { + "accuracy": 0.377451, + "f1": 0.357429, + "f1_weighted": 0.373991 + }, + { + "accuracy": 0.284314, + "f1": 0.266898, + "f1_weighted": 0.282758 + }, + { + "accuracy": 0.289216, + "f1": 0.268071, + "f1_weighted": 0.297563 + } + ], + "main_score": 0.312745, + "hf_subset": "slv_Latn", + "languages": [ + "slv-Latn" + ] + }, + { + "accuracy": 0.347549, + "f1": 0.327031, + "f1_weighted": 0.35267, + "scores_per_experiment": [ + { + "accuracy": 0.436275, + "f1": 0.378073, + "f1_weighted": 0.430265 + }, + { + "accuracy": 0.338235, + "f1": 0.318148, + "f1_weighted": 0.352929 + }, + { + "accuracy": 0.362745, + "f1": 0.356595, + "f1_weighted": 0.378876 + }, + { + "accuracy": 0.372549, + "f1": 0.356706, + "f1_weighted": 0.375402 + }, + { + "accuracy": 0.308824, + "f1": 0.278736, + "f1_weighted": 0.30246 + }, + { + "accuracy": 0.27451, + "f1": 0.275058, + "f1_weighted": 0.283372 + }, + { + "accuracy": 0.372549, + "f1": 0.34868, + "f1_weighted": 0.374932 + }, + { + "accuracy": 0.367647, + "f1": 0.354604, + "f1_weighted": 0.379196 + }, + { + "accuracy": 0.328431, + "f1": 0.294544, + "f1_weighted": 0.320187 + }, + { + "accuracy": 0.313725, + "f1": 0.309168, + "f1_weighted": 0.329084 + } + ], + "main_score": 0.347549, + "hf_subset": "smo_Latn", + "languages": [ + "smo-Latn" + ] + }, + { + "accuracy": 0.310784, + "f1": 0.299383, + "f1_weighted": 0.309568, + "scores_per_experiment": [ + { + "accuracy": 0.338235, + "f1": 0.328548, + "f1_weighted": 0.343507 + }, + { + "accuracy": 0.269608, + "f1": 0.267935, + "f1_weighted": 0.2635 + }, + { + "accuracy": 0.284314, + "f1": 0.268453, + "f1_weighted": 0.278656 + }, + { + "accuracy": 0.323529, + "f1": 0.309664, + "f1_weighted": 0.316318 + }, + { + "accuracy": 0.289216, + "f1": 0.278803, + "f1_weighted": 0.28831 + }, + { + "accuracy": 0.269608, + "f1": 0.263882, + "f1_weighted": 0.271098 + }, + { + "accuracy": 0.323529, + "f1": 0.298347, + "f1_weighted": 0.317901 + }, + { + "accuracy": 0.362745, + "f1": 0.358903, + "f1_weighted": 0.361246 + }, + { + "accuracy": 0.318627, + "f1": 0.303966, + "f1_weighted": 0.314326 + }, + { + "accuracy": 0.328431, + "f1": 0.315324, + "f1_weighted": 0.340817 + } + ], + "main_score": 0.310784, + "hf_subset": "sna_Latn", + "languages": [ + "sna-Latn" + ] + }, + { + "accuracy": 0.176471, + "f1": 0.118813, + "f1_weighted": 0.125207, + "scores_per_experiment": [ + { + "accuracy": 0.254902, + "f1": 0.14787, + "f1_weighted": 0.184999 + }, + { + "accuracy": 0.161765, + "f1": 0.114187, + "f1_weighted": 0.099892 + }, + { + "accuracy": 0.132353, + "f1": 0.106352, + "f1_weighted": 0.085698 + }, + { + "accuracy": 0.215686, + "f1": 0.110223, + "f1_weighted": 0.117701 + }, + { + "accuracy": 0.186275, + "f1": 0.09979, + "f1_weighted": 0.121229 + }, + { + "accuracy": 0.186275, + "f1": 0.144536, + "f1_weighted": 0.142253 + }, + { + "accuracy": 0.161765, + "f1": 0.14206, + "f1_weighted": 0.151127 + }, + { + "accuracy": 0.22549, + "f1": 0.14406, + "f1_weighted": 0.17109 + }, + { + "accuracy": 0.122549, + "f1": 0.092243, + "f1_weighted": 0.08711 + }, + { + "accuracy": 0.117647, + "f1": 0.086805, + "f1_weighted": 0.090969 + } + ], + "main_score": 0.176471, + "hf_subset": "snd_Arab", + "languages": [ + "snd-Arab" + ] + }, + { + "accuracy": 0.251961, + "f1": 0.242066, + "f1_weighted": 0.253999, + "scores_per_experiment": [ + { + "accuracy": 0.27451, + "f1": 0.274344, + "f1_weighted": 0.277135 + }, + { + "accuracy": 0.259804, + "f1": 0.252914, + "f1_weighted": 0.27149 + }, + { + "accuracy": 0.240196, + "f1": 0.230123, + "f1_weighted": 0.230018 + }, + { + "accuracy": 0.230392, + "f1": 0.231615, + "f1_weighted": 0.22842 + }, + { + "accuracy": 0.240196, + "f1": 0.230215, + "f1_weighted": 0.249067 + }, + { + "accuracy": 0.284314, + "f1": 0.256877, + "f1_weighted": 0.27007 + }, + { + "accuracy": 0.210784, + "f1": 0.201584, + "f1_weighted": 0.212564 + }, + { + "accuracy": 0.230392, + "f1": 0.211804, + "f1_weighted": 0.231352 + }, + { + "accuracy": 0.235294, + "f1": 0.230022, + "f1_weighted": 0.252095 + }, + { + "accuracy": 0.313725, + "f1": 0.301163, + "f1_weighted": 0.317782 + } + ], + "main_score": 0.251961, + "hf_subset": "som_Latn", + "languages": [ + "som-Latn" + ] + }, + { + "accuracy": 0.295588, + "f1": 0.291316, + "f1_weighted": 0.302719, + "scores_per_experiment": [ + { + "accuracy": 0.328431, + "f1": 0.331345, + "f1_weighted": 0.342766 + }, + { + "accuracy": 0.245098, + "f1": 0.243517, + "f1_weighted": 0.251531 + }, + { + "accuracy": 0.235294, + "f1": 0.233725, + "f1_weighted": 0.238137 + }, + { + "accuracy": 0.279412, + "f1": 0.267918, + "f1_weighted": 0.292781 + }, + { + "accuracy": 0.289216, + "f1": 0.296804, + "f1_weighted": 0.295363 + }, + { + "accuracy": 0.318627, + "f1": 0.311871, + "f1_weighted": 0.322762 + }, + { + "accuracy": 0.279412, + "f1": 0.254791, + "f1_weighted": 0.270837 + }, + { + "accuracy": 0.343137, + "f1": 0.33059, + "f1_weighted": 0.3553 + }, + { + "accuracy": 0.318627, + "f1": 0.315855, + "f1_weighted": 0.324521 + }, + { + "accuracy": 0.318627, + "f1": 0.326741, + "f1_weighted": 0.33319 + } + ], + "main_score": 0.295588, + "hf_subset": "sot_Latn", + "languages": [ + "sot-Latn" + ] + }, + { + "accuracy": 0.593627, + "f1": 0.576132, + "f1_weighted": 0.59405, + "scores_per_experiment": [ + { + "accuracy": 0.637255, + "f1": 0.616978, + "f1_weighted": 0.634882 + }, + { + "accuracy": 0.602941, + "f1": 0.589507, + "f1_weighted": 0.592478 + }, + { + "accuracy": 0.598039, + "f1": 0.584386, + "f1_weighted": 0.598087 + }, + { + "accuracy": 0.583333, + "f1": 0.561295, + "f1_weighted": 0.580782 + }, + { + "accuracy": 0.593137, + "f1": 0.582571, + "f1_weighted": 0.594477 + }, + { + "accuracy": 0.612745, + "f1": 0.577263, + "f1_weighted": 0.611942 + }, + { + "accuracy": 0.578431, + "f1": 0.564451, + "f1_weighted": 0.585015 + }, + { + "accuracy": 0.52451, + "f1": 0.524265, + "f1_weighted": 0.530405 + }, + { + "accuracy": 0.622549, + "f1": 0.592336, + "f1_weighted": 0.618388 + }, + { + "accuracy": 0.583333, + "f1": 0.568272, + "f1_weighted": 0.594047 + } + ], + "main_score": 0.593627, + "hf_subset": "spa_Latn", + "languages": [ + "spa-Latn" + ] + }, + { + "accuracy": 0.408824, + "f1": 0.393049, + "f1_weighted": 0.41109, + "scores_per_experiment": [ + { + "accuracy": 0.436275, + "f1": 0.409046, + "f1_weighted": 0.437209 + }, + { + "accuracy": 0.362745, + "f1": 0.344794, + "f1_weighted": 0.369617 + }, + { + "accuracy": 0.377451, + "f1": 0.37624, + "f1_weighted": 0.374219 + }, + { + "accuracy": 0.446078, + "f1": 0.42912, + "f1_weighted": 0.442862 + }, + { + "accuracy": 0.426471, + "f1": 0.417785, + "f1_weighted": 0.435488 + }, + { + "accuracy": 0.431373, + "f1": 0.42553, + "f1_weighted": 0.435577 + }, + { + "accuracy": 0.416667, + "f1": 0.398817, + "f1_weighted": 0.41874 + }, + { + "accuracy": 0.411765, + "f1": 0.392222, + "f1_weighted": 0.419838 + }, + { + "accuracy": 0.377451, + "f1": 0.33905, + "f1_weighted": 0.372272 + }, + { + "accuracy": 0.401961, + "f1": 0.397888, + "f1_weighted": 0.405076 + } + ], + "main_score": 0.408824, + "hf_subset": "srd_Latn", + "languages": [ + "srd-Latn" + ] + }, + { + "accuracy": 0.22549, + "f1": 0.199498, + "f1_weighted": 0.21178, + "scores_per_experiment": [ + { + "accuracy": 0.269608, + "f1": 0.225493, + "f1_weighted": 0.257283 + }, + { + "accuracy": 0.210784, + "f1": 0.203983, + "f1_weighted": 0.212466 + }, + { + "accuracy": 0.181373, + "f1": 0.167794, + "f1_weighted": 0.171451 + }, + { + "accuracy": 0.240196, + "f1": 0.204005, + "f1_weighted": 0.226363 + }, + { + "accuracy": 0.230392, + "f1": 0.218027, + "f1_weighted": 0.209207 + }, + { + "accuracy": 0.210784, + "f1": 0.213691, + "f1_weighted": 0.208823 + }, + { + "accuracy": 0.245098, + "f1": 0.209385, + "f1_weighted": 0.212416 + }, + { + "accuracy": 0.245098, + "f1": 0.206196, + "f1_weighted": 0.230107 + }, + { + "accuracy": 0.230392, + "f1": 0.204686, + "f1_weighted": 0.222367 + }, + { + "accuracy": 0.191176, + "f1": 0.141715, + "f1_weighted": 0.167319 + } + ], + "main_score": 0.22549, + "hf_subset": "srp_Cyrl", + "languages": [ + "srp-Cyrl" + ] + }, + { + "accuracy": 0.3, + "f1": 0.290687, + "f1_weighted": 0.301916, + "scores_per_experiment": [ + { + "accuracy": 0.333333, + "f1": 0.329919, + "f1_weighted": 0.341651 + }, + { + "accuracy": 0.333333, + "f1": 0.336771, + "f1_weighted": 0.340057 + }, + { + "accuracy": 0.205882, + "f1": 0.192211, + "f1_weighted": 0.204536 + }, + { + "accuracy": 0.308824, + "f1": 0.301684, + "f1_weighted": 0.303944 + }, + { + "accuracy": 0.323529, + "f1": 0.313399, + "f1_weighted": 0.32426 + }, + { + "accuracy": 0.259804, + "f1": 0.245461, + "f1_weighted": 0.264265 + }, + { + "accuracy": 0.343137, + "f1": 0.312036, + "f1_weighted": 0.329761 + }, + { + "accuracy": 0.29902, + "f1": 0.285628, + "f1_weighted": 0.298045 + }, + { + "accuracy": 0.264706, + "f1": 0.258261, + "f1_weighted": 0.268956 + }, + { + "accuracy": 0.328431, + "f1": 0.3315, + "f1_weighted": 0.343688 + } + ], + "main_score": 0.3, + "hf_subset": "ssw_Latn", + "languages": [ + "ssw-Latn" + ] + }, + { + "accuracy": 0.352941, + "f1": 0.337022, + "f1_weighted": 0.358775, + "scores_per_experiment": [ + { + "accuracy": 0.387255, + "f1": 0.370579, + "f1_weighted": 0.404108 + }, + { + "accuracy": 0.367647, + "f1": 0.340519, + "f1_weighted": 0.364417 + }, + { + "accuracy": 0.372549, + "f1": 0.370659, + "f1_weighted": 0.374228 + }, + { + "accuracy": 0.318627, + "f1": 0.315575, + "f1_weighted": 0.322821 + }, + { + "accuracy": 0.348039, + "f1": 0.333877, + "f1_weighted": 0.357117 + }, + { + "accuracy": 0.303922, + "f1": 0.290791, + "f1_weighted": 0.311342 + }, + { + "accuracy": 0.348039, + "f1": 0.314775, + "f1_weighted": 0.35292 + }, + { + "accuracy": 0.362745, + "f1": 0.348159, + "f1_weighted": 0.373241 + }, + { + "accuracy": 0.392157, + "f1": 0.371093, + "f1_weighted": 0.392069 + }, + { + "accuracy": 0.328431, + "f1": 0.314189, + "f1_weighted": 0.335487 + } + ], + "main_score": 0.352941, + "hf_subset": "sun_Latn", + "languages": [ + "sun-Latn" + ] + }, + { + "accuracy": 0.447059, + "f1": 0.435303, + "f1_weighted": 0.446303, + "scores_per_experiment": [ + { + "accuracy": 0.490196, + "f1": 0.488012, + "f1_weighted": 0.490377 + }, + { + "accuracy": 0.446078, + "f1": 0.440138, + "f1_weighted": 0.450722 + }, + { + "accuracy": 0.446078, + "f1": 0.43025, + "f1_weighted": 0.426114 + }, + { + "accuracy": 0.480392, + "f1": 0.469951, + "f1_weighted": 0.480813 + }, + { + "accuracy": 0.495098, + "f1": 0.492889, + "f1_weighted": 0.495227 + }, + { + "accuracy": 0.392157, + "f1": 0.380691, + "f1_weighted": 0.392264 + }, + { + "accuracy": 0.446078, + "f1": 0.418335, + "f1_weighted": 0.444432 + }, + { + "accuracy": 0.397059, + "f1": 0.387261, + "f1_weighted": 0.409174 + }, + { + "accuracy": 0.401961, + "f1": 0.37476, + "f1_weighted": 0.389704 + }, + { + "accuracy": 0.47549, + "f1": 0.470745, + "f1_weighted": 0.484207 + } + ], + "main_score": 0.447059, + "hf_subset": "swe_Latn", + "languages": [ + "swe-Latn" + ] + }, + { + "accuracy": 0.318627, + "f1": 0.308303, + "f1_weighted": 0.322353, + "scores_per_experiment": [ + { + "accuracy": 0.377451, + "f1": 0.350874, + "f1_weighted": 0.376239 + }, + { + "accuracy": 0.343137, + "f1": 0.334232, + "f1_weighted": 0.350184 + }, + { + "accuracy": 0.27451, + "f1": 0.268745, + "f1_weighted": 0.286197 + }, + { + "accuracy": 0.289216, + "f1": 0.26986, + "f1_weighted": 0.276792 + }, + { + "accuracy": 0.318627, + "f1": 0.314917, + "f1_weighted": 0.321813 + }, + { + "accuracy": 0.323529, + "f1": 0.326503, + "f1_weighted": 0.329725 + }, + { + "accuracy": 0.294118, + "f1": 0.282032, + "f1_weighted": 0.293573 + }, + { + "accuracy": 0.289216, + "f1": 0.293904, + "f1_weighted": 0.299266 + }, + { + "accuracy": 0.377451, + "f1": 0.349604, + "f1_weighted": 0.384376 + }, + { + "accuracy": 0.29902, + "f1": 0.29236, + "f1_weighted": 0.30536 + } + ], + "main_score": 0.318627, + "hf_subset": "swh_Latn", + "languages": [ + "swh-Latn" + ] + }, + { + "accuracy": 0.314216, + "f1": 0.30749, + "f1_weighted": 0.317255, + "scores_per_experiment": [ + { + "accuracy": 0.362745, + "f1": 0.35918, + "f1_weighted": 0.367908 + }, + { + "accuracy": 0.294118, + "f1": 0.274875, + "f1_weighted": 0.30021 + }, + { + "accuracy": 0.294118, + "f1": 0.284373, + "f1_weighted": 0.299961 + }, + { + "accuracy": 0.279412, + "f1": 0.285845, + "f1_weighted": 0.286018 + }, + { + "accuracy": 0.303922, + "f1": 0.30863, + "f1_weighted": 0.30415 + }, + { + "accuracy": 0.269608, + "f1": 0.252321, + "f1_weighted": 0.246727 + }, + { + "accuracy": 0.343137, + "f1": 0.326482, + "f1_weighted": 0.347676 + }, + { + "accuracy": 0.372549, + "f1": 0.365915, + "f1_weighted": 0.386866 + }, + { + "accuracy": 0.289216, + "f1": 0.274678, + "f1_weighted": 0.297136 + }, + { + "accuracy": 0.333333, + "f1": 0.342595, + "f1_weighted": 0.335899 + } + ], + "main_score": 0.314216, + "hf_subset": "szl_Latn", + "languages": [ + "szl-Latn" + ] + }, + { + "accuracy": 0.188235, + "f1": 0.141063, + "f1_weighted": 0.153125, + "scores_per_experiment": [ + { + "accuracy": 0.230392, + "f1": 0.153242, + "f1_weighted": 0.179612 + }, + { + "accuracy": 0.176471, + "f1": 0.131036, + "f1_weighted": 0.145795 + }, + { + "accuracy": 0.156863, + "f1": 0.142845, + "f1_weighted": 0.124226 + }, + { + "accuracy": 0.196078, + "f1": 0.159697, + "f1_weighted": 0.175204 + }, + { + "accuracy": 0.122549, + "f1": 0.109994, + "f1_weighted": 0.104747 + }, + { + "accuracy": 0.166667, + "f1": 0.11509, + "f1_weighted": 0.122699 + }, + { + "accuracy": 0.22549, + "f1": 0.152898, + "f1_weighted": 0.187273 + }, + { + "accuracy": 0.215686, + "f1": 0.169948, + "f1_weighted": 0.186687 + }, + { + "accuracy": 0.215686, + "f1": 0.148086, + "f1_weighted": 0.175684 + }, + { + "accuracy": 0.176471, + "f1": 0.127795, + "f1_weighted": 0.129321 + } + ], + "main_score": 0.188235, + "hf_subset": "tam_Taml", + "languages": [ + "tam-Taml" + ] + }, + { + "accuracy": 0.172059, + "f1": 0.131998, + "f1_weighted": 0.140142, + "scores_per_experiment": [ + { + "accuracy": 0.210784, + "f1": 0.127644, + "f1_weighted": 0.142535 + }, + { + "accuracy": 0.151961, + "f1": 0.13056, + "f1_weighted": 0.140022 + }, + { + "accuracy": 0.137255, + "f1": 0.113135, + "f1_weighted": 0.096167 + }, + { + "accuracy": 0.122549, + "f1": 0.100567, + "f1_weighted": 0.095828 + }, + { + "accuracy": 0.196078, + "f1": 0.151551, + "f1_weighted": 0.163081 + }, + { + "accuracy": 0.166667, + "f1": 0.121202, + "f1_weighted": 0.130062 + }, + { + "accuracy": 0.230392, + "f1": 0.175888, + "f1_weighted": 0.206045 + }, + { + "accuracy": 0.191176, + "f1": 0.154467, + "f1_weighted": 0.155423 + }, + { + "accuracy": 0.186275, + "f1": 0.142795, + "f1_weighted": 0.17773 + }, + { + "accuracy": 0.127451, + "f1": 0.10217, + "f1_weighted": 0.094529 + } + ], + "main_score": 0.172059, + "hf_subset": "taq_Tfng", + "languages": [ + "taq-Tfng" + ] + }, + { + "accuracy": 0.204902, + "f1": 0.18159, + "f1_weighted": 0.193382, + "scores_per_experiment": [ + { + "accuracy": 0.29902, + "f1": 0.223857, + "f1_weighted": 0.272251 + }, + { + "accuracy": 0.171569, + "f1": 0.162496, + "f1_weighted": 0.158593 + }, + { + "accuracy": 0.196078, + "f1": 0.168027, + "f1_weighted": 0.195364 + }, + { + "accuracy": 0.181373, + "f1": 0.152874, + "f1_weighted": 0.16915 + }, + { + "accuracy": 0.186275, + "f1": 0.191018, + "f1_weighted": 0.192781 + }, + { + "accuracy": 0.215686, + "f1": 0.201841, + "f1_weighted": 0.191948 + }, + { + "accuracy": 0.25, + "f1": 0.236975, + "f1_weighted": 0.240706 + }, + { + "accuracy": 0.186275, + "f1": 0.16478, + "f1_weighted": 0.180191 + }, + { + "accuracy": 0.176471, + "f1": 0.14612, + "f1_weighted": 0.160882 + }, + { + "accuracy": 0.186275, + "f1": 0.167909, + "f1_weighted": 0.171955 + } + ], + "main_score": 0.204902, + "hf_subset": "tat_Cyrl", + "languages": [ + "tat-Cyrl" + ] + }, + { + "accuracy": 0.186765, + "f1": 0.150792, + "f1_weighted": 0.157835, + "scores_per_experiment": [ + { + "accuracy": 0.259804, + "f1": 0.177365, + "f1_weighted": 0.196277 + }, + { + "accuracy": 0.181373, + "f1": 0.157994, + "f1_weighted": 0.154177 + }, + { + "accuracy": 0.112745, + "f1": 0.105213, + "f1_weighted": 0.085986 + }, + { + "accuracy": 0.191176, + "f1": 0.192661, + "f1_weighted": 0.181015 + }, + { + "accuracy": 0.171569, + "f1": 0.135871, + "f1_weighted": 0.13955 + }, + { + "accuracy": 0.186275, + "f1": 0.163995, + "f1_weighted": 0.157371 + }, + { + "accuracy": 0.220588, + "f1": 0.146893, + "f1_weighted": 0.16664 + }, + { + "accuracy": 0.245098, + "f1": 0.190519, + "f1_weighted": 0.226205 + }, + { + "accuracy": 0.161765, + "f1": 0.132146, + "f1_weighted": 0.143634 + }, + { + "accuracy": 0.137255, + "f1": 0.105263, + "f1_weighted": 0.127495 + } + ], + "main_score": 0.186765, + "hf_subset": "tel_Telu", + "languages": [ + "tel-Telu" + ] + }, + { + "accuracy": 0.2, + "f1": 0.179041, + "f1_weighted": 0.189301, + "scores_per_experiment": [ + { + "accuracy": 0.245098, + "f1": 0.201457, + "f1_weighted": 0.232754 + }, + { + "accuracy": 0.171569, + "f1": 0.165292, + "f1_weighted": 0.161221 + }, + { + "accuracy": 0.181373, + "f1": 0.173253, + "f1_weighted": 0.177034 + }, + { + "accuracy": 0.289216, + "f1": 0.236572, + "f1_weighted": 0.267983 + }, + { + "accuracy": 0.147059, + "f1": 0.12382, + "f1_weighted": 0.113026 + }, + { + "accuracy": 0.181373, + "f1": 0.184728, + "f1_weighted": 0.180474 + }, + { + "accuracy": 0.215686, + "f1": 0.196804, + "f1_weighted": 0.208719 + }, + { + "accuracy": 0.171569, + "f1": 0.160244, + "f1_weighted": 0.173793 + }, + { + "accuracy": 0.254902, + "f1": 0.226508, + "f1_weighted": 0.24981 + }, + { + "accuracy": 0.142157, + "f1": 0.121735, + "f1_weighted": 0.128199 + } + ], + "main_score": 0.2, + "hf_subset": "tgk_Cyrl", + "languages": [ + "tgk-Cyrl" + ] + }, + { + "accuracy": 0.441667, + "f1": 0.420356, + "f1_weighted": 0.448474, + "scores_per_experiment": [ + { + "accuracy": 0.539216, + "f1": 0.506014, + "f1_weighted": 0.542424 + }, + { + "accuracy": 0.416667, + "f1": 0.378374, + "f1_weighted": 0.422105 + }, + { + "accuracy": 0.411765, + "f1": 0.388919, + "f1_weighted": 0.419292 + }, + { + "accuracy": 0.426471, + "f1": 0.39769, + "f1_weighted": 0.430533 + }, + { + "accuracy": 0.416667, + "f1": 0.403629, + "f1_weighted": 0.425618 + }, + { + "accuracy": 0.421569, + "f1": 0.398171, + "f1_weighted": 0.426958 + }, + { + "accuracy": 0.465686, + "f1": 0.436639, + "f1_weighted": 0.466641 + }, + { + "accuracy": 0.446078, + "f1": 0.435644, + "f1_weighted": 0.457562 + }, + { + "accuracy": 0.431373, + "f1": 0.409049, + "f1_weighted": 0.438298 + }, + { + "accuracy": 0.441176, + "f1": 0.449436, + "f1_weighted": 0.455306 + } + ], + "main_score": 0.441667, + "hf_subset": "tgl_Latn", + "languages": [ + "tgl-Latn" + ] + }, + { + "accuracy": 0.19951, + "f1": 0.151471, + "f1_weighted": 0.171795, + "scores_per_experiment": [ + { + "accuracy": 0.181373, + "f1": 0.170826, + "f1_weighted": 0.189377 + }, + { + "accuracy": 0.235294, + "f1": 0.153913, + "f1_weighted": 0.192486 + }, + { + "accuracy": 0.161765, + "f1": 0.132082, + "f1_weighted": 0.14368 + }, + { + "accuracy": 0.220588, + "f1": 0.186498, + "f1_weighted": 0.1996 + }, + { + "accuracy": 0.191176, + "f1": 0.139226, + "f1_weighted": 0.14108 + }, + { + "accuracy": 0.230392, + "f1": 0.151217, + "f1_weighted": 0.169721 + }, + { + "accuracy": 0.186275, + "f1": 0.128633, + "f1_weighted": 0.154863 + }, + { + "accuracy": 0.235294, + "f1": 0.169968, + "f1_weighted": 0.205355 + }, + { + "accuracy": 0.137255, + "f1": 0.119115, + "f1_weighted": 0.124294 + }, + { + "accuracy": 0.215686, + "f1": 0.163234, + "f1_weighted": 0.19749 + } + ], + "main_score": 0.19951, + "hf_subset": "tha_Thai", + "languages": [ + "tha-Thai" + ] + }, + { + "accuracy": 0.195588, + "f1": 0.137816, + "f1_weighted": 0.148724, + "scores_per_experiment": [ + { + "accuracy": 0.25, + "f1": 0.15151, + "f1_weighted": 0.168985 + }, + { + "accuracy": 0.156863, + "f1": 0.100291, + "f1_weighted": 0.097499 + }, + { + "accuracy": 0.137255, + "f1": 0.106364, + "f1_weighted": 0.079655 + }, + { + "accuracy": 0.230392, + "f1": 0.148207, + "f1_weighted": 0.168115 + }, + { + "accuracy": 0.151961, + "f1": 0.137253, + "f1_weighted": 0.147617 + }, + { + "accuracy": 0.181373, + "f1": 0.170494, + "f1_weighted": 0.156936 + }, + { + "accuracy": 0.22549, + "f1": 0.149978, + "f1_weighted": 0.178155 + }, + { + "accuracy": 0.240196, + "f1": 0.17755, + "f1_weighted": 0.196182 + }, + { + "accuracy": 0.259804, + "f1": 0.152809, + "f1_weighted": 0.208755 + }, + { + "accuracy": 0.122549, + "f1": 0.083709, + "f1_weighted": 0.08534 + } + ], + "main_score": 0.195588, + "hf_subset": "tir_Ethi", + "languages": [ + "tir-Ethi" + ] + }, + { + "accuracy": 0.511765, + "f1": 0.49305, + "f1_weighted": 0.515543, + "scores_per_experiment": [ + { + "accuracy": 0.52451, + "f1": 0.518757, + "f1_weighted": 0.525505 + }, + { + "accuracy": 0.509804, + "f1": 0.480197, + "f1_weighted": 0.507761 + }, + { + "accuracy": 0.490196, + "f1": 0.479212, + "f1_weighted": 0.492562 + }, + { + "accuracy": 0.460784, + "f1": 0.448817, + "f1_weighted": 0.474538 + }, + { + "accuracy": 0.490196, + "f1": 0.478509, + "f1_weighted": 0.500865 + }, + { + "accuracy": 0.568627, + "f1": 0.539527, + "f1_weighted": 0.574362 + }, + { + "accuracy": 0.563725, + "f1": 0.525185, + "f1_weighted": 0.568397 + }, + { + "accuracy": 0.504902, + "f1": 0.499434, + "f1_weighted": 0.505526 + }, + { + "accuracy": 0.485294, + "f1": 0.449361, + "f1_weighted": 0.48394 + }, + { + "accuracy": 0.519608, + "f1": 0.511497, + "f1_weighted": 0.521979 + } + ], + "main_score": 0.511765, + "hf_subset": "tpi_Latn", + "languages": [ + "tpi-Latn" + ] + }, + { + "accuracy": 0.330392, + "f1": 0.317531, + "f1_weighted": 0.330479, + "scores_per_experiment": [ + { + "accuracy": 0.387255, + "f1": 0.361481, + "f1_weighted": 0.390523 + }, + { + "accuracy": 0.25, + "f1": 0.258027, + "f1_weighted": 0.248787 + }, + { + "accuracy": 0.387255, + "f1": 0.3622, + "f1_weighted": 0.380867 + }, + { + "accuracy": 0.25, + "f1": 0.234957, + "f1_weighted": 0.254328 + }, + { + "accuracy": 0.29902, + "f1": 0.280689, + "f1_weighted": 0.286617 + }, + { + "accuracy": 0.348039, + "f1": 0.316559, + "f1_weighted": 0.342108 + }, + { + "accuracy": 0.348039, + "f1": 0.349004, + "f1_weighted": 0.347051 + }, + { + "accuracy": 0.377451, + "f1": 0.368499, + "f1_weighted": 0.375107 + }, + { + "accuracy": 0.323529, + "f1": 0.316527, + "f1_weighted": 0.334285 + }, + { + "accuracy": 0.333333, + "f1": 0.327366, + "f1_weighted": 0.345118 + } + ], + "main_score": 0.330392, + "hf_subset": "tsn_Latn", + "languages": [ + "tsn-Latn" + ] + }, + { + "accuracy": 0.315686, + "f1": 0.298483, + "f1_weighted": 0.319278, + "scores_per_experiment": [ + { + "accuracy": 0.362745, + "f1": 0.337316, + "f1_weighted": 0.368469 + }, + { + "accuracy": 0.303922, + "f1": 0.282009, + "f1_weighted": 0.311748 + }, + { + "accuracy": 0.235294, + "f1": 0.228144, + "f1_weighted": 0.231902 + }, + { + "accuracy": 0.284314, + "f1": 0.26246, + "f1_weighted": 0.295141 + }, + { + "accuracy": 0.411765, + "f1": 0.391456, + "f1_weighted": 0.410976 + }, + { + "accuracy": 0.259804, + "f1": 0.245058, + "f1_weighted": 0.275642 + }, + { + "accuracy": 0.308824, + "f1": 0.289766, + "f1_weighted": 0.306227 + }, + { + "accuracy": 0.328431, + "f1": 0.314388, + "f1_weighted": 0.331589 + }, + { + "accuracy": 0.284314, + "f1": 0.271113, + "f1_weighted": 0.286061 + }, + { + "accuracy": 0.377451, + "f1": 0.363116, + "f1_weighted": 0.375024 + } + ], + "main_score": 0.315686, + "hf_subset": "tso_Latn", + "languages": [ + "tso-Latn" + ] + }, + { + "accuracy": 0.316176, + "f1": 0.306427, + "f1_weighted": 0.320385, + "scores_per_experiment": [ + { + "accuracy": 0.318627, + "f1": 0.31179, + "f1_weighted": 0.327233 + }, + { + "accuracy": 0.284314, + "f1": 0.274347, + "f1_weighted": 0.277734 + }, + { + "accuracy": 0.323529, + "f1": 0.313897, + "f1_weighted": 0.328241 + }, + { + "accuracy": 0.362745, + "f1": 0.343217, + "f1_weighted": 0.367101 + }, + { + "accuracy": 0.289216, + "f1": 0.290948, + "f1_weighted": 0.290121 + }, + { + "accuracy": 0.338235, + "f1": 0.323538, + "f1_weighted": 0.340177 + }, + { + "accuracy": 0.367647, + "f1": 0.336498, + "f1_weighted": 0.358531 + }, + { + "accuracy": 0.333333, + "f1": 0.337708, + "f1_weighted": 0.35392 + }, + { + "accuracy": 0.264706, + "f1": 0.255808, + "f1_weighted": 0.27535 + }, + { + "accuracy": 0.279412, + "f1": 0.276518, + "f1_weighted": 0.28544 + } + ], + "main_score": 0.316176, + "hf_subset": "tuk_Latn", + "languages": [ + "tuk-Latn" + ] + }, + { + "accuracy": 0.340686, + "f1": 0.325941, + "f1_weighted": 0.342938, + "scores_per_experiment": [ + { + "accuracy": 0.397059, + "f1": 0.367119, + "f1_weighted": 0.396473 + }, + { + "accuracy": 0.338235, + "f1": 0.331187, + "f1_weighted": 0.348589 + }, + { + "accuracy": 0.308824, + "f1": 0.300721, + "f1_weighted": 0.312739 + }, + { + "accuracy": 0.323529, + "f1": 0.316842, + "f1_weighted": 0.32706 + }, + { + "accuracy": 0.352941, + "f1": 0.31797, + "f1_weighted": 0.352312 + }, + { + "accuracy": 0.294118, + "f1": 0.284642, + "f1_weighted": 0.289676 + }, + { + "accuracy": 0.308824, + "f1": 0.285683, + "f1_weighted": 0.296854 + }, + { + "accuracy": 0.343137, + "f1": 0.329079, + "f1_weighted": 0.335887 + }, + { + "accuracy": 0.338235, + "f1": 0.334209, + "f1_weighted": 0.356271 + }, + { + "accuracy": 0.401961, + "f1": 0.391956, + "f1_weighted": 0.413516 + } + ], + "main_score": 0.340686, + "hf_subset": "tum_Latn", + "languages": [ + "tum-Latn" + ] + }, + { + "accuracy": 0.317157, + "f1": 0.305571, + "f1_weighted": 0.318488, + "scores_per_experiment": [ + { + "accuracy": 0.323529, + "f1": 0.316057, + "f1_weighted": 0.343133 + }, + { + "accuracy": 0.269608, + "f1": 0.267375, + "f1_weighted": 0.255666 + }, + { + "accuracy": 0.308824, + "f1": 0.301017, + "f1_weighted": 0.319875 + }, + { + "accuracy": 0.372549, + "f1": 0.356782, + "f1_weighted": 0.367599 + }, + { + "accuracy": 0.27451, + "f1": 0.268385, + "f1_weighted": 0.272712 + }, + { + "accuracy": 0.308824, + "f1": 0.287189, + "f1_weighted": 0.312332 + }, + { + "accuracy": 0.362745, + "f1": 0.358776, + "f1_weighted": 0.365879 + }, + { + "accuracy": 0.308824, + "f1": 0.297195, + "f1_weighted": 0.312202 + }, + { + "accuracy": 0.323529, + "f1": 0.286351, + "f1_weighted": 0.316348 + }, + { + "accuracy": 0.318627, + "f1": 0.316587, + "f1_weighted": 0.319138 + } + ], + "main_score": 0.317157, + "hf_subset": "tur_Latn", + "languages": [ + "tur-Latn" + ] + }, + { + "accuracy": 0.392647, + "f1": 0.377337, + "f1_weighted": 0.391736, + "scores_per_experiment": [ + { + "accuracy": 0.352941, + "f1": 0.315302, + "f1_weighted": 0.347086 + }, + { + "accuracy": 0.406863, + "f1": 0.395848, + "f1_weighted": 0.404391 + }, + { + "accuracy": 0.357843, + "f1": 0.350987, + "f1_weighted": 0.347374 + }, + { + "accuracy": 0.372549, + "f1": 0.363246, + "f1_weighted": 0.380167 + }, + { + "accuracy": 0.392157, + "f1": 0.395624, + "f1_weighted": 0.397123 + }, + { + "accuracy": 0.333333, + "f1": 0.327456, + "f1_weighted": 0.331017 + }, + { + "accuracy": 0.421569, + "f1": 0.396648, + "f1_weighted": 0.424699 + }, + { + "accuracy": 0.406863, + "f1": 0.391624, + "f1_weighted": 0.405967 + }, + { + "accuracy": 0.421569, + "f1": 0.381442, + "f1_weighted": 0.420107 + }, + { + "accuracy": 0.460784, + "f1": 0.455196, + "f1_weighted": 0.45943 + } + ], + "main_score": 0.392647, + "hf_subset": "twi_Latn", + "languages": [ + "twi-Latn" + ] + }, + { + "accuracy": 0.161765, + "f1": 0.12606, + "f1_weighted": 0.127887, + "scores_per_experiment": [ + { + "accuracy": 0.156863, + "f1": 0.114941, + "f1_weighted": 0.132246 + }, + { + "accuracy": 0.176471, + "f1": 0.137069, + "f1_weighted": 0.135424 + }, + { + "accuracy": 0.142157, + "f1": 0.104926, + "f1_weighted": 0.087358 + }, + { + "accuracy": 0.142157, + "f1": 0.105844, + "f1_weighted": 0.097259 + }, + { + "accuracy": 0.166667, + "f1": 0.153412, + "f1_weighted": 0.150406 + }, + { + "accuracy": 0.181373, + "f1": 0.159286, + "f1_weighted": 0.162359 + }, + { + "accuracy": 0.171569, + "f1": 0.151187, + "f1_weighted": 0.145362 + }, + { + "accuracy": 0.166667, + "f1": 0.124031, + "f1_weighted": 0.12958 + }, + { + "accuracy": 0.151961, + "f1": 0.110876, + "f1_weighted": 0.132515 + }, + { + "accuracy": 0.161765, + "f1": 0.09903, + "f1_weighted": 0.106356 + } + ], + "main_score": 0.161765, + "hf_subset": "tzm_Tfng", + "languages": [ + "tzm-Tfng" + ] + }, + { + "accuracy": 0.172059, + "f1": 0.127611, + "f1_weighted": 0.135625, + "scores_per_experiment": [ + { + "accuracy": 0.25, + "f1": 0.16183, + "f1_weighted": 0.182262 + }, + { + "accuracy": 0.147059, + "f1": 0.111276, + "f1_weighted": 0.092784 + }, + { + "accuracy": 0.151961, + "f1": 0.134704, + "f1_weighted": 0.135322 + }, + { + "accuracy": 0.20098, + "f1": 0.116207, + "f1_weighted": 0.136997 + }, + { + "accuracy": 0.107843, + "f1": 0.092118, + "f1_weighted": 0.086166 + }, + { + "accuracy": 0.181373, + "f1": 0.153862, + "f1_weighted": 0.159981 + }, + { + "accuracy": 0.186275, + "f1": 0.150496, + "f1_weighted": 0.162063 + }, + { + "accuracy": 0.254902, + "f1": 0.186357, + "f1_weighted": 0.222685 + }, + { + "accuracy": 0.142157, + "f1": 0.10363, + "f1_weighted": 0.116892 + }, + { + "accuracy": 0.098039, + "f1": 0.065635, + "f1_weighted": 0.061096 + } + ], + "main_score": 0.172059, + "hf_subset": "uig_Arab", + "languages": [ + "uig-Arab" + ] + }, + { + "accuracy": 0.210294, + "f1": 0.18245, + "f1_weighted": 0.203412, + "scores_per_experiment": [ + { + "accuracy": 0.245098, + "f1": 0.171344, + "f1_weighted": 0.214704 + }, + { + "accuracy": 0.196078, + "f1": 0.17788, + "f1_weighted": 0.179885 + }, + { + "accuracy": 0.191176, + "f1": 0.187609, + "f1_weighted": 0.199847 + }, + { + "accuracy": 0.196078, + "f1": 0.173352, + "f1_weighted": 0.196161 + }, + { + "accuracy": 0.151961, + "f1": 0.151187, + "f1_weighted": 0.14233 + }, + { + "accuracy": 0.176471, + "f1": 0.165071, + "f1_weighted": 0.178569 + }, + { + "accuracy": 0.22549, + "f1": 0.208143, + "f1_weighted": 0.215248 + }, + { + "accuracy": 0.259804, + "f1": 0.222716, + "f1_weighted": 0.260315 + }, + { + "accuracy": 0.264706, + "f1": 0.211594, + "f1_weighted": 0.258425 + }, + { + "accuracy": 0.196078, + "f1": 0.155608, + "f1_weighted": 0.188631 + } + ], + "main_score": 0.210294, + "hf_subset": "ukr_Cyrl", + "languages": [ + "ukr-Cyrl" + ] + }, + { + "accuracy": 0.296569, + "f1": 0.27473, + "f1_weighted": 0.299185, + "scores_per_experiment": [ + { + "accuracy": 0.392157, + "f1": 0.349414, + "f1_weighted": 0.389049 + }, + { + "accuracy": 0.240196, + "f1": 0.238065, + "f1_weighted": 0.254613 + }, + { + "accuracy": 0.27451, + "f1": 0.258838, + "f1_weighted": 0.263387 + }, + { + "accuracy": 0.27451, + "f1": 0.256485, + "f1_weighted": 0.284368 + }, + { + "accuracy": 0.29902, + "f1": 0.277599, + "f1_weighted": 0.294992 + }, + { + "accuracy": 0.279412, + "f1": 0.261668, + "f1_weighted": 0.281947 + }, + { + "accuracy": 0.284314, + "f1": 0.248558, + "f1_weighted": 0.285616 + }, + { + "accuracy": 0.294118, + "f1": 0.276707, + "f1_weighted": 0.291588 + }, + { + "accuracy": 0.294118, + "f1": 0.271626, + "f1_weighted": 0.306627 + }, + { + "accuracy": 0.333333, + "f1": 0.308335, + "f1_weighted": 0.339662 + } + ], + "main_score": 0.296569, + "hf_subset": "umb_Latn", + "languages": [ + "umb-Latn" + ] + }, + { + "accuracy": 0.176471, + "f1": 0.119484, + "f1_weighted": 0.125525, + "scores_per_experiment": [ + { + "accuracy": 0.240196, + "f1": 0.136846, + "f1_weighted": 0.163736 + }, + { + "accuracy": 0.166667, + "f1": 0.135395, + "f1_weighted": 0.11524 + }, + { + "accuracy": 0.132353, + "f1": 0.104611, + "f1_weighted": 0.084979 + }, + { + "accuracy": 0.215686, + "f1": 0.123225, + "f1_weighted": 0.138152 + }, + { + "accuracy": 0.176471, + "f1": 0.089727, + "f1_weighted": 0.103394 + }, + { + "accuracy": 0.20098, + "f1": 0.168849, + "f1_weighted": 0.179609 + }, + { + "accuracy": 0.191176, + "f1": 0.130084, + "f1_weighted": 0.153474 + }, + { + "accuracy": 0.132353, + "f1": 0.085469, + "f1_weighted": 0.077311 + }, + { + "accuracy": 0.151961, + "f1": 0.107275, + "f1_weighted": 0.110022 + }, + { + "accuracy": 0.156863, + "f1": 0.113361, + "f1_weighted": 0.129332 + } + ], + "main_score": 0.176471, + "hf_subset": "urd_Arab", + "languages": [ + "urd-Arab" + ] + }, + { + "accuracy": 0.268627, + "f1": 0.265118, + "f1_weighted": 0.269569, + "scores_per_experiment": [ + { + "accuracy": 0.289216, + "f1": 0.283583, + "f1_weighted": 0.294421 + }, + { + "accuracy": 0.240196, + "f1": 0.239716, + "f1_weighted": 0.231571 + }, + { + "accuracy": 0.264706, + "f1": 0.257212, + "f1_weighted": 0.275556 + }, + { + "accuracy": 0.240196, + "f1": 0.238783, + "f1_weighted": 0.235281 + }, + { + "accuracy": 0.254902, + "f1": 0.245844, + "f1_weighted": 0.25256 + }, + { + "accuracy": 0.264706, + "f1": 0.273355, + "f1_weighted": 0.25439 + }, + { + "accuracy": 0.318627, + "f1": 0.308844, + "f1_weighted": 0.32716 + }, + { + "accuracy": 0.27451, + "f1": 0.282308, + "f1_weighted": 0.280733 + }, + { + "accuracy": 0.25, + "f1": 0.229249, + "f1_weighted": 0.25637 + }, + { + "accuracy": 0.289216, + "f1": 0.29229, + "f1_weighted": 0.287645 + } + ], + "main_score": 0.268627, + "hf_subset": "uzn_Latn", + "languages": [ + "uzn-Latn" + ] + }, + { + "accuracy": 0.457353, + "f1": 0.434971, + "f1_weighted": 0.458533, + "scores_per_experiment": [ + { + "accuracy": 0.465686, + "f1": 0.433315, + "f1_weighted": 0.475903 + }, + { + "accuracy": 0.446078, + "f1": 0.428049, + "f1_weighted": 0.452271 + }, + { + "accuracy": 0.397059, + "f1": 0.384985, + "f1_weighted": 0.395336 + }, + { + "accuracy": 0.47549, + "f1": 0.452006, + "f1_weighted": 0.47266 + }, + { + "accuracy": 0.52451, + "f1": 0.509686, + "f1_weighted": 0.534361 + }, + { + "accuracy": 0.431373, + "f1": 0.41357, + "f1_weighted": 0.434284 + }, + { + "accuracy": 0.47549, + "f1": 0.455963, + "f1_weighted": 0.476276 + }, + { + "accuracy": 0.455882, + "f1": 0.441762, + "f1_weighted": 0.446763 + }, + { + "accuracy": 0.455882, + "f1": 0.419267, + "f1_weighted": 0.454443 + }, + { + "accuracy": 0.446078, + "f1": 0.411103, + "f1_weighted": 0.443034 + } + ], + "main_score": 0.457353, + "hf_subset": "vec_Latn", + "languages": [ + "vec-Latn" + ] + }, + { + "accuracy": 0.308333, + "f1": 0.29505, + "f1_weighted": 0.305784, + "scores_per_experiment": [ + { + "accuracy": 0.29902, + "f1": 0.299216, + "f1_weighted": 0.309701 + }, + { + "accuracy": 0.245098, + "f1": 0.230011, + "f1_weighted": 0.221884 + }, + { + "accuracy": 0.279412, + "f1": 0.26975, + "f1_weighted": 0.287346 + }, + { + "accuracy": 0.323529, + "f1": 0.315729, + "f1_weighted": 0.334932 + }, + { + "accuracy": 0.323529, + "f1": 0.305601, + "f1_weighted": 0.32336 + }, + { + "accuracy": 0.318627, + "f1": 0.249617, + "f1_weighted": 0.274316 + }, + { + "accuracy": 0.338235, + "f1": 0.316105, + "f1_weighted": 0.322827 + }, + { + "accuracy": 0.348039, + "f1": 0.363994, + "f1_weighted": 0.358114 + }, + { + "accuracy": 0.328431, + "f1": 0.325873, + "f1_weighted": 0.341662 + }, + { + "accuracy": 0.279412, + "f1": 0.274603, + "f1_weighted": 0.283698 + } + ], + "main_score": 0.308333, + "hf_subset": "vie_Latn", + "languages": [ + "vie-Latn" + ] + }, + { + "accuracy": 0.486765, + "f1": 0.46983, + "f1_weighted": 0.491873, + "scores_per_experiment": [ + { + "accuracy": 0.539216, + "f1": 0.518548, + "f1_weighted": 0.545805 + }, + { + "accuracy": 0.47549, + "f1": 0.461456, + "f1_weighted": 0.464455 + }, + { + "accuracy": 0.5, + "f1": 0.495148, + "f1_weighted": 0.502324 + }, + { + "accuracy": 0.490196, + "f1": 0.468441, + "f1_weighted": 0.490309 + }, + { + "accuracy": 0.553922, + "f1": 0.561118, + "f1_weighted": 0.556484 + }, + { + "accuracy": 0.455882, + "f1": 0.416703, + "f1_weighted": 0.456874 + }, + { + "accuracy": 0.460784, + "f1": 0.423051, + "f1_weighted": 0.467299 + }, + { + "accuracy": 0.485294, + "f1": 0.464381, + "f1_weighted": 0.490004 + }, + { + "accuracy": 0.441176, + "f1": 0.419654, + "f1_weighted": 0.451221 + }, + { + "accuracy": 0.465686, + "f1": 0.469799, + "f1_weighted": 0.49396 + } + ], + "main_score": 0.486765, + "hf_subset": "war_Latn", + "languages": [ + "war-Latn" + ] + }, + { + "accuracy": 0.348039, + "f1": 0.331208, + "f1_weighted": 0.349643, + "scores_per_experiment": [ + { + "accuracy": 0.382353, + "f1": 0.35636, + "f1_weighted": 0.389405 + }, + { + "accuracy": 0.294118, + "f1": 0.287425, + "f1_weighted": 0.295936 + }, + { + "accuracy": 0.318627, + "f1": 0.286573, + "f1_weighted": 0.313199 + }, + { + "accuracy": 0.323529, + "f1": 0.310727, + "f1_weighted": 0.337555 + }, + { + "accuracy": 0.348039, + "f1": 0.326092, + "f1_weighted": 0.344991 + }, + { + "accuracy": 0.392157, + "f1": 0.382704, + "f1_weighted": 0.38806 + }, + { + "accuracy": 0.338235, + "f1": 0.331935, + "f1_weighted": 0.340506 + }, + { + "accuracy": 0.338235, + "f1": 0.328922, + "f1_weighted": 0.342166 + }, + { + "accuracy": 0.348039, + "f1": 0.333418, + "f1_weighted": 0.345522 + }, + { + "accuracy": 0.397059, + "f1": 0.367926, + "f1_weighted": 0.399092 + } + ], + "main_score": 0.348039, + "hf_subset": "wol_Latn", + "languages": [ + "wol-Latn" + ] + }, + { + "accuracy": 0.298529, + "f1": 0.290021, + "f1_weighted": 0.300128, + "scores_per_experiment": [ + { + "accuracy": 0.318627, + "f1": 0.30641, + "f1_weighted": 0.322336 + }, + { + "accuracy": 0.240196, + "f1": 0.2385, + "f1_weighted": 0.243462 + }, + { + "accuracy": 0.303922, + "f1": 0.296447, + "f1_weighted": 0.299707 + }, + { + "accuracy": 0.308824, + "f1": 0.294168, + "f1_weighted": 0.300323 + }, + { + "accuracy": 0.308824, + "f1": 0.291763, + "f1_weighted": 0.307632 + }, + { + "accuracy": 0.294118, + "f1": 0.281024, + "f1_weighted": 0.305161 + }, + { + "accuracy": 0.29902, + "f1": 0.292031, + "f1_weighted": 0.294346 + }, + { + "accuracy": 0.328431, + "f1": 0.317447, + "f1_weighted": 0.330158 + }, + { + "accuracy": 0.289216, + "f1": 0.28926, + "f1_weighted": 0.289994 + }, + { + "accuracy": 0.294118, + "f1": 0.293162, + "f1_weighted": 0.308166 + } + ], + "main_score": 0.298529, + "hf_subset": "xho_Latn", + "languages": [ + "xho-Latn" + ] + }, + { + "accuracy": 0.184804, + "f1": 0.144747, + "f1_weighted": 0.156508, + "scores_per_experiment": [ + { + "accuracy": 0.20098, + "f1": 0.127276, + "f1_weighted": 0.157386 + }, + { + "accuracy": 0.196078, + "f1": 0.138765, + "f1_weighted": 0.154935 + }, + { + "accuracy": 0.142157, + "f1": 0.120598, + "f1_weighted": 0.118607 + }, + { + "accuracy": 0.235294, + "f1": 0.202752, + "f1_weighted": 0.213828 + }, + { + "accuracy": 0.156863, + "f1": 0.134835, + "f1_weighted": 0.126675 + }, + { + "accuracy": 0.196078, + "f1": 0.184705, + "f1_weighted": 0.169719 + }, + { + "accuracy": 0.186275, + "f1": 0.162869, + "f1_weighted": 0.175882 + }, + { + "accuracy": 0.240196, + "f1": 0.157046, + "f1_weighted": 0.198144 + }, + { + "accuracy": 0.151961, + "f1": 0.122955, + "f1_weighted": 0.145555 + }, + { + "accuracy": 0.142157, + "f1": 0.095667, + "f1_weighted": 0.104353 + } + ], + "main_score": 0.184804, + "hf_subset": "ydd_Hebr", + "languages": [ + "ydd-Hebr" + ] + }, + { + "accuracy": 0.280392, + "f1": 0.272641, + "f1_weighted": 0.281961, + "scores_per_experiment": [ + { + "accuracy": 0.264706, + "f1": 0.266198, + "f1_weighted": 0.277158 + }, + { + "accuracy": 0.318627, + "f1": 0.303278, + "f1_weighted": 0.323431 + }, + { + "accuracy": 0.27451, + "f1": 0.275696, + "f1_weighted": 0.282269 + }, + { + "accuracy": 0.323529, + "f1": 0.3007, + "f1_weighted": 0.319869 + }, + { + "accuracy": 0.303922, + "f1": 0.287056, + "f1_weighted": 0.281292 + }, + { + "accuracy": 0.27451, + "f1": 0.256385, + "f1_weighted": 0.275805 + }, + { + "accuracy": 0.303922, + "f1": 0.28405, + "f1_weighted": 0.309986 + }, + { + "accuracy": 0.279412, + "f1": 0.269645, + "f1_weighted": 0.274827 + }, + { + "accuracy": 0.245098, + "f1": 0.255307, + "f1_weighted": 0.250084 + }, + { + "accuracy": 0.215686, + "f1": 0.22809, + "f1_weighted": 0.224891 + } + ], + "main_score": 0.280392, + "hf_subset": "yor_Latn", + "languages": [ + "yor-Latn" + ] + }, + { + "accuracy": 0.156373, + "f1": 0.108989, + "f1_weighted": 0.105954, + "scores_per_experiment": [ + { + "accuracy": 0.210784, + "f1": 0.10287, + "f1_weighted": 0.119182 + }, + { + "accuracy": 0.166667, + "f1": 0.098694, + "f1_weighted": 0.094897 + }, + { + "accuracy": 0.142157, + "f1": 0.114729, + "f1_weighted": 0.102576 + }, + { + "accuracy": 0.181373, + "f1": 0.150659, + "f1_weighted": 0.145963 + }, + { + "accuracy": 0.127451, + "f1": 0.087341, + "f1_weighted": 0.065396 + }, + { + "accuracy": 0.137255, + "f1": 0.129721, + "f1_weighted": 0.101877 + }, + { + "accuracy": 0.191176, + "f1": 0.141821, + "f1_weighted": 0.129084 + }, + { + "accuracy": 0.20098, + "f1": 0.104819, + "f1_weighted": 0.133831 + }, + { + "accuracy": 0.107843, + "f1": 0.077501, + "f1_weighted": 0.085438 + }, + { + "accuracy": 0.098039, + "f1": 0.081739, + "f1_weighted": 0.081295 + } + ], + "main_score": 0.156373, + "hf_subset": "yue_Hant", + "languages": [ + "yue-Hant" + ] + }, + { + "accuracy": 0.191667, + "f1": 0.126422, + "f1_weighted": 0.13662, + "scores_per_experiment": [ + { + "accuracy": 0.215686, + "f1": 0.134385, + "f1_weighted": 0.153113 + }, + { + "accuracy": 0.176471, + "f1": 0.114156, + "f1_weighted": 0.107993 + }, + { + "accuracy": 0.171569, + "f1": 0.123573, + "f1_weighted": 0.139918 + }, + { + "accuracy": 0.196078, + "f1": 0.151106, + "f1_weighted": 0.171244 + }, + { + "accuracy": 0.22549, + "f1": 0.14135, + "f1_weighted": 0.150891 + }, + { + "accuracy": 0.210784, + "f1": 0.130025, + "f1_weighted": 0.136799 + }, + { + "accuracy": 0.230392, + "f1": 0.137192, + "f1_weighted": 0.167037 + }, + { + "accuracy": 0.166667, + "f1": 0.08117, + "f1_weighted": 0.074609 + }, + { + "accuracy": 0.171569, + "f1": 0.152725, + "f1_weighted": 0.163296 + }, + { + "accuracy": 0.151961, + "f1": 0.098536, + "f1_weighted": 0.101298 + } + ], + "main_score": 0.191667, + "hf_subset": "zho_Hant", + "languages": [ + "zho-Hant" + ] + }, + { + "accuracy": 0.394608, + "f1": 0.375423, + "f1_weighted": 0.400177, + "scores_per_experiment": [ + { + "accuracy": 0.426471, + "f1": 0.419921, + "f1_weighted": 0.429303 + }, + { + "accuracy": 0.377451, + "f1": 0.357652, + "f1_weighted": 0.376234 + }, + { + "accuracy": 0.392157, + "f1": 0.366545, + "f1_weighted": 0.398176 + }, + { + "accuracy": 0.426471, + "f1": 0.395987, + "f1_weighted": 0.433093 + }, + { + "accuracy": 0.401961, + "f1": 0.391115, + "f1_weighted": 0.412743 + }, + { + "accuracy": 0.372549, + "f1": 0.360015, + "f1_weighted": 0.379962 + }, + { + "accuracy": 0.382353, + "f1": 0.36478, + "f1_weighted": 0.385561 + }, + { + "accuracy": 0.416667, + "f1": 0.39783, + "f1_weighted": 0.428543 + }, + { + "accuracy": 0.377451, + "f1": 0.334717, + "f1_weighted": 0.378116 + }, + { + "accuracy": 0.372549, + "f1": 0.365668, + "f1_weighted": 0.380042 + } + ], + "main_score": 0.394608, + "hf_subset": "zsm_Latn", + "languages": [ + "zsm-Latn" + ] + }, + { + "accuracy": 0.260784, + "f1": 0.248714, + "f1_weighted": 0.262008, + "scores_per_experiment": [ + { + "accuracy": 0.269608, + "f1": 0.252456, + "f1_weighted": 0.270946 + }, + { + "accuracy": 0.27451, + "f1": 0.25485, + "f1_weighted": 0.275128 + }, + { + "accuracy": 0.210784, + "f1": 0.201251, + "f1_weighted": 0.206476 + }, + { + "accuracy": 0.279412, + "f1": 0.278281, + "f1_weighted": 0.27614 + }, + { + "accuracy": 0.318627, + "f1": 0.298613, + "f1_weighted": 0.31665 + }, + { + "accuracy": 0.313725, + "f1": 0.307034, + "f1_weighted": 0.319698 + }, + { + "accuracy": 0.254902, + "f1": 0.232408, + "f1_weighted": 0.252269 + }, + { + "accuracy": 0.22549, + "f1": 0.209649, + "f1_weighted": 0.222984 + }, + { + "accuracy": 0.215686, + "f1": 0.209575, + "f1_weighted": 0.225338 + }, + { + "accuracy": 0.245098, + "f1": 0.243022, + "f1_weighted": 0.254453 + } + ], + "main_score": 0.260784, + "hf_subset": "zul_Latn", + "languages": [ + "zul-Latn" + ] + } + ] + }, + "evaluation_time": 1055.3851170539856, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SICK-R.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SICK-R.json deleted file mode 100644 index 183df72ca6..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/SICK-R.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "mteb_version": "0.0.2", - "test": { - "cos_sim": { - "pearson": 0.7821761612002267, - "spearman": 0.7144730243032524 - }, - "euclidean": { - "pearson": 0.7514583639970599, - "spearman": 0.7144730604433791 - }, - "evaluation_time": 8.86, - "manhattan": { - "pearson": 0.7511305146860671, - "spearman": 0.714628691909426 - } - }, - "mteb_dataset_name": "SICK-R", - "dataset_revision": "20a6d6f312dd54037fe07a32d58e5e168867909d" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SIDClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SIDClassification.json new file mode 100644 index 0000000000..b8f7eb0d6e --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SIDClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "29bed651bb980395f5aa473607154d93226945e1", + "task_name": "SIDClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.28581, + "f1": 0.238913, + "f1_weighted": 0.263295, + "scores_per_experiment": [ + { + "accuracy": 0.300402, + "f1": 0.244098, + "f1_weighted": 0.282863 + }, + { + "accuracy": 0.315663, + "f1": 0.248433, + "f1_weighted": 0.317056 + }, + { + "accuracy": 0.280054, + "f1": 0.241888, + "f1_weighted": 0.253453 + }, + { + "accuracy": 0.265596, + "f1": 0.22791, + "f1_weighted": 0.23085 + }, + { + "accuracy": 0.234806, + "f1": 0.200713, + "f1_weighted": 0.199408 + }, + { + "accuracy": 0.276841, + "f1": 0.230799, + "f1_weighted": 0.249121 + }, + { + "accuracy": 0.278715, + "f1": 0.241549, + "f1_weighted": 0.248708 + }, + { + "accuracy": 0.303079, + "f1": 0.261727, + "f1_weighted": 0.283827 + }, + { + "accuracy": 0.293976, + "f1": 0.236766, + "f1_weighted": 0.275469 + }, + { + "accuracy": 0.308969, + "f1": 0.255248, + "f1_weighted": 0.292198 + } + ], + "main_score": 0.28581, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 64.65900897979736, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/STS12.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/STS12.json deleted file mode 100644 index 611856d81b..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/STS12.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "test": { - "cos_sim": { - "pearson": 0.7425145515932046, - "spearman": 0.6859358272758398 - }, - "euclidean": { - "pearson": 0.7046414956702067, - "spearman": 0.6859384294933573 - }, - "evaluation_time": 5.7, - "manhattan": { - "pearson": 0.7042527620830559, - "spearman": 0.6855305407716232 - } - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "STS12", - "dataset_revision": "fdf84275bb8ce4b49c971d02e84dd1abc677a50f" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/STS13.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/STS13.json deleted file mode 100644 index cb17015810..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/STS13.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "test": { - "cos_sim": { - "pearson": 0.7896003609986886, - "spearman": 0.7909019447848391 - }, - "euclidean": { - "pearson": 0.7851948939193585, - "spearman": 0.7909019440818148 - }, - "evaluation_time": 3.87, - "manhattan": { - "pearson": 0.7848990152663227, - "spearman": 0.790271568020597 - } - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "STS13", - "dataset_revision": "1591bfcbe8c69d4bf7fe2a16e2451017832cafb9" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/STS14.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/STS14.json deleted file mode 100644 index d37f621556..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/STS14.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "test": { - "cos_sim": { - "pearson": 0.7924882828122645, - "spearman": 0.7464141703710193 - }, - "euclidean": { - "pearson": 0.7727493703044144, - "spearman": 0.7464141596597346 - }, - "evaluation_time": 6.02, - "manhattan": { - "pearson": 0.7723350410987001, - "spearman": 0.7459634717090692 - } - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "STS14", - "dataset_revision": "e2125984e7df8b7871f6ae9949cf6b6795e7c54b" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/STS15.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/STS15.json deleted file mode 100644 index e6c78b8980..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/STS15.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "test": { - "cos_sim": { - "pearson": 0.840472804722847, - "spearman": 0.8484757279716068 - }, - "euclidean": { - "pearson": 0.8422689224021953, - "spearman": 0.8484754250954344 - }, - "evaluation_time": 5.35, - "manhattan": { - "pearson": 0.841849715909246, - "spearman": 0.8479971271119682 - } - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "STS15", - "dataset_revision": "1cd7298cac12a96a373b6a2f18738bb3e739a9b6" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/STS16.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/STS16.json deleted file mode 100644 index 62a5b2e3db..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/STS16.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "test": { - "cos_sim": { - "pearson": 0.8097346210014907, - "spearman": 0.8156700264541153 - }, - "euclidean": { - "pearson": 0.8059803734160776, - "spearman": 0.8156700264541153 - }, - "evaluation_time": 3.67, - "manhattan": { - "pearson": 0.8057381807785478, - "spearman": 0.8151356810657332 - } - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "STS16", - "dataset_revision": "360a0b2dff98700d09e634a01e1cc1624d3e42cd" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/STS17.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/STS17.json deleted file mode 100644 index 8ecac50a45..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/STS17.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "test": { - "en-en": { - "cos_sim": { - "pearson": 0.8443962074313307, - "spearman": 0.8579615701004483 - }, - "euclidean": { - "pearson": 0.8467151237057283, - "spearman": 0.8579615701004483 - }, - "manhattan": { - "pearson": 0.847186766598335, - "spearman": 0.8598262777697079 - } - }, - "evaluation_time": 2.49 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "STS17", - "dataset_revision": "9fc37e8c632af1c87a3d23e685d49552a02582a0" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/STS22.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/STS22.json deleted file mode 100644 index a6bde94d47..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/STS22.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "test": { - "en": { - "cos_sim": { - "pearson": 0.6520695114759218, - "spearman": 0.6616950180914625 - }, - "euclidean": { - "pearson": 0.6753540863448494, - "spearman": 0.6616950180914625 - }, - "manhattan": { - "pearson": 0.67790826279066, - "spearman": 0.6646849937199816 - } - }, - "evaluation_time": 6.94 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "STS22", - "dataset_revision": "2de6ce8c1921b71a755b262c6b57fef195dd7906" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/STSBenchmark.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/STSBenchmark.json deleted file mode 100644 index 6a07e5fd7e..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/STSBenchmark.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "test": { - "cos_sim": { - "pearson": 0.8064062065343579, - "spearman": 0.7957572102026641 - }, - "euclidean": { - "pearson": 0.7972755759413925, - "spearman": 0.7957566675381056 - }, - "evaluation_time": 3.06, - "manhattan": { - "pearson": 0.7976168619616728, - "spearman": 0.7962373879220337 - } - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "STSBenchmark", - "dataset_revision": "8913289635987208e6e7c72789e4be2fe94b6abd" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SanskritShlokasClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SanskritShlokasClassification.json new file mode 100644 index 0000000000..5db42d86e7 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SanskritShlokasClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "5a79d6472db143690c7ce6e974995d3610eee7f0", + "task_name": "SanskritShlokasClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.643342, + "f1": 0.557431, + "f1_weighted": 0.558606, + "scores_per_experiment": [ + { + "accuracy": 0.697128, + "f1": 0.667435, + "f1_weighted": 0.674083 + }, + { + "accuracy": 0.657963, + "f1": 0.538311, + "f1_weighted": 0.554234 + }, + { + "accuracy": 0.655352, + "f1": 0.526787, + "f1_weighted": 0.543597 + }, + { + "accuracy": 0.655352, + "f1": 0.64438, + "f1_weighted": 0.643444 + }, + { + "accuracy": 0.67624, + "f1": 0.552704, + "f1_weighted": 0.569609 + }, + { + "accuracy": 0.616188, + "f1": 0.553651, + "f1_weighted": 0.540477 + }, + { + "accuracy": 0.660574, + "f1": 0.540071, + "f1_weighted": 0.556028 + }, + { + "accuracy": 0.597911, + "f1": 0.497855, + "f1_weighted": 0.481291 + }, + { + "accuracy": 0.618799, + "f1": 0.555261, + "f1_weighted": 0.542009 + }, + { + "accuracy": 0.597911, + "f1": 0.497855, + "f1_weighted": 0.481291 + } + ], + "main_score": 0.643342, + "hf_subset": "default", + "languages": [ + "san-Deva" + ] + } + ], + "validation": [ + { + "accuracy": 0.638542, + "f1": 0.566027, + "f1_weighted": 0.556295, + "scores_per_experiment": [ + { + "accuracy": 0.729167, + "f1": 0.698899, + "f1_weighted": 0.704774 + }, + { + "accuracy": 0.6875, + "f1": 0.580429, + "f1_weighted": 0.60058 + }, + { + "accuracy": 0.666667, + "f1": 0.529107, + "f1_weighted": 0.554675 + }, + { + "accuracy": 0.666667, + "f1": 0.661422, + "f1_weighted": 0.6553 + }, + { + "accuracy": 0.708333, + "f1": 0.588294, + "f1_weighted": 0.610038 + }, + { + "accuracy": 0.572917, + "f1": 0.539679, + "f1_weighted": 0.497401 + }, + { + "accuracy": 0.6875, + "f1": 0.58243, + "f1_weighted": 0.60161 + }, + { + "accuracy": 0.541667, + "f1": 0.466852, + "f1_weighted": 0.417513 + }, + { + "accuracy": 0.583333, + "f1": 0.546302, + "f1_weighted": 0.503544 + }, + { + "accuracy": 0.541667, + "f1": 0.466852, + "f1_weighted": 0.417513 + } + ], + "main_score": 0.638542, + "hf_subset": "default", + "languages": [ + "san-Deva" + ] + } + ] + }, + "evaluation_time": 1.1404359340667725, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ScalaClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ScalaClassification.json new file mode 100644 index 0000000000..70bd1d8cbd --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/ScalaClassification.json @@ -0,0 +1,347 @@ +{ + "dataset_revision": "ec85bb6c69679ed15ac66c0bf6e180bf563eb137", + "task_name": "ScalaClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.502539, + "f1": 0.500248, + "f1_weighted": 0.500248, + "ap": 0.5013, + "ap_weighted": 0.5013, + "scores_per_experiment": [ + { + "accuracy": 0.505371, + "f1": 0.494302, + "f1_weighted": 0.494302, + "ap": 0.502708, + "ap_weighted": 0.502708 + }, + { + "accuracy": 0.506836, + "f1": 0.505561, + "f1_weighted": 0.505561, + "ap": 0.50347, + "ap_weighted": 0.50347 + }, + { + "accuracy": 0.494141, + "f1": 0.493288, + "f1_weighted": 0.493288, + "ap": 0.497108, + "ap_weighted": 0.497108 + }, + { + "accuracy": 0.505371, + "f1": 0.505297, + "f1_weighted": 0.505297, + "ap": 0.502715, + "ap_weighted": 0.502715 + }, + { + "accuracy": 0.5, + "f1": 0.498606, + "f1_weighted": 0.498606, + "ap": 0.5, + "ap_weighted": 0.5 + }, + { + "accuracy": 0.496094, + "f1": 0.495793, + "f1_weighted": 0.495793, + "ap": 0.498061, + "ap_weighted": 0.498061 + }, + { + "accuracy": 0.504883, + "f1": 0.501842, + "f1_weighted": 0.501842, + "ap": 0.50247, + "ap_weighted": 0.50247 + }, + { + "accuracy": 0.51123, + "f1": 0.506666, + "f1_weighted": 0.506666, + "ap": 0.505721, + "ap_weighted": 0.505721 + }, + { + "accuracy": 0.503418, + "f1": 0.503404, + "f1_weighted": 0.503404, + "ap": 0.501721, + "ap_weighted": 0.501721 + }, + { + "accuracy": 0.498047, + "f1": 0.497723, + "f1_weighted": 0.497723, + "ap": 0.499027, + "ap_weighted": 0.499027 + } + ], + "main_score": 0.502539, + "hf_subset": "Danish", + "languages": [ + "dan-Latn" + ] + }, + { + "accuracy": 0.499072, + "f1": 0.497544, + "f1_weighted": 0.497544, + "ap": 0.49956, + "ap_weighted": 0.49956, + "scores_per_experiment": [ + { + "accuracy": 0.48877, + "f1": 0.488565, + "f1_weighted": 0.488565, + "ap": 0.494516, + "ap_weighted": 0.494516 + }, + { + "accuracy": 0.495605, + "f1": 0.494065, + "f1_weighted": 0.494065, + "ap": 0.497824, + "ap_weighted": 0.497824 + }, + { + "accuracy": 0.505371, + "f1": 0.501031, + "f1_weighted": 0.501031, + "ap": 0.50271, + "ap_weighted": 0.50271 + }, + { + "accuracy": 0.495605, + "f1": 0.492064, + "f1_weighted": 0.492064, + "ap": 0.497826, + "ap_weighted": 0.497826 + }, + { + "accuracy": 0.499023, + "f1": 0.498621, + "f1_weighted": 0.498621, + "ap": 0.499513, + "ap_weighted": 0.499513 + }, + { + "accuracy": 0.503906, + "f1": 0.503894, + "f1_weighted": 0.503894, + "ap": 0.501968, + "ap_weighted": 0.501968 + }, + { + "accuracy": 0.500977, + "f1": 0.498428, + "f1_weighted": 0.498428, + "ap": 0.500489, + "ap_weighted": 0.500489 + }, + { + "accuracy": 0.496582, + "f1": 0.494514, + "f1_weighted": 0.494514, + "ap": 0.498304, + "ap_weighted": 0.498304 + }, + { + "accuracy": 0.501953, + "f1": 0.501466, + "f1_weighted": 0.501466, + "ap": 0.500981, + "ap_weighted": 0.500981 + }, + { + "accuracy": 0.50293, + "f1": 0.502793, + "f1_weighted": 0.502793, + "ap": 0.501473, + "ap_weighted": 0.501473 + } + ], + "main_score": 0.499072, + "hf_subset": "Norwegian_b", + "languages": [ + "nob-Latn" + ] + }, + { + "accuracy": 0.49873, + "f1": 0.496752, + "f1_weighted": 0.496752, + "ap": 0.499396, + "ap_weighted": 0.499396, + "scores_per_experiment": [ + { + "accuracy": 0.500977, + "f1": 0.493305, + "f1_weighted": 0.493305, + "ap": 0.500489, + "ap_weighted": 0.500489 + }, + { + "accuracy": 0.488281, + "f1": 0.488086, + "f1_weighted": 0.488086, + "ap": 0.494273, + "ap_weighted": 0.494273 + }, + { + "accuracy": 0.504395, + "f1": 0.50436, + "f1_weighted": 0.50436, + "ap": 0.502217, + "ap_weighted": 0.502217 + }, + { + "accuracy": 0.494629, + "f1": 0.494609, + "f1_weighted": 0.494609, + "ap": 0.497344, + "ap_weighted": 0.497344 + }, + { + "accuracy": 0.497559, + "f1": 0.494512, + "f1_weighted": 0.494512, + "ap": 0.498784, + "ap_weighted": 0.498784 + }, + { + "accuracy": 0.493652, + "f1": 0.487466, + "f1_weighted": 0.487466, + "ap": 0.496859, + "ap_weighted": 0.496859 + }, + { + "accuracy": 0.507324, + "f1": 0.507323, + "f1_weighted": 0.507323, + "ap": 0.503716, + "ap_weighted": 0.503716 + }, + { + "accuracy": 0.504395, + "f1": 0.502034, + "f1_weighted": 0.502034, + "ap": 0.50222, + "ap_weighted": 0.50222 + }, + { + "accuracy": 0.498535, + "f1": 0.498314, + "f1_weighted": 0.498314, + "ap": 0.49927, + "ap_weighted": 0.49927 + }, + { + "accuracy": 0.497559, + "f1": 0.497515, + "f1_weighted": 0.497515, + "ap": 0.498785, + "ap_weighted": 0.498785 + } + ], + "main_score": 0.49873, + "hf_subset": "Norwegian_n", + "languages": [ + "nno-Latn" + ] + }, + { + "accuracy": 0.498291, + "f1": 0.496222, + "f1_weighted": 0.496222, + "ap": 0.499167, + "ap_weighted": 0.499167, + "scores_per_experiment": [ + { + "accuracy": 0.504395, + "f1": 0.501757, + "f1_weighted": 0.501757, + "ap": 0.50222, + "ap_weighted": 0.50222 + }, + { + "accuracy": 0.494141, + "f1": 0.493789, + "f1_weighted": 0.493789, + "ap": 0.497107, + "ap_weighted": 0.497107 + }, + { + "accuracy": 0.499512, + "f1": 0.498037, + "f1_weighted": 0.498037, + "ap": 0.499756, + "ap_weighted": 0.499756 + }, + { + "accuracy": 0.496582, + "f1": 0.496562, + "f1_weighted": 0.496562, + "ap": 0.498303, + "ap_weighted": 0.498303 + }, + { + "accuracy": 0.50293, + "f1": 0.49624, + "f1_weighted": 0.49624, + "ap": 0.501476, + "ap_weighted": 0.501476 + }, + { + "accuracy": 0.494629, + "f1": 0.494406, + "f1_weighted": 0.494406, + "ap": 0.497342, + "ap_weighted": 0.497342 + }, + { + "accuracy": 0.498535, + "f1": 0.497862, + "f1_weighted": 0.497862, + "ap": 0.49927, + "ap_weighted": 0.49927 + }, + { + "accuracy": 0.495605, + "f1": 0.494119, + "f1_weighted": 0.494119, + "ap": 0.497824, + "ap_weighted": 0.497824 + }, + { + "accuracy": 0.492188, + "f1": 0.492013, + "f1_weighted": 0.492013, + "ap": 0.496153, + "ap_weighted": 0.496153 + }, + { + "accuracy": 0.504395, + "f1": 0.497435, + "f1_weighted": 0.497435, + "ap": 0.502223, + "ap_weighted": 0.502223 + } + ], + "main_score": 0.498291, + "hf_subset": "Swedish", + "languages": [ + "swe-Latn" + ] + } + ] + }, + "evaluation_time": 14.531367301940918, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SciDocsRR.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SciDocsRR.json deleted file mode 100644 index ebf070aaf9..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/SciDocsRR.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "test": { - "evaluation_time": 69.28, - "map": 0.737089144684744, - "mrr": 0.9153676570343237 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "SciDocsRR", - "dataset_revision": "56a6d0140cf6356659e2a7c1413286a774468d44" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SciFact.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SciFact.json deleted file mode 100644 index 86f4036a35..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/SciFact.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "test": { - "evaluation_time": 48.42, - "map_at_1": 0.4565, - "map_at_10": 0.55108, - "map_at_100": 0.55854, - "map_at_1000": 0.55902, - "map_at_3": 0.52906, - "map_at_5": 0.53867, - "ndcg_at_1": 0.48333, - "ndcg_at_10": 0.59738, - "ndcg_at_100": 0.62896, - "ndcg_at_1000": 0.64163, - "ndcg_at_3": 0.55624, - "ndcg_at_5": 0.57061, - "precision_at_1": 0.48333, - "precision_at_10": 0.081, - "precision_at_100": 0.00983, - "precision_at_1000": 0.00109, - "precision_at_3": 0.22, - "precision_at_5": 0.14133, - "recall_at_1": 0.4565, - "recall_at_10": 0.72428, - "recall_at_100": 0.86367, - "recall_at_1000": 0.96333, - "recall_at_3": 0.60728, - "recall_at_5": 0.64683 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "SciFact", - "dataset_revision": "a75ae049398addde9b70f6b268875f5cbce99089" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SentimentAnalysisHindi.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SentimentAnalysisHindi.json new file mode 100644 index 0000000000..1ec2dbc0fc --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SentimentAnalysisHindi.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "1beac1b941da76a9c51e3e5b39d230fde9a80983", + "task_name": "SentimentAnalysisHindi", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.369434, + "f1": 0.343406, + "f1_weighted": 0.359102, + "scores_per_experiment": [ + { + "accuracy": 0.442871, + "f1": 0.426606, + "f1_weighted": 0.465737 + }, + { + "accuracy": 0.246094, + "f1": 0.230952, + "f1_weighted": 0.215156 + }, + { + "accuracy": 0.329102, + "f1": 0.307974, + "f1_weighted": 0.297478 + }, + { + "accuracy": 0.327148, + "f1": 0.296697, + "f1_weighted": 0.280228 + }, + { + "accuracy": 0.325684, + "f1": 0.303473, + "f1_weighted": 0.291848 + }, + { + "accuracy": 0.341309, + "f1": 0.326564, + "f1_weighted": 0.324242 + }, + { + "accuracy": 0.494141, + "f1": 0.427567, + "f1_weighted": 0.497683 + }, + { + "accuracy": 0.48584, + "f1": 0.431031, + "f1_weighted": 0.49439 + }, + { + "accuracy": 0.271484, + "f1": 0.272577, + "f1_weighted": 0.270804 + }, + { + "accuracy": 0.430664, + "f1": 0.410621, + "f1_weighted": 0.453455 + } + ], + "main_score": 0.343406, + "hf_subset": "default", + "languages": [ + "hin-Deva" + ] + } + ] + }, + "evaluation_time": 2.8130462169647217, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SentimentDKSF.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SentimentDKSF.json new file mode 100644 index 0000000000..35aac5553e --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SentimentDKSF.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "b4d5a8dd501db610b5ad89e9aa13f863b842b395", + "task_name": "SentimentDKSF", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.442549, + "f1": 0.379355, + "f1_weighted": 0.444911, + "scores_per_experiment": [ + { + "accuracy": 0.363283, + "f1": 0.316563, + "f1_weighted": 0.401234 + }, + { + "accuracy": 0.493305, + "f1": 0.43636, + "f1_weighted": 0.49689 + }, + { + "accuracy": 0.45054, + "f1": 0.401588, + "f1_weighted": 0.467037 + }, + { + "accuracy": 0.408207, + "f1": 0.34195, + "f1_weighted": 0.394199 + }, + { + "accuracy": 0.451404, + "f1": 0.362455, + "f1_weighted": 0.4347 + }, + { + "accuracy": 0.493305, + "f1": 0.428808, + "f1_weighted": 0.488567 + }, + { + "accuracy": 0.420302, + "f1": 0.337952, + "f1_weighted": 0.420437 + }, + { + "accuracy": 0.485961, + "f1": 0.426078, + "f1_weighted": 0.480768 + }, + { + "accuracy": 0.475162, + "f1": 0.402685, + "f1_weighted": 0.480705 + }, + { + "accuracy": 0.384017, + "f1": 0.339116, + "f1_weighted": 0.384574 + } + ], + "main_score": 0.442549, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 6.319506883621216, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SinhalaNewsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SinhalaNewsClassification.json new file mode 100644 index 0000000000..a671f5af20 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SinhalaNewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "7fb2f514ea683c5282dfec0a9672ece8de90ac50", + "task_name": "SinhalaNewsClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.362695, + "f1": 0.349851, + "f1_weighted": 0.34884, + "scores_per_experiment": [ + { + "accuracy": 0.396973, + "f1": 0.381279, + "f1_weighted": 0.378368 + }, + { + "accuracy": 0.382324, + "f1": 0.365069, + "f1_weighted": 0.371667 + }, + { + "accuracy": 0.353516, + "f1": 0.363922, + "f1_weighted": 0.351203 + }, + { + "accuracy": 0.353516, + "f1": 0.326416, + "f1_weighted": 0.328821 + }, + { + "accuracy": 0.305176, + "f1": 0.302436, + "f1_weighted": 0.278853 + }, + { + "accuracy": 0.287109, + "f1": 0.266385, + "f1_weighted": 0.258506 + }, + { + "accuracy": 0.36377, + "f1": 0.369368, + "f1_weighted": 0.356724 + }, + { + "accuracy": 0.376953, + "f1": 0.362875, + "f1_weighted": 0.375545 + }, + { + "accuracy": 0.425781, + "f1": 0.370899, + "f1_weighted": 0.395578 + }, + { + "accuracy": 0.381836, + "f1": 0.389865, + "f1_weighted": 0.393129 + } + ], + "main_score": 0.362695, + "hf_subset": "default", + "languages": [ + "sin-Sinh" + ] + } + ] + }, + "evaluation_time": 4.544425964355469, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SinhalaNewsSourceClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SinhalaNewsSourceClassification.json new file mode 100644 index 0000000000..094c797832 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SinhalaNewsSourceClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "ac4d14eeb68efbef95e247542d4432ce674faeb1", + "task_name": "SinhalaNewsSourceClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.190186, + "f1": 0.159134, + "f1_weighted": 0.160341, + "scores_per_experiment": [ + { + "accuracy": 0.187988, + "f1": 0.168491, + "f1_weighted": 0.165177 + }, + { + "accuracy": 0.174316, + "f1": 0.133388, + "f1_weighted": 0.130605 + }, + { + "accuracy": 0.190918, + "f1": 0.184639, + "f1_weighted": 0.181051 + }, + { + "accuracy": 0.163574, + "f1": 0.130558, + "f1_weighted": 0.130252 + }, + { + "accuracy": 0.190918, + "f1": 0.154197, + "f1_weighted": 0.157154 + }, + { + "accuracy": 0.182129, + "f1": 0.134938, + "f1_weighted": 0.137125 + }, + { + "accuracy": 0.198242, + "f1": 0.172561, + "f1_weighted": 0.172383 + }, + { + "accuracy": 0.205566, + "f1": 0.167381, + "f1_weighted": 0.171247 + }, + { + "accuracy": 0.214844, + "f1": 0.177311, + "f1_weighted": 0.185492 + }, + { + "accuracy": 0.193359, + "f1": 0.167871, + "f1_weighted": 0.172923 + } + ], + "main_score": 0.190186, + "hf_subset": "default", + "languages": [ + "sin-Sinh" + ] + } + ] + }, + "evaluation_time": 2.733680009841919, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SiswatiNewsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SiswatiNewsClassification.json new file mode 100644 index 0000000000..d0ec68bbf9 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SiswatiNewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f5502326c4e48adc99b18b1582f68b8fb5e7ec30", + "task_name": "SiswatiNewsClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.49125, + "f1": 0.199223, + "f1_weighted": 0.425819, + "scores_per_experiment": [ + { + "accuracy": 0.5375, + "f1": 0.221825, + "f1_weighted": 0.474327 + }, + { + "accuracy": 0.4875, + "f1": 0.186781, + "f1_weighted": 0.412072 + }, + { + "accuracy": 0.5, + "f1": 0.194771, + "f1_weighted": 0.425319 + }, + { + "accuracy": 0.4875, + "f1": 0.19369, + "f1_weighted": 0.415008 + }, + { + "accuracy": 0.4875, + "f1": 0.20129, + "f1_weighted": 0.422708 + }, + { + "accuracy": 0.4625, + "f1": 0.192707, + "f1_weighted": 0.400358 + }, + { + "accuracy": 0.525, + "f1": 0.21417, + "f1_weighted": 0.468985 + }, + { + "accuracy": 0.4625, + "f1": 0.196096, + "f1_weighted": 0.400056 + }, + { + "accuracy": 0.4625, + "f1": 0.184162, + "f1_weighted": 0.403865 + }, + { + "accuracy": 0.5, + "f1": 0.206741, + "f1_weighted": 0.435495 + } + ], + "main_score": 0.49125, + "hf_subset": "default", + "languages": [ + "ssw-Latn" + ] + } + ] + }, + "evaluation_time": 2.722259998321533, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SlovakHateSpeechClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SlovakHateSpeechClassification.json new file mode 100644 index 0000000000..905c9e1c1d --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SlovakHateSpeechClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "f9301b9937128c9c0b636fa6da203aeb046479f4", + "task_name": "SlovakHateSpeechClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.495906, + "f1": 0.476979, + "f1_weighted": 0.513276, + "ap": 0.28897, + "ap_weighted": 0.28897, + "scores_per_experiment": [ + { + "accuracy": 0.482183, + "f1": 0.479652, + "f1_weighted": 0.496134, + "ap": 0.303754, + "ap_weighted": 0.303754 + }, + { + "accuracy": 0.473844, + "f1": 0.471437, + "f1_weighted": 0.487634, + "ap": 0.299073, + "ap_weighted": 0.299073 + }, + { + "accuracy": 0.426839, + "f1": 0.420402, + "f1_weighted": 0.44814, + "ap": 0.264456, + "ap_weighted": 0.264456 + }, + { + "accuracy": 0.512509, + "f1": 0.482022, + "f1_weighted": 0.539091, + "ap": 0.275716, + "ap_weighted": 0.275716 + }, + { + "accuracy": 0.590599, + "f1": 0.508529, + "f1_weighted": 0.599735, + "ap": 0.277194, + "ap_weighted": 0.277194 + }, + { + "accuracy": 0.529947, + "f1": 0.511992, + "f1_weighted": 0.554502, + "ap": 0.299527, + "ap_weighted": 0.299527 + }, + { + "accuracy": 0.468537, + "f1": 0.466948, + "f1_weighted": 0.480163, + "ap": 0.299358, + "ap_weighted": 0.299358 + }, + { + "accuracy": 0.433662, + "f1": 0.433629, + "f1_weighted": 0.435579, + "ap": 0.291017, + "ap_weighted": 0.291017 + }, + { + "accuracy": 0.502654, + "f1": 0.472976, + "f1_weighted": 0.529771, + "ap": 0.271944, + "ap_weighted": 0.271944 + }, + { + "accuracy": 0.538287, + "f1": 0.522203, + "f1_weighted": 0.562013, + "ap": 0.307661, + "ap_weighted": 0.307661 + } + ], + "main_score": 0.495906, + "hf_subset": "default", + "languages": [ + "slk-Latn" + ] + } + ] + }, + "evaluation_time": 2.8134305477142334, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SlovakMovieReviewSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SlovakMovieReviewSentimentClassification.json new file mode 100644 index 0000000000..b2a7ace0f9 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SlovakMovieReviewSentimentClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "0c47583c9d339b3b6f89e4db76088af5f1ec8d39", + "task_name": "SlovakMovieReviewSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.555176, + "f1": 0.547196, + "f1_weighted": 0.557748, + "ap": 0.434452, + "ap_weighted": 0.434452, + "scores_per_experiment": [ + { + "accuracy": 0.514648, + "f1": 0.505803, + "f1_weighted": 0.518393, + "ap": 0.408508, + "ap_weighted": 0.408508 + }, + { + "accuracy": 0.496094, + "f1": 0.476522, + "f1_weighted": 0.495797, + "ap": 0.394272, + "ap_weighted": 0.394272 + }, + { + "accuracy": 0.587891, + "f1": 0.571205, + "f1_weighted": 0.587312, + "ap": 0.446287, + "ap_weighted": 0.446287 + }, + { + "accuracy": 0.582031, + "f1": 0.579654, + "f1_weighted": 0.585674, + "ap": 0.456459, + "ap_weighted": 0.456459 + }, + { + "accuracy": 0.555664, + "f1": 0.553976, + "f1_weighted": 0.559201, + "ap": 0.440285, + "ap_weighted": 0.440285 + }, + { + "accuracy": 0.576172, + "f1": 0.566364, + "f1_weighted": 0.578783, + "ap": 0.443604, + "ap_weighted": 0.443604 + }, + { + "accuracy": 0.54834, + "f1": 0.536965, + "f1_weighted": 0.550785, + "ap": 0.424986, + "ap_weighted": 0.424986 + }, + { + "accuracy": 0.56543, + "f1": 0.56227, + "f1_weighted": 0.569352, + "ap": 0.443972, + "ap_weighted": 0.443972 + }, + { + "accuracy": 0.553711, + "f1": 0.552847, + "f1_weighted": 0.556589, + "ap": 0.441036, + "ap_weighted": 0.441036 + }, + { + "accuracy": 0.571777, + "f1": 0.566356, + "f1_weighted": 0.575589, + "ap": 0.445108, + "ap_weighted": 0.445108 + } + ], + "main_score": 0.555176, + "hf_subset": "default", + "languages": [ + "svk-Latn" + ] + } + ] + }, + "evaluation_time": 9.742007970809937, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SouthAfricanLangClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SouthAfricanLangClassification.json new file mode 100644 index 0000000000..d316339dc1 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SouthAfricanLangClassification.json @@ -0,0 +1,83 @@ +{ + "dataset_revision": "5ccda92ffd7e74fa91fed595a1cbcff1bb68ec2d", + "task_name": "SouthAfricanLangClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.780078, + "f1": 0.775963, + "f1_weighted": 0.77607, + "scores_per_experiment": [ + { + "accuracy": 0.760742, + "f1": 0.755222, + "f1_weighted": 0.755333 + }, + { + "accuracy": 0.785645, + "f1": 0.784294, + "f1_weighted": 0.784397 + }, + { + "accuracy": 0.789551, + "f1": 0.785944, + "f1_weighted": 0.78607 + }, + { + "accuracy": 0.76709, + "f1": 0.76495, + "f1_weighted": 0.765078 + }, + { + "accuracy": 0.75293, + "f1": 0.754049, + "f1_weighted": 0.754093 + }, + { + "accuracy": 0.765137, + "f1": 0.754936, + "f1_weighted": 0.75504 + }, + { + "accuracy": 0.778809, + "f1": 0.773785, + "f1_weighted": 0.773915 + }, + { + "accuracy": 0.80957, + "f1": 0.807898, + "f1_weighted": 0.807987 + }, + { + "accuracy": 0.799805, + "f1": 0.792878, + "f1_weighted": 0.792999 + }, + { + "accuracy": 0.791504, + "f1": 0.785678, + "f1_weighted": 0.785788 + } + ], + "main_score": 0.780078, + "hf_subset": "default", + "languages": [ + "afr-Latn", + "eng-Latn", + "nbl-Latn", + "nso-Latn", + "sot-Latn", + "ssw-Latn", + "tsn-Latn", + "tso-Latn", + "ven-Latn", + "xho-Latn", + "zul-Latn" + ] + } + ] + }, + "evaluation_time": 9.16186237335205, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SpanishNewsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SpanishNewsClassification.json new file mode 100644 index 0000000000..38a04f36b0 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SpanishNewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "0086c197b914690a9dace258a19398890a05299a", + "task_name": "SpanishNewsClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.80957, + "f1": 0.80873, + "f1_weighted": 0.808672, + "scores_per_experiment": [ + { + "accuracy": 0.788086, + "f1": 0.785468, + "f1_weighted": 0.785402 + }, + { + "accuracy": 0.811523, + "f1": 0.809783, + "f1_weighted": 0.809701 + }, + { + "accuracy": 0.811523, + "f1": 0.809876, + "f1_weighted": 0.809836 + }, + { + "accuracy": 0.814453, + "f1": 0.813973, + "f1_weighted": 0.813905 + }, + { + "accuracy": 0.817871, + "f1": 0.81725, + "f1_weighted": 0.817208 + }, + { + "accuracy": 0.811035, + "f1": 0.810073, + "f1_weighted": 0.81003 + }, + { + "accuracy": 0.808594, + "f1": 0.807671, + "f1_weighted": 0.807622 + }, + { + "accuracy": 0.818848, + "f1": 0.818151, + "f1_weighted": 0.818088 + }, + { + "accuracy": 0.803223, + "f1": 0.803761, + "f1_weighted": 0.803686 + }, + { + "accuracy": 0.810547, + "f1": 0.811295, + "f1_weighted": 0.811247 + } + ], + "main_score": 0.80957, + "hf_subset": "default", + "languages": [ + "spa-Latn" + ] + } + ] + }, + "evaluation_time": 39.09053635597229, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SpanishSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SpanishSentimentClassification.json new file mode 100644 index 0000000000..489067cb1a --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SpanishSentimentClassification.json @@ -0,0 +1,181 @@ +{ + "dataset_revision": "2a6e340e4b59b7c0a78c03a0b79ac27e1b4a2662", + "task_name": "SpanishSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "validation": [ + { + "accuracy": 0.668707, + "f1": 0.568906, + "f1_weighted": 0.699819, + "ap": 0.862836, + "ap_weighted": 0.862836, + "scores_per_experiment": [ + { + "accuracy": 0.646259, + "f1": 0.546296, + "f1_weighted": 0.683925, + "ap": 0.850256, + "ap_weighted": 0.850256 + }, + { + "accuracy": 0.544218, + "f1": 0.507772, + "f1_weighted": 0.594331, + "ap": 0.865755, + "ap_weighted": 0.865755 + }, + { + "accuracy": 0.714286, + "f1": 0.598256, + "f1_weighted": 0.737785, + "ap": 0.863473, + "ap_weighted": 0.863473 + }, + { + "accuracy": 0.62585, + "f1": 0.567485, + "f1_weighted": 0.670165, + "ap": 0.877047, + "ap_weighted": 0.877047 + }, + { + "accuracy": 0.680272, + "f1": 0.563633, + "f1_weighted": 0.709432, + "ap": 0.851941, + "ap_weighted": 0.851941 + }, + { + "accuracy": 0.659864, + "f1": 0.540165, + "f1_weighted": 0.691784, + "ap": 0.843283, + "ap_weighted": 0.843283 + }, + { + "accuracy": 0.693878, + "f1": 0.627639, + "f1_weighted": 0.729134, + "ap": 0.896786, + "ap_weighted": 0.896786 + }, + { + "accuracy": 0.70068, + "f1": 0.514706, + "f1_weighted": 0.708855, + "ap": 0.827884, + "ap_weighted": 0.827884 + }, + { + "accuracy": 0.666667, + "f1": 0.600033, + "f1_weighted": 0.705536, + "ap": 0.885464, + "ap_weighted": 0.885464 + }, + { + "accuracy": 0.755102, + "f1": 0.623077, + "f1_weighted": 0.767242, + "ap": 0.866474, + "ap_weighted": 0.866474 + } + ], + "main_score": 0.668707, + "hf_subset": "default", + "languages": [ + "spa-Latn" + ] + } + ], + "test": [ + { + "accuracy": 0.676689, + "f1": 0.592954, + "f1_weighted": 0.708916, + "ap": 0.876196, + "ap_weighted": 0.876196, + "scores_per_experiment": [ + { + "accuracy": 0.668919, + "f1": 0.583577, + "f1_weighted": 0.705858, + "ap": 0.870688, + "ap_weighted": 0.870688 + }, + { + "accuracy": 0.496622, + "f1": 0.467697, + "f1_weighted": 0.548184, + "ap": 0.854528, + "ap_weighted": 0.854528 + }, + { + "accuracy": 0.77027, + "f1": 0.675439, + "f1_weighted": 0.789237, + "ap": 0.896494, + "ap_weighted": 0.896494 + }, + { + "accuracy": 0.550676, + "f1": 0.499078, + "f1_weighted": 0.60336, + "ap": 0.852465, + "ap_weighted": 0.852465 + }, + { + "accuracy": 0.75, + "f1": 0.653856, + "f1_weighted": 0.772187, + "ap": 0.889645, + "ap_weighted": 0.889645 + }, + { + "accuracy": 0.709459, + "f1": 0.640025, + "f1_weighted": 0.742574, + "ap": 0.900784, + "ap_weighted": 0.900784 + }, + { + "accuracy": 0.689189, + "f1": 0.635058, + "f1_weighted": 0.726227, + "ap": 0.911323, + "ap_weighted": 0.911323 + }, + { + "accuracy": 0.743243, + "f1": 0.612352, + "f1_weighted": 0.758463, + "ap": 0.865031, + "ap_weighted": 0.865031 + }, + { + "accuracy": 0.658784, + "f1": 0.565762, + "f1_weighted": 0.696128, + "ap": 0.86105, + "ap_weighted": 0.86105 + }, + { + "accuracy": 0.72973, + "f1": 0.596703, + "f1_weighted": 0.746945, + "ap": 0.859954, + "ap_weighted": 0.859954 + } + ], + "main_score": 0.676689, + "hf_subset": "default", + "languages": [ + "spa-Latn" + ] + } + ] + }, + "evaluation_time": 1.7732319831848145, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SprintDuplicateQuestions.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SprintDuplicateQuestions.json deleted file mode 100644 index 7d5d128f3e..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/SprintDuplicateQuestions.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "test": { - "cos_sim": { - "accuracy": 0.997960396039604, - "accuracy_threshold": 0.8219642043113708, - "ap": 0.9454933848900439, - "f1": 0.8967935871743486, - "f1_threshold": 0.820462703704834, - "precision": 0.8985943775100401, - "recall": 0.895 - }, - "dot": { - "accuracy": 0.997960396039604, - "accuracy_threshold": 0.8219641447067261, - "ap": 0.9454933848900441, - "f1": 0.8967935871743486, - "f1_threshold": 0.8204627633094788, - "precision": 0.8985943775100401, - "recall": 0.895 - }, - "euclidean": { - "accuracy": 0.997960396039604, - "accuracy_threshold": 0.596717357635498, - "ap": 0.945493384890044, - "f1": 0.8967935871743486, - "f1_threshold": 0.5992283225059509, - "precision": 0.8985943775100401, - "recall": 0.895 - }, - "evaluation_time": 11.48, - "manhattan": { - "accuracy": 0.9979306930693069, - "accuracy_threshold": 13.289129257202148, - "ap": 0.9452019688650029, - "f1": 0.8963807635101637, - "f1_threshold": 13.34301471710205, - "precision": 0.8888888888888888, - "recall": 0.904 - }, - "max": { - "accuracy": 0.997960396039604, - "ap": 0.9454933848900441, - "f1": 0.8967935871743486 - } - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "SprintDuplicateQuestions", - "dataset_revision": "5a8256d0dff9c4bd3be3ba3e67e4e70173f802ea" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/StackExchangeClustering.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/StackExchangeClustering.json deleted file mode 100644 index b680388e95..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/StackExchangeClustering.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "test": { - "evaluation_time": 255.46, - "v_measure": 0.6421338133178497, - "v_measure_std": 0.048122638752667826 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "StackExchangeClustering", - "dataset_revision": "70a89468f6dccacc6aa2b12a6eac54e74328f235" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/StackExchangeClusteringP2P.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/StackExchangeClusteringP2P.json deleted file mode 100644 index 397384af2c..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/StackExchangeClusteringP2P.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "test": { - "evaluation_time": 752.69, - "v_measure": 0.3301453796958954, - "v_measure_std": 0.014356606846064176 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "StackExchangeClusteringP2P", - "dataset_revision": "d88009ab563dd0b16cfaf4436abaf97fa3550cf0" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/StackOverflowDupQuestions.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/StackOverflowDupQuestions.json deleted file mode 100644 index 68dc777867..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/StackOverflowDupQuestions.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "test": { - "evaluation_time": 40.95, - "map": 0.510120398337891, - "mrr": 0.5192411244800951 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "StackOverflowDupQuestions", - "dataset_revision": "ef807ea29a75ec4f91b50fd4191cb4ee4589a9f9" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SummEval.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SummEval.json deleted file mode 100644 index 39b21f7958..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/SummEval.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "test": { - "cos_sim": { - "pearson": 0.3080162749242551, - "spearman": 0.29669058153671907 - }, - "dot": { - "pearson": 0.3080162304936266, - "spearman": 0.29669058153671907 - }, - "evaluation_time": 24.01 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "SummEval", - "dataset_revision": "8753c2788d36c01fc6f05d03fe3f7268d63f9122" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SwahiliNewsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SwahiliNewsClassification.json new file mode 100644 index 0000000000..de4e02bf1a --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SwahiliNewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "24fcf066e6b96f9e0d743e8b79184e0c599f73c3", + "task_name": "SwahiliNewsClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.464648, + "f1": 0.42049, + "f1_weighted": 0.474564, + "scores_per_experiment": [ + { + "accuracy": 0.508301, + "f1": 0.454469, + "f1_weighted": 0.527696 + }, + { + "accuracy": 0.399902, + "f1": 0.3415, + "f1_weighted": 0.397928 + }, + { + "accuracy": 0.530762, + "f1": 0.475858, + "f1_weighted": 0.553268 + }, + { + "accuracy": 0.40918, + "f1": 0.375054, + "f1_weighted": 0.406667 + }, + { + "accuracy": 0.416016, + "f1": 0.399618, + "f1_weighted": 0.403743 + }, + { + "accuracy": 0.466797, + "f1": 0.428925, + "f1_weighted": 0.477292 + }, + { + "accuracy": 0.469238, + "f1": 0.426885, + "f1_weighted": 0.463717 + }, + { + "accuracy": 0.533691, + "f1": 0.471322, + "f1_weighted": 0.563072 + }, + { + "accuracy": 0.437988, + "f1": 0.406734, + "f1_weighted": 0.448275 + }, + { + "accuracy": 0.474609, + "f1": 0.424541, + "f1_weighted": 0.503984 + } + ], + "main_score": 0.464648, + "hf_subset": "default", + "languages": [ + "swa-Latn" + ] + } + ] + }, + "evaluation_time": 27.613702535629272, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SweRecClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SweRecClassification.json new file mode 100644 index 0000000000..79171e1af3 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SweRecClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "b07c6ce548f6a7ac8d546e1bbe197a0086409190", + "task_name": "SweRecClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.527295, + "f1": 0.474005, + "f1_weighted": 0.55626, + "scores_per_experiment": [ + { + "accuracy": 0.539062, + "f1": 0.451048, + "f1_weighted": 0.542283 + }, + { + "accuracy": 0.602051, + "f1": 0.536402, + "f1_weighted": 0.627852 + }, + { + "accuracy": 0.512695, + "f1": 0.480433, + "f1_weighted": 0.552507 + }, + { + "accuracy": 0.483398, + "f1": 0.436741, + "f1_weighted": 0.519066 + }, + { + "accuracy": 0.538574, + "f1": 0.476738, + "f1_weighted": 0.569272 + }, + { + "accuracy": 0.502441, + "f1": 0.442866, + "f1_weighted": 0.521155 + }, + { + "accuracy": 0.521484, + "f1": 0.482865, + "f1_weighted": 0.565389 + }, + { + "accuracy": 0.578125, + "f1": 0.52344, + "f1_weighted": 0.610399 + }, + { + "accuracy": 0.515625, + "f1": 0.478855, + "f1_weighted": 0.541969 + }, + { + "accuracy": 0.479492, + "f1": 0.430663, + "f1_weighted": 0.51271 + } + ], + "main_score": 0.527295, + "hf_subset": "default", + "languages": [ + "swe-Latn" + ] + } + ] + }, + "evaluation_time": 7.810220241546631, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SwedishSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SwedishSentimentClassification.json new file mode 100644 index 0000000000..e98697a004 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SwedishSentimentClassification.json @@ -0,0 +1,181 @@ +{ + "dataset_revision": "105ba6b3cb99b9fd64880215be469d60ebf44a1b", + "task_name": "SwedishSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "validation": [ + { + "accuracy": 0.695068, + "f1": 0.693974, + "f1_weighted": 0.693965, + "ap": 0.638417, + "ap_weighted": 0.638417, + "scores_per_experiment": [ + { + "accuracy": 0.712402, + "f1": 0.709879, + "f1_weighted": 0.709958, + "ap": 0.645441, + "ap_weighted": 0.645441 + }, + { + "accuracy": 0.710449, + "f1": 0.709996, + "f1_weighted": 0.709962, + "ap": 0.65491, + "ap_weighted": 0.65491 + }, + { + "accuracy": 0.661621, + "f1": 0.660273, + "f1_weighted": 0.660211, + "ap": 0.612339, + "ap_weighted": 0.612339 + }, + { + "accuracy": 0.678711, + "f1": 0.677815, + "f1_weighted": 0.677765, + "ap": 0.626679, + "ap_weighted": 0.626679 + }, + { + "accuracy": 0.71582, + "f1": 0.715722, + "f1_weighted": 0.715707, + "ap": 0.65781, + "ap_weighted": 0.65781 + }, + { + "accuracy": 0.67334, + "f1": 0.670652, + "f1_weighted": 0.67074, + "ap": 0.613366, + "ap_weighted": 0.613366 + }, + { + "accuracy": 0.737305, + "f1": 0.737148, + "f1_weighted": 0.737167, + "ap": 0.673738, + "ap_weighted": 0.673738 + }, + { + "accuracy": 0.723145, + "f1": 0.723143, + "f1_weighted": 0.723145, + "ap": 0.662586, + "ap_weighted": 0.662586 + }, + { + "accuracy": 0.672363, + "f1": 0.671824, + "f1_weighted": 0.671785, + "ap": 0.620092, + "ap_weighted": 0.620092 + }, + { + "accuracy": 0.665527, + "f1": 0.663288, + "f1_weighted": 0.663208, + "ap": 0.617204, + "ap_weighted": 0.617204 + } + ], + "main_score": 0.695068, + "hf_subset": "default", + "languages": [ + "swe-Latn" + ] + } + ], + "test": [ + { + "accuracy": 0.695312, + "f1": 0.694101, + "f1_weighted": 0.694091, + "ap": 0.638422, + "ap_weighted": 0.638422, + "scores_per_experiment": [ + { + "accuracy": 0.722168, + "f1": 0.720124, + "f1_weighted": 0.720171, + "ap": 0.654058, + "ap_weighted": 0.654058 + }, + { + "accuracy": 0.697754, + "f1": 0.697348, + "f1_weighted": 0.697326, + "ap": 0.642125, + "ap_weighted": 0.642125 + }, + { + "accuracy": 0.662598, + "f1": 0.660753, + "f1_weighted": 0.660705, + "ap": 0.613426, + "ap_weighted": 0.613426 + }, + { + "accuracy": 0.690918, + "f1": 0.690007, + "f1_weighted": 0.689975, + "ap": 0.637427, + "ap_weighted": 0.637427 + }, + { + "accuracy": 0.712891, + "f1": 0.712644, + "f1_weighted": 0.712628, + "ap": 0.655628, + "ap_weighted": 0.655628 + }, + { + "accuracy": 0.67334, + "f1": 0.671047, + "f1_weighted": 0.6711, + "ap": 0.613258, + "ap_weighted": 0.613258 + }, + { + "accuracy": 0.729004, + "f1": 0.728808, + "f1_weighted": 0.728823, + "ap": 0.665196, + "ap_weighted": 0.665196 + }, + { + "accuracy": 0.737793, + "f1": 0.737725, + "f1_weighted": 0.737733, + "ap": 0.674622, + "ap_weighted": 0.674622 + }, + { + "accuracy": 0.669434, + "f1": 0.668237, + "f1_weighted": 0.668198, + "ap": 0.618435, + "ap_weighted": 0.618435 + }, + { + "accuracy": 0.657227, + "f1": 0.654314, + "f1_weighted": 0.654252, + "ap": 0.610043, + "ap_weighted": 0.610043 + } + ], + "main_score": 0.695312, + "hf_subset": "default", + "languages": [ + "swe-Latn" + ] + } + ] + }, + "evaluation_time": 17.027771711349487, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SwissJudgementClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SwissJudgementClassification.json new file mode 100644 index 0000000000..f6af784cc2 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SwissJudgementClassification.json @@ -0,0 +1,263 @@ +{ + "dataset_revision": "29806f87bba4f23d0707d3b6d9ea5432afefbe2f", + "task_name": "SwissJudgementClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.506494, + "f1": 0.444007, + "f1_weighted": 0.543563, + "ap": 0.198274, + "ap_weighted": 0.198274, + "scores_per_experiment": [ + { + "accuracy": 0.440918, + "f1": 0.419769, + "f1_weighted": 0.486949, + "ap": 0.201459, + "ap_weighted": 0.201459 + }, + { + "accuracy": 0.544434, + "f1": 0.467372, + "f1_weighted": 0.590235, + "ap": 0.195618, + "ap_weighted": 0.195618 + }, + { + "accuracy": 0.356934, + "f1": 0.354606, + "f1_weighted": 0.378112, + "ap": 0.19901, + "ap_weighted": 0.19901 + }, + { + "accuracy": 0.631836, + "f1": 0.495726, + "f1_weighted": 0.654606, + "ap": 0.197108, + "ap_weighted": 0.197108 + }, + { + "accuracy": 0.592285, + "f1": 0.49657, + "f1_weighted": 0.629692, + "ap": 0.203125, + "ap_weighted": 0.203125 + }, + { + "accuracy": 0.484375, + "f1": 0.433452, + "f1_weighted": 0.536459, + "ap": 0.190465, + "ap_weighted": 0.190465 + }, + { + "accuracy": 0.566895, + "f1": 0.48104, + "f1_weighted": 0.609049, + "ap": 0.198891, + "ap_weighted": 0.198891 + }, + { + "accuracy": 0.628906, + "f1": 0.504563, + "f1_weighted": 0.655084, + "ap": 0.201606, + "ap_weighted": 0.201606 + }, + { + "accuracy": 0.352539, + "f1": 0.347499, + "f1_weighted": 0.382278, + "ap": 0.188846, + "ap_weighted": 0.188846 + }, + { + "accuracy": 0.46582, + "f1": 0.439475, + "f1_weighted": 0.51317, + "ap": 0.206614, + "ap_weighted": 0.206614 + } + ], + "main_score": 0.506494, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ] + }, + { + "accuracy": 0.530029, + "f1": 0.452843, + "f1_weighted": 0.566733, + "ap": 0.19007, + "ap_weighted": 0.19007, + "scores_per_experiment": [ + { + "accuracy": 0.685547, + "f1": 0.521145, + "f1_weighted": 0.697603, + "ap": 0.194244, + "ap_weighted": 0.194244 + }, + { + "accuracy": 0.532715, + "f1": 0.470301, + "f1_weighted": 0.584652, + "ap": 0.193455, + "ap_weighted": 0.193455 + }, + { + "accuracy": 0.447266, + "f1": 0.428824, + "f1_weighted": 0.49337, + "ap": 0.203734, + "ap_weighted": 0.203734 + }, + { + "accuracy": 0.580566, + "f1": 0.493032, + "f1_weighted": 0.625517, + "ap": 0.195023, + "ap_weighted": 0.195023 + }, + { + "accuracy": 0.630859, + "f1": 0.510981, + "f1_weighted": 0.663253, + "ap": 0.19542, + "ap_weighted": 0.19542 + }, + { + "accuracy": 0.317383, + "f1": 0.316391, + "f1_weighted": 0.332765, + "ap": 0.181277, + "ap_weighted": 0.181277 + }, + { + "accuracy": 0.44043, + "f1": 0.400588, + "f1_weighted": 0.497777, + "ap": 0.174961, + "ap_weighted": 0.174961 + }, + { + "accuracy": 0.402344, + "f1": 0.383506, + "f1_weighted": 0.45128, + "ap": 0.180835, + "ap_weighted": 0.180835 + }, + { + "accuracy": 0.585938, + "f1": 0.482392, + "f1_weighted": 0.627989, + "ap": 0.186774, + "ap_weighted": 0.186774 + }, + { + "accuracy": 0.677246, + "f1": 0.521269, + "f1_weighted": 0.693124, + "ap": 0.194981, + "ap_weighted": 0.194981 + } + ], + "main_score": 0.530029, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ] + }, + { + "accuracy": 0.545443, + "f1": 0.463476, + "f1_weighted": 0.590713, + "ap": 0.188873, + "ap_weighted": 0.188873, + "scores_per_experiment": [ + { + "accuracy": 0.477833, + "f1": 0.43314, + "f1_weighted": 0.532718, + "ap": 0.186136, + "ap_weighted": 0.186136 + }, + { + "accuracy": 0.479064, + "f1": 0.436569, + "f1_weighted": 0.533374, + "ap": 0.188688, + "ap_weighted": 0.188688 + }, + { + "accuracy": 0.575123, + "f1": 0.473317, + "f1_weighted": 0.618184, + "ap": 0.185139, + "ap_weighted": 0.185139 + }, + { + "accuracy": 0.48399, + "f1": 0.453093, + "f1_weighted": 0.534418, + "ap": 0.204916, + "ap_weighted": 0.204916 + }, + { + "accuracy": 0.637931, + "f1": 0.473272, + "f1_weighted": 0.657516, + "ap": 0.180486, + "ap_weighted": 0.180486 + }, + { + "accuracy": 0.580049, + "f1": 0.483548, + "f1_weighted": 0.623214, + "ap": 0.189948, + "ap_weighted": 0.189948 + }, + { + "accuracy": 0.541872, + "f1": 0.463707, + "f1_weighted": 0.591797, + "ap": 0.18656, + "ap_weighted": 0.18656 + }, + { + "accuracy": 0.507389, + "f1": 0.436502, + "f1_weighted": 0.561539, + "ap": 0.17882, + "ap_weighted": 0.17882 + }, + { + "accuracy": 0.544335, + "f1": 0.468913, + "f1_weighted": 0.594123, + "ap": 0.189383, + "ap_weighted": 0.189383 + }, + { + "accuracy": 0.626847, + "f1": 0.512699, + "f1_weighted": 0.66025, + "ap": 0.198653, + "ap_weighted": 0.198653 + } + ], + "main_score": 0.545443, + "hf_subset": "it", + "languages": [ + "ita-Latn" + ] + } + ] + }, + "evaluation_time": 67.11631178855896, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAAnger.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAAnger.json new file mode 100644 index 0000000000..2985caa58f --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAAnger.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "5cae68b7fc094cb2fa6890a464e4d836e8107f5e", + "task_name": "SynPerChatbotConvSAAnger", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.640976, + "f1": 0.568623, + "f1_weighted": 0.62633, + "ap": 0.709992, + "ap_weighted": 0.709992, + "scores_per_experiment": [ + { + "accuracy": 0.534146, + "f1": 0.528965, + "f1_weighted": 0.546075, + "ap": 0.704944, + "ap_weighted": 0.704944 + }, + { + "accuracy": 0.602439, + "f1": 0.582779, + "f1_weighted": 0.614147, + "ap": 0.721984, + "ap_weighted": 0.721984 + }, + { + "accuracy": 0.660976, + "f1": 0.544994, + "f1_weighted": 0.624556, + "ap": 0.697165, + "ap_weighted": 0.697165 + }, + { + "accuracy": 0.685366, + "f1": 0.638954, + "f1_weighted": 0.683787, + "ap": 0.741709, + "ap_weighted": 0.741709 + }, + { + "accuracy": 0.67561, + "f1": 0.583266, + "f1_weighted": 0.651208, + "ap": 0.711898, + "ap_weighted": 0.711898 + }, + { + "accuracy": 0.64878, + "f1": 0.541956, + "f1_weighted": 0.618568, + "ap": 0.69471, + "ap_weighted": 0.69471 + }, + { + "accuracy": 0.65122, + "f1": 0.554548, + "f1_weighted": 0.626419, + "ap": 0.699261, + "ap_weighted": 0.699261 + }, + { + "accuracy": 0.65122, + "f1": 0.53795, + "f1_weighted": 0.617183, + "ap": 0.693741, + "ap_weighted": 0.693741 + }, + { + "accuracy": 0.665854, + "f1": 0.568172, + "f1_weighted": 0.639304, + "ap": 0.705452, + "ap_weighted": 0.705452 + }, + { + "accuracy": 0.634146, + "f1": 0.604644, + "f1_weighted": 0.642049, + "ap": 0.729056, + "ap_weighted": 0.729056 + } + ], + "main_score": 0.640976, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 17.048197031021118, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAClassification.json new file mode 100644 index 0000000000..cc547b198e --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAClassification.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "1", + "task_name": "SynPerChatbotConvSAClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.592283, + "main_score": 0.592283, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 71.24925923347473, + "kg_co2_emissions": NaN +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAFear.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAFear.json new file mode 100644 index 0000000000..2662b4af4b --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAFear.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "3c22f7e6bf4e366c86d69293c9164bf9e9d80aac", + "task_name": "SynPerChatbotConvSAFear", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.624786, + "f1": 0.598189, + "f1_weighted": 0.617412, + "ap": 0.66391, + "ap_weighted": 0.66391, + "scores_per_experiment": [ + { + "accuracy": 0.564103, + "f1": 0.562953, + "f1_weighted": 0.567742, + "ap": 0.650175, + "ap_weighted": 0.650175 + }, + { + "accuracy": 0.589744, + "f1": 0.584615, + "f1_weighted": 0.594477, + "ap": 0.657699, + "ap_weighted": 0.657699 + }, + { + "accuracy": 0.683761, + "f1": 0.658192, + "f1_weighted": 0.678168, + "ap": 0.694475, + "ap_weighted": 0.694475 + }, + { + "accuracy": 0.649573, + "f1": 0.631369, + "f1_weighted": 0.648872, + "ap": 0.679743, + "ap_weighted": 0.679743 + }, + { + "accuracy": 0.649573, + "f1": 0.628284, + "f1_weighted": 0.647292, + "ap": 0.677049, + "ap_weighted": 0.677049 + }, + { + "accuracy": 0.623932, + "f1": 0.558642, + "f1_weighted": 0.594914, + "ap": 0.641335, + "ap_weighted": 0.641335 + }, + { + "accuracy": 0.623932, + "f1": 0.582143, + "f1_weighted": 0.610379, + "ap": 0.650073, + "ap_weighted": 0.650073 + }, + { + "accuracy": 0.641026, + "f1": 0.571951, + "f1_weighted": 0.608693, + "ap": 0.649002, + "ap_weighted": 0.649002 + }, + { + "accuracy": 0.57265, + "f1": 0.572368, + "f1_weighted": 0.574711, + "ap": 0.659805, + "ap_weighted": 0.659805 + }, + { + "accuracy": 0.649573, + "f1": 0.631369, + "f1_weighted": 0.648872, + "ap": 0.679743, + "ap_weighted": 0.679743 + } + ], + "main_score": 0.624786, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 6.569813251495361, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAFriendship.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAFriendship.json new file mode 100644 index 0000000000..2fe939ff30 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAFriendship.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "9dae119101e9b4e9bb40d5b9d29ffd7a621f9942", + "task_name": "SynPerChatbotConvSAFriendship", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.548551, + "f1": 0.538511, + "f1_weighted": 0.554981, + "ap": 0.705496, + "ap_weighted": 0.705496, + "scores_per_experiment": [ + { + "accuracy": 0.57971, + "f1": 0.564242, + "f1_weighted": 0.591609, + "ap": 0.706673, + "ap_weighted": 0.706673 + }, + { + "accuracy": 0.557971, + "f1": 0.556083, + "f1_weighted": 0.565733, + "ap": 0.720487, + "ap_weighted": 0.720487 + }, + { + "accuracy": 0.586957, + "f1": 0.542541, + "f1_weighted": 0.590055, + "ap": 0.686859, + "ap_weighted": 0.686859 + }, + { + "accuracy": 0.550725, + "f1": 0.539009, + "f1_weighted": 0.563506, + "ap": 0.695386, + "ap_weighted": 0.695386 + }, + { + "accuracy": 0.514493, + "f1": 0.514263, + "f1_weighted": 0.517783, + "ap": 0.702233, + "ap_weighted": 0.702233 + }, + { + "accuracy": 0.557971, + "f1": 0.556083, + "f1_weighted": 0.565733, + "ap": 0.720487, + "ap_weighted": 0.720487 + }, + { + "accuracy": 0.521739, + "f1": 0.521639, + "f1_weighted": 0.519328, + "ap": 0.715662, + "ap_weighted": 0.715662 + }, + { + "accuracy": 0.471014, + "f1": 0.470987, + "f1_weighted": 0.469709, + "ap": 0.682126, + "ap_weighted": 0.682126 + }, + { + "accuracy": 0.543478, + "f1": 0.522386, + "f1_weighted": 0.555842, + "ap": 0.681739, + "ap_weighted": 0.681739 + }, + { + "accuracy": 0.601449, + "f1": 0.597881, + "f1_weighted": 0.610508, + "ap": 0.743306, + "ap_weighted": 0.743306 + } + ], + "main_score": 0.548551, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 6.410936594009399, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAHappiness.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAHappiness.json new file mode 100644 index 0000000000..535015ed1e --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAHappiness.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "e60893b7a8d01c9b8c12fadfe8f0a06e9d548a63", + "task_name": "SynPerChatbotConvSAHappiness", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.548305, + "f1": 0.526172, + "f1_weighted": 0.554356, + "ap": 0.702735, + "ap_weighted": 0.702735, + "scores_per_experiment": [ + { + "accuracy": 0.610169, + "f1": 0.579225, + "f1_weighted": 0.61984, + "ap": 0.720145, + "ap_weighted": 0.720145 + }, + { + "accuracy": 0.525424, + "f1": 0.508333, + "f1_weighted": 0.54096, + "ap": 0.689534, + "ap_weighted": 0.689534 + }, + { + "accuracy": 0.415254, + "f1": 0.413189, + "f1_weighted": 0.425579, + "ap": 0.658456, + "ap_weighted": 0.658456 + }, + { + "accuracy": 0.686441, + "f1": 0.638427, + "f1_weighted": 0.685324, + "ap": 0.745752, + "ap_weighted": 0.745752 + }, + { + "accuracy": 0.415254, + "f1": 0.414202, + "f1_weighted": 0.423037, + "ap": 0.661071, + "ap_weighted": 0.661071 + }, + { + "accuracy": 0.483051, + "f1": 0.478519, + "f1_weighted": 0.495822, + "ap": 0.684728, + "ap_weighted": 0.684728 + }, + { + "accuracy": 0.610169, + "f1": 0.58691, + "f1_weighted": 0.621799, + "ap": 0.727722, + "ap_weighted": 0.727722 + }, + { + "accuracy": 0.415254, + "f1": 0.414876, + "f1_weighted": 0.409581, + "ap": 0.671979, + "ap_weighted": 0.671979 + }, + { + "accuracy": 0.610169, + "f1": 0.579225, + "f1_weighted": 0.61984, + "ap": 0.720145, + "ap_weighted": 0.720145 + }, + { + "accuracy": 0.711864, + "f1": 0.64881, + "f1_weighted": 0.701776, + "ap": 0.747812, + "ap_weighted": 0.747812 + } + ], + "main_score": 0.548305, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 6.224488258361816, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAJealousy.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAJealousy.json new file mode 100644 index 0000000000..5e5e6a93fe --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAJealousy.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "0d5104ecaa109d2448afe1f40dbf860f5e4458a8", + "task_name": "SynPerChatbotConvSAJealousy", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.582759, + "f1": 0.569379, + "f1_weighted": 0.586732, + "ap": 0.66124, + "ap_weighted": 0.66124, + "scores_per_experiment": [ + { + "accuracy": 0.655172, + "f1": 0.644608, + "f1_weighted": 0.659398, + "ap": 0.706897, + "ap_weighted": 0.706897 + }, + { + "accuracy": 0.586207, + "f1": 0.560606, + "f1_weighted": 0.586207, + "ap": 0.651341, + "ap_weighted": 0.651341 + }, + { + "accuracy": 0.551724, + "f1": 0.549582, + "f1_weighted": 0.55708, + "ap": 0.656499, + "ap_weighted": 0.656499 + }, + { + "accuracy": 0.586207, + "f1": 0.573529, + "f1_weighted": 0.591278, + "ap": 0.661518, + "ap_weighted": 0.661518 + }, + { + "accuracy": 0.551724, + "f1": 0.54303, + "f1_weighted": 0.558245, + "ap": 0.646232, + "ap_weighted": 0.646232 + }, + { + "accuracy": 0.586207, + "f1": 0.560606, + "f1_weighted": 0.586207, + "ap": 0.651341, + "ap_weighted": 0.651341 + }, + { + "accuracy": 0.586207, + "f1": 0.573529, + "f1_weighted": 0.591278, + "ap": 0.661518, + "ap_weighted": 0.661518 + }, + { + "accuracy": 0.655172, + "f1": 0.633838, + "f1_weighted": 0.655172, + "ap": 0.694019, + "ap_weighted": 0.694019 + }, + { + "accuracy": 0.482759, + "f1": 0.472727, + "f1_weighted": 0.490282, + "ap": 0.610345, + "ap_weighted": 0.610345 + }, + { + "accuracy": 0.586207, + "f1": 0.581731, + "f1_weighted": 0.592175, + "ap": 0.672687, + "ap_weighted": 0.672687 + } + ], + "main_score": 0.582759, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 5.722012758255005, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSALove.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSALove.json new file mode 100644 index 0000000000..eada7fb981 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSALove.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "0e000b2f73e9bb74ec8fc6da10011c52725b8469", + "task_name": "SynPerChatbotConvSALove", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.508571, + "f1": 0.474869, + "f1_weighted": 0.506832, + "ap": 0.747217, + "ap_weighted": 0.747217, + "scores_per_experiment": [ + { + "accuracy": 0.457143, + "f1": 0.449959, + "f1_weighted": 0.423018, + "ap": 0.782857, + "ap_weighted": 0.782857 + }, + { + "accuracy": 0.485714, + "f1": 0.485294, + "f1_weighted": 0.478992, + "ap": 0.770159, + "ap_weighted": 0.770159 + }, + { + "accuracy": 0.485714, + "f1": 0.403409, + "f1_weighted": 0.498377, + "ap": 0.677019, + "ap_weighted": 0.677019 + }, + { + "accuracy": 0.457143, + "f1": 0.427218, + "f1_weighted": 0.483327, + "ap": 0.691429, + "ap_weighted": 0.691429 + }, + { + "accuracy": 0.514286, + "f1": 0.421769, + "f1_weighted": 0.520894, + "ap": 0.68381, + "ap_weighted": 0.68381 + }, + { + "accuracy": 0.542857, + "f1": 0.542484, + "f1_weighted": 0.536881, + "ap": 0.817143, + "ap_weighted": 0.817143 + }, + { + "accuracy": 0.542857, + "f1": 0.542484, + "f1_weighted": 0.548086, + "ap": 0.792208, + "ap_weighted": 0.792208 + }, + { + "accuracy": 0.6, + "f1": 0.476496, + "f1_weighted": 0.58547, + "ap": 0.706243, + "ap_weighted": 0.706243 + }, + { + "accuracy": 0.485714, + "f1": 0.485294, + "f1_weighted": 0.478992, + "ap": 0.770159, + "ap_weighted": 0.770159 + }, + { + "accuracy": 0.514286, + "f1": 0.514286, + "f1_weighted": 0.514286, + "ap": 0.781143, + "ap_weighted": 0.781143 + } + ], + "main_score": 0.508571, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 5.711996793746948, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSASadness.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSASadness.json new file mode 100644 index 0000000000..5b4be57b75 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSASadness.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "e9c678325565a5e4dadc43fd6693a8ccff1dd6b2", + "task_name": "SynPerChatbotConvSASadness", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.728431, + "f1": 0.673258, + "f1_weighted": 0.716269, + "ap": 0.760871, + "ap_weighted": 0.760871, + "scores_per_experiment": [ + { + "accuracy": 0.784314, + "f1": 0.749554, + "f1_weighted": 0.780655, + "ap": 0.799224, + "ap_weighted": 0.799224 + }, + { + "accuracy": 0.666667, + "f1": 0.647131, + "f1_weighted": 0.674807, + "ap": 0.752197, + "ap_weighted": 0.752197 + }, + { + "accuracy": 0.754902, + "f1": 0.722192, + "f1_weighted": 0.753967, + "ap": 0.786019, + "ap_weighted": 0.786019 + }, + { + "accuracy": 0.784314, + "f1": 0.740278, + "f1_weighted": 0.775926, + "ap": 0.788635, + "ap_weighted": 0.788635 + }, + { + "accuracy": 0.686275, + "f1": 0.564103, + "f1_weighted": 0.641026, + "ap": 0.701203, + "ap_weighted": 0.701203 + }, + { + "accuracy": 0.666667, + "f1": 0.561235, + "f1_weighted": 0.632928, + "ap": 0.697654, + "ap_weighted": 0.697654 + }, + { + "accuracy": 0.754902, + "f1": 0.701929, + "f1_weighted": 0.743815, + "ap": 0.765979, + "ap_weighted": 0.765979 + }, + { + "accuracy": 0.676471, + "f1": 0.579408, + "f1_weighted": 0.646757, + "ap": 0.704996, + "ap_weighted": 0.704996 + }, + { + "accuracy": 0.705882, + "f1": 0.698463, + "f1_weighted": 0.714229, + "ap": 0.804706, + "ap_weighted": 0.804706 + }, + { + "accuracy": 0.803922, + "f1": 0.768287, + "f1_weighted": 0.798576, + "ap": 0.808095, + "ap_weighted": 0.808095 + } + ], + "main_score": 0.728431, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 6.317902326583862, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSASatisfaction.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSASatisfaction.json new file mode 100644 index 0000000000..c9a1cfc981 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSASatisfaction.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "50fd9d5d09edd53af89af765636be5db6f983f0e", + "task_name": "SynPerChatbotConvSASatisfaction", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.57296, + "f1": 0.556831, + "f1_weighted": 0.569236, + "ap": 0.68588, + "ap_weighted": 0.68588, + "scores_per_experiment": [ + { + "accuracy": 0.592075, + "f1": 0.583373, + "f1_weighted": 0.599794, + "ap": 0.68753, + "ap_weighted": 0.68753 + }, + { + "accuracy": 0.615385, + "f1": 0.608224, + "f1_weighted": 0.622669, + "ap": 0.704758, + "ap_weighted": 0.704758 + }, + { + "accuracy": 0.559441, + "f1": 0.559402, + "f1_weighted": 0.560523, + "ap": 0.692732, + "ap_weighted": 0.692732 + }, + { + "accuracy": 0.58042, + "f1": 0.575752, + "f1_weighted": 0.587888, + "ap": 0.687507, + "ap_weighted": 0.687507 + }, + { + "accuracy": 0.503497, + "f1": 0.499142, + "f1_weighted": 0.486406, + "ap": 0.675773, + "ap_weighted": 0.675773 + }, + { + "accuracy": 0.641026, + "f1": 0.628425, + "f1_weighted": 0.647087, + "ap": 0.712109, + "ap_weighted": 0.712109 + }, + { + "accuracy": 0.512821, + "f1": 0.507641, + "f1_weighted": 0.493869, + "ap": 0.684506, + "ap_weighted": 0.684506 + }, + { + "accuracy": 0.596737, + "f1": 0.588691, + "f1_weighted": 0.60438, + "ap": 0.691414, + "ap_weighted": 0.691414 + }, + { + "accuracy": 0.543124, + "f1": 0.433091, + "f1_weighted": 0.501206, + "ap": 0.617219, + "ap_weighted": 0.617219 + }, + { + "accuracy": 0.585082, + "f1": 0.584574, + "f1_weighted": 0.588535, + "ap": 0.705258, + "ap_weighted": 0.705258 + } + ], + "main_score": 0.57296, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 10.7613525390625, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSASurprise.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSASurprise.json new file mode 100644 index 0000000000..3ec5fc40a3 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSASurprise.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "62dad66fc2837b0ac5e5175fe7c265d2d502a386", + "task_name": "SynPerChatbotConvSASurprise", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.575207, + "f1": 0.549464, + "f1_weighted": 0.574273, + "ap": 0.657859, + "ap_weighted": 0.657859, + "scores_per_experiment": [ + { + "accuracy": 0.561983, + "f1": 0.487984, + "f1_weighted": 0.537853, + "ap": 0.626805, + "ap_weighted": 0.626805 + }, + { + "accuracy": 0.578512, + "f1": 0.571011, + "f1_weighted": 0.585545, + "ap": 0.671562, + "ap_weighted": 0.671562 + }, + { + "accuracy": 0.586777, + "f1": 0.561594, + "f1_weighted": 0.588514, + "ap": 0.6594, + "ap_weighted": 0.6594 + }, + { + "accuracy": 0.512397, + "f1": 0.501362, + "f1_weighted": 0.520366, + "ap": 0.631689, + "ap_weighted": 0.631689 + }, + { + "accuracy": 0.61157, + "f1": 0.589594, + "f1_weighted": 0.613925, + "ap": 0.675429, + "ap_weighted": 0.675429 + }, + { + "accuracy": 0.570248, + "f1": 0.565229, + "f1_weighted": 0.577197, + "ap": 0.670608, + "ap_weighted": 0.670608 + }, + { + "accuracy": 0.586777, + "f1": 0.513353, + "f1_weighted": 0.561781, + "ap": 0.638351, + "ap_weighted": 0.638351 + }, + { + "accuracy": 0.545455, + "f1": 0.543459, + "f1_weighted": 0.551192, + "ap": 0.662185, + "ap_weighted": 0.662185 + }, + { + "accuracy": 0.595041, + "f1": 0.575439, + "f1_weighted": 0.598811, + "ap": 0.6681, + "ap_weighted": 0.6681 + }, + { + "accuracy": 0.603306, + "f1": 0.585616, + "f1_weighted": 0.607551, + "ap": 0.674456, + "ap_weighted": 0.674456 + } + ], + "main_score": 0.575207, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 6.482559680938721, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAToneChatbotClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAToneChatbotClassification.json new file mode 100644 index 0000000000..90fecf8222 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAToneChatbotClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "1f403cfadb85004fbf7e2480334fffc4c999b4ab", + "task_name": "SynPerChatbotConvSAToneChatbotClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.620747, + "f1": 0.547341, + "f1_weighted": 0.623388, + "scores_per_experiment": [ + { + "accuracy": 0.619746, + "f1": 0.503114, + "f1_weighted": 0.609475 + }, + { + "accuracy": 0.642428, + "f1": 0.615076, + "f1_weighted": 0.65179 + }, + { + "accuracy": 0.659106, + "f1": 0.567872, + "f1_weighted": 0.65276 + }, + { + "accuracy": 0.555037, + "f1": 0.510155, + "f1_weighted": 0.574954 + }, + { + "accuracy": 0.585724, + "f1": 0.483405, + "f1_weighted": 0.590279 + }, + { + "accuracy": 0.617078, + "f1": 0.543185, + "f1_weighted": 0.616675 + }, + { + "accuracy": 0.649099, + "f1": 0.559667, + "f1_weighted": 0.641138 + }, + { + "accuracy": 0.602402, + "f1": 0.544728, + "f1_weighted": 0.616026 + }, + { + "accuracy": 0.657105, + "f1": 0.606776, + "f1_weighted": 0.656248 + }, + { + "accuracy": 0.619746, + "f1": 0.539434, + "f1_weighted": 0.624534 + } + ], + "main_score": 0.620747, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 19.683387756347656, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAToneUserClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAToneUserClassification.json new file mode 100644 index 0000000000..bcc2731e02 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotConvSAToneUserClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "dd0f76661bef69819cc38c8a455b10af86ac3571", + "task_name": "SynPerChatbotConvSAToneUserClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.44036, + "f1": 0.444972, + "f1_weighted": 0.429056, + "scores_per_experiment": [ + { + "accuracy": 0.362909, + "f1": 0.37549, + "f1_weighted": 0.317018 + }, + { + "accuracy": 0.529019, + "f1": 0.521503, + "f1_weighted": 0.545444 + }, + { + "accuracy": 0.487658, + "f1": 0.476924, + "f1_weighted": 0.4868 + }, + { + "accuracy": 0.393596, + "f1": 0.406393, + "f1_weighted": 0.363962 + }, + { + "accuracy": 0.36958, + "f1": 0.385952, + "f1_weighted": 0.340313 + }, + { + "accuracy": 0.401601, + "f1": 0.413126, + "f1_weighted": 0.370163 + }, + { + "accuracy": 0.460974, + "f1": 0.469266, + "f1_weighted": 0.46798 + }, + { + "accuracy": 0.469646, + "f1": 0.468234, + "f1_weighted": 0.464933 + }, + { + "accuracy": 0.495664, + "f1": 0.492908, + "f1_weighted": 0.506425 + }, + { + "accuracy": 0.432955, + "f1": 0.439928, + "f1_weighted": 0.427521 + } + ], + "main_score": 0.44036, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 19.940046787261963, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotRAGToneChatbotClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotRAGToneChatbotClassification.json new file mode 100644 index 0000000000..014b6d709a --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotRAGToneChatbotClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "76f15a203fc13bd98a8f0fdddab1b68c28d7d674", + "task_name": "SynPerChatbotRAGToneChatbotClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.326679, + "f1": 0.264656, + "f1_weighted": 0.334606, + "scores_per_experiment": [ + { + "accuracy": 0.281509, + "f1": 0.25413, + "f1_weighted": 0.294943 + }, + { + "accuracy": 0.302668, + "f1": 0.244399, + "f1_weighted": 0.30654 + }, + { + "accuracy": 0.24655, + "f1": 0.221368, + "f1_weighted": 0.259406 + }, + { + "accuracy": 0.379945, + "f1": 0.310528, + "f1_weighted": 0.393313 + }, + { + "accuracy": 0.429623, + "f1": 0.326288, + "f1_weighted": 0.445239 + }, + { + "accuracy": 0.25575, + "f1": 0.245267, + "f1_weighted": 0.264664 + }, + { + "accuracy": 0.377185, + "f1": 0.265133, + "f1_weighted": 0.384971 + }, + { + "accuracy": 0.275989, + "f1": 0.24289, + "f1_weighted": 0.271776 + }, + { + "accuracy": 0.357866, + "f1": 0.236828, + "f1_weighted": 0.348369 + }, + { + "accuracy": 0.359706, + "f1": 0.299733, + "f1_weighted": 0.376844 + } + ], + "main_score": 0.326679, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 13.74454402923584, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotRAGToneUserClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotRAGToneUserClassification.json new file mode 100644 index 0000000000..3e76c979a8 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotRAGToneUserClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f1f6ad83bb135dc94fbf1ca05c3ba164f5619369", + "task_name": "SynPerChatbotRAGToneUserClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.34977, + "f1": 0.316263, + "f1_weighted": 0.334344, + "scores_per_experiment": [ + { + "accuracy": 0.311868, + "f1": 0.283245, + "f1_weighted": 0.262944 + }, + { + "accuracy": 0.333027, + "f1": 0.323057, + "f1_weighted": 0.304778 + }, + { + "accuracy": 0.346826, + "f1": 0.301292, + "f1_weighted": 0.299232 + }, + { + "accuracy": 0.298988, + "f1": 0.282854, + "f1_weighted": 0.311739 + }, + { + "accuracy": 0.352346, + "f1": 0.302797, + "f1_weighted": 0.32634 + }, + { + "accuracy": 0.353266, + "f1": 0.326152, + "f1_weighted": 0.346957 + }, + { + "accuracy": 0.386385, + "f1": 0.322104, + "f1_weighted": 0.384877 + }, + { + "accuracy": 0.371665, + "f1": 0.355675, + "f1_weighted": 0.370587 + }, + { + "accuracy": 0.328427, + "f1": 0.316628, + "f1_weighted": 0.331898 + }, + { + "accuracy": 0.414903, + "f1": 0.348826, + "f1_weighted": 0.404092 + } + ], + "main_score": 0.34977, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 14.070990800857544, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotSatisfactionLevelClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotSatisfactionLevelClassification.json new file mode 100644 index 0000000000..b8fecbdf08 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotSatisfactionLevelClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "e72db473602d750f1bcdc9f0436e1e3b967e088f", + "task_name": "SynPerChatbotSatisfactionLevelClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.229668, + "f1": 0.191816, + "f1_weighted": 0.195324, + "scores_per_experiment": [ + { + "accuracy": 0.278465, + "f1": 0.197105, + "f1_weighted": 0.235525 + }, + { + "accuracy": 0.24203, + "f1": 0.20869, + "f1_weighted": 0.221784 + }, + { + "accuracy": 0.20039, + "f1": 0.17328, + "f1_weighted": 0.164078 + }, + { + "accuracy": 0.191932, + "f1": 0.160848, + "f1_weighted": 0.139407 + }, + { + "accuracy": 0.245934, + "f1": 0.218368, + "f1_weighted": 0.237908 + }, + { + "accuracy": 0.224463, + "f1": 0.209332, + "f1_weighted": 0.202798 + }, + { + "accuracy": 0.225764, + "f1": 0.2213, + "f1_weighted": 0.212994 + }, + { + "accuracy": 0.25244, + "f1": 0.183802, + "f1_weighted": 0.197757 + }, + { + "accuracy": 0.229018, + "f1": 0.174928, + "f1_weighted": 0.182672 + }, + { + "accuracy": 0.206246, + "f1": 0.170506, + "f1_weighted": 0.15832 + } + ], + "main_score": 0.229668, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 24.15795588493347, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotToneChatbotClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotToneChatbotClassification.json new file mode 100644 index 0000000000..980a22912f --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotToneChatbotClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "a5a739a036fa7bb8ae0be91bc081fdd260d4bdab", + "task_name": "SynPerChatbotToneChatbotClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.400846, + "f1": 0.311039, + "f1_weighted": 0.386933, + "scores_per_experiment": [ + { + "accuracy": 0.380612, + "f1": 0.295448, + "f1_weighted": 0.358644 + }, + { + "accuracy": 0.311646, + "f1": 0.267339, + "f1_weighted": 0.319985 + }, + { + "accuracy": 0.449577, + "f1": 0.336362, + "f1_weighted": 0.405751 + }, + { + "accuracy": 0.478204, + "f1": 0.333176, + "f1_weighted": 0.443881 + }, + { + "accuracy": 0.384515, + "f1": 0.342823, + "f1_weighted": 0.414139 + }, + { + "accuracy": 0.475602, + "f1": 0.335292, + "f1_weighted": 0.451413 + }, + { + "accuracy": 0.465843, + "f1": 0.340154, + "f1_weighted": 0.438588 + }, + { + "accuracy": 0.288224, + "f1": 0.239151, + "f1_weighted": 0.275223 + }, + { + "accuracy": 0.469096, + "f1": 0.34299, + "f1_weighted": 0.445729 + }, + { + "accuracy": 0.30514, + "f1": 0.277653, + "f1_weighted": 0.31598 + } + ], + "main_score": 0.400846, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 23.584723472595215, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotToneUserClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotToneUserClassification.json new file mode 100644 index 0000000000..7c9121dc30 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/SynPerChatbotToneUserClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "780d629437f7be127c6b287a61776372f9f333b9", + "task_name": "SynPerChatbotToneUserClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.311126, + "f1": 0.279023, + "f1_weighted": 0.281392, + "scores_per_experiment": [ + { + "accuracy": 0.359141, + "f1": 0.319711, + "f1_weighted": 0.347999 + }, + { + "accuracy": 0.324658, + "f1": 0.2913, + "f1_weighted": 0.292247 + }, + { + "accuracy": 0.35784, + "f1": 0.323753, + "f1_weighted": 0.340094 + }, + { + "accuracy": 0.309694, + "f1": 0.2643, + "f1_weighted": 0.293539 + }, + { + "accuracy": 0.283669, + "f1": 0.260644, + "f1_weighted": 0.225381 + }, + { + "accuracy": 0.266103, + "f1": 0.260017, + "f1_weighted": 0.229615 + }, + { + "accuracy": 0.338972, + "f1": 0.299014, + "f1_weighted": 0.303824 + }, + { + "accuracy": 0.350033, + "f1": 0.287522, + "f1_weighted": 0.327175 + }, + { + "accuracy": 0.264151, + "f1": 0.227564, + "f1_weighted": 0.21411 + }, + { + "accuracy": 0.256994, + "f1": 0.256405, + "f1_weighted": 0.239931 + } + ], + "main_score": 0.311126, + "hf_subset": "default", + "languages": [ + "fas-Arab" + ] + } + ] + }, + "evaluation_time": 24.0379958152771, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/TNews.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/TNews.json new file mode 100644 index 0000000000..1e5cd91776 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/TNews.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "317f262bf1e6126357bbe89e875451e4b0938fe4", + "task_name": "TNews", + "mteb_version": "1.34.14", + "scores": { + "validation": [ + { + "accuracy": 0.11467, + "f1": 0.094324, + "f1_weighted": 0.099418, + "scores_per_experiment": [ + { + "accuracy": 0.1092, + "f1": 0.089552, + "f1_weighted": 0.095499 + }, + { + "accuracy": 0.1327, + "f1": 0.097628, + "f1_weighted": 0.106081 + }, + { + "accuracy": 0.1208, + "f1": 0.105832, + "f1_weighted": 0.11397 + }, + { + "accuracy": 0.1143, + "f1": 0.093526, + "f1_weighted": 0.100729 + }, + { + "accuracy": 0.1115, + "f1": 0.093978, + "f1_weighted": 0.094767 + }, + { + "accuracy": 0.1052, + "f1": 0.082352, + "f1_weighted": 0.085534 + }, + { + "accuracy": 0.1188, + "f1": 0.093244, + "f1_weighted": 0.093384 + }, + { + "accuracy": 0.098, + "f1": 0.090432, + "f1_weighted": 0.093341 + }, + { + "accuracy": 0.107, + "f1": 0.080955, + "f1_weighted": 0.083251 + }, + { + "accuracy": 0.1292, + "f1": 0.115736, + "f1_weighted": 0.127629 + } + ], + "main_score": 0.11467, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 19.234732389450073, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/TRECCOVID.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/TRECCOVID.json deleted file mode 100644 index cd56db07b3..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/TRECCOVID.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "test": { - "evaluation_time": 1163.74, - "map_at_1": 0.00194, - "map_at_10": 0.01265, - "map_at_100": 0.05963, - "map_at_1000": 0.14234, - "map_at_3": 0.00489, - "map_at_5": 0.00739, - "ndcg_at_1": 0.61, - "ndcg_at_10": 0.56047, - "ndcg_at_100": 0.40695, - "ndcg_at_1000": 0.35494, - "ndcg_at_3": 0.60704, - "ndcg_at_5": 0.59182, - "precision_at_1": 0.7, - "precision_at_10": 0.602, - "precision_at_100": 0.4162, - "precision_at_1000": 0.16264, - "precision_at_3": 0.66667, - "precision_at_5": 0.652, - "recall_at_1": 0.00194, - "recall_at_10": 0.01541, - "recall_at_100": 0.09161, - "recall_at_1000": 0.33144, - "recall_at_3": 0.00531, - "recall_at_5": 0.00844 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "TRECCOVID", - "dataset_revision": "2c8041b2c07a79b6f7ba8fe6acc72e5d9f92d217" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/TamilNewsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/TamilNewsClassification.json new file mode 100644 index 0000000000..32a4dfe4cf --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/TamilNewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "bb34dd6690cf17aa731d75d45388c5801b8c4e4b", + "task_name": "TamilNewsClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.205322, + "f1": 0.182977, + "f1_weighted": 0.205953, + "scores_per_experiment": [ + { + "accuracy": 0.219727, + "f1": 0.193423, + "f1_weighted": 0.196527 + }, + { + "accuracy": 0.217773, + "f1": 0.199639, + "f1_weighted": 0.22949 + }, + { + "accuracy": 0.175781, + "f1": 0.165689, + "f1_weighted": 0.182426 + }, + { + "accuracy": 0.178223, + "f1": 0.160693, + "f1_weighted": 0.17538 + }, + { + "accuracy": 0.226562, + "f1": 0.173879, + "f1_weighted": 0.210094 + }, + { + "accuracy": 0.237305, + "f1": 0.210447, + "f1_weighted": 0.24911 + }, + { + "accuracy": 0.241699, + "f1": 0.23116, + "f1_weighted": 0.251777 + }, + { + "accuracy": 0.194824, + "f1": 0.171422, + "f1_weighted": 0.194957 + }, + { + "accuracy": 0.19043, + "f1": 0.165702, + "f1_weighted": 0.191574 + }, + { + "accuracy": 0.170898, + "f1": 0.157718, + "f1_weighted": 0.178193 + } + ], + "main_score": 0.182977, + "hf_subset": "default", + "languages": [ + "tam-Taml" + ] + } + ] + }, + "evaluation_time": 2.5826635360717773, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/TelemarketingSalesRuleLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/TelemarketingSalesRuleLegalBenchClassification.json new file mode 100644 index 0000000000..a03d230f51 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/TelemarketingSalesRuleLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "TelemarketingSalesRuleLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.531915, + "f1": 0.526557, + "f1_weighted": 0.516912, + "ap": 0.638081, + "ap_weighted": 0.638081, + "scores_per_experiment": [ + { + "accuracy": 0.531915, + "f1": 0.526557, + "f1_weighted": 0.516912, + "ap": 0.638081, + "ap_weighted": 0.638081 + }, + { + "accuracy": 0.531915, + "f1": 0.526557, + "f1_weighted": 0.516912, + "ap": 0.638081, + "ap_weighted": 0.638081 + }, + { + "accuracy": 0.531915, + "f1": 0.526557, + "f1_weighted": 0.516912, + "ap": 0.638081, + "ap_weighted": 0.638081 + }, + { + "accuracy": 0.531915, + "f1": 0.526557, + "f1_weighted": 0.516912, + "ap": 0.638081, + "ap_weighted": 0.638081 + }, + { + "accuracy": 0.531915, + "f1": 0.526557, + "f1_weighted": 0.516912, + "ap": 0.638081, + "ap_weighted": 0.638081 + }, + { + "accuracy": 0.531915, + "f1": 0.526557, + "f1_weighted": 0.516912, + "ap": 0.638081, + "ap_weighted": 0.638081 + }, + { + "accuracy": 0.531915, + "f1": 0.526557, + "f1_weighted": 0.516912, + "ap": 0.638081, + "ap_weighted": 0.638081 + }, + { + "accuracy": 0.531915, + "f1": 0.526557, + "f1_weighted": 0.516912, + "ap": 0.638081, + "ap_weighted": 0.638081 + }, + { + "accuracy": 0.531915, + "f1": 0.526557, + "f1_weighted": 0.516912, + "ap": 0.638081, + "ap_weighted": 0.638081 + }, + { + "accuracy": 0.531915, + "f1": 0.526557, + "f1_weighted": 0.516912, + "ap": 0.638081, + "ap_weighted": 0.638081 + } + ], + "main_score": 0.531915, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.5478470325469971, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/TeluguAndhraJyotiNewsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/TeluguAndhraJyotiNewsClassification.json new file mode 100644 index 0000000000..5ad3d484d9 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/TeluguAndhraJyotiNewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "3821aa93aa461c9263071e0897234e8d775ad616", + "task_name": "TeluguAndhraJyotiNewsClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.455957, + "f1": 0.403816, + "f1_weighted": 0.426089, + "scores_per_experiment": [ + { + "accuracy": 0.469238, + "f1": 0.421775, + "f1_weighted": 0.429734 + }, + { + "accuracy": 0.45459, + "f1": 0.393263, + "f1_weighted": 0.439373 + }, + { + "accuracy": 0.480957, + "f1": 0.442395, + "f1_weighted": 0.463547 + }, + { + "accuracy": 0.477539, + "f1": 0.439396, + "f1_weighted": 0.462459 + }, + { + "accuracy": 0.42334, + "f1": 0.373056, + "f1_weighted": 0.385138 + }, + { + "accuracy": 0.532227, + "f1": 0.481573, + "f1_weighted": 0.538405 + }, + { + "accuracy": 0.388184, + "f1": 0.326611, + "f1_weighted": 0.3454 + }, + { + "accuracy": 0.447754, + "f1": 0.381985, + "f1_weighted": 0.403147 + }, + { + "accuracy": 0.458496, + "f1": 0.377217, + "f1_weighted": 0.422174 + }, + { + "accuracy": 0.427246, + "f1": 0.400888, + "f1_weighted": 0.371518 + } + ], + "main_score": 0.403816, + "hf_subset": "default", + "languages": [ + "tel-Telu" + ] + } + ] + }, + "evaluation_time": 19.20594811439514, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/TenKGnadClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/TenKGnadClassification.json new file mode 100644 index 0000000000..cf42232017 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/TenKGnadClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "0798affe9b3f88cfda4267b6fbc50fac67046ee5", + "task_name": "TenKGnadClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.656615, + "f1": 0.633006, + "f1_weighted": 0.653854, + "scores_per_experiment": [ + { + "accuracy": 0.656615, + "f1": 0.63612, + "f1_weighted": 0.65595 + }, + { + "accuracy": 0.632296, + "f1": 0.603416, + "f1_weighted": 0.627492 + }, + { + "accuracy": 0.654669, + "f1": 0.634985, + "f1_weighted": 0.649044 + }, + { + "accuracy": 0.64786, + "f1": 0.615009, + "f1_weighted": 0.64678 + }, + { + "accuracy": 0.670233, + "f1": 0.649085, + "f1_weighted": 0.668879 + }, + { + "accuracy": 0.661479, + "f1": 0.63883, + "f1_weighted": 0.659391 + }, + { + "accuracy": 0.653696, + "f1": 0.625036, + "f1_weighted": 0.645502 + }, + { + "accuracy": 0.652724, + "f1": 0.634403, + "f1_weighted": 0.64962 + }, + { + "accuracy": 0.663424, + "f1": 0.640596, + "f1_weighted": 0.663466 + }, + { + "accuracy": 0.673152, + "f1": 0.652582, + "f1_weighted": 0.672418 + } + ], + "main_score": 0.656615, + "hf_subset": "default", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 18.763173818588257, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/TextualismToolDictionariesLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/TextualismToolDictionariesLegalBenchClassification.json new file mode 100644 index 0000000000..8126061e2d --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/TextualismToolDictionariesLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "TextualismToolDictionariesLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.869159, + "f1": 0.712366, + "f1_weighted": 0.889006, + "ap": 0.305241, + "ap_weighted": 0.305241, + "scores_per_experiment": [ + { + "accuracy": 0.869159, + "f1": 0.712366, + "f1_weighted": 0.889006, + "ap": 0.305241, + "ap_weighted": 0.305241 + }, + { + "accuracy": 0.869159, + "f1": 0.712366, + "f1_weighted": 0.889006, + "ap": 0.305241, + "ap_weighted": 0.305241 + }, + { + "accuracy": 0.869159, + "f1": 0.712366, + "f1_weighted": 0.889006, + "ap": 0.305241, + "ap_weighted": 0.305241 + }, + { + "accuracy": 0.869159, + "f1": 0.712366, + "f1_weighted": 0.889006, + "ap": 0.305241, + "ap_weighted": 0.305241 + }, + { + "accuracy": 0.869159, + "f1": 0.712366, + "f1_weighted": 0.889006, + "ap": 0.305241, + "ap_weighted": 0.305241 + }, + { + "accuracy": 0.869159, + "f1": 0.712366, + "f1_weighted": 0.889006, + "ap": 0.305241, + "ap_weighted": 0.305241 + }, + { + "accuracy": 0.869159, + "f1": 0.712366, + "f1_weighted": 0.889006, + "ap": 0.305241, + "ap_weighted": 0.305241 + }, + { + "accuracy": 0.869159, + "f1": 0.712366, + "f1_weighted": 0.889006, + "ap": 0.305241, + "ap_weighted": 0.305241 + }, + { + "accuracy": 0.869159, + "f1": 0.712366, + "f1_weighted": 0.889006, + "ap": 0.305241, + "ap_weighted": 0.305241 + }, + { + "accuracy": 0.869159, + "f1": 0.712366, + "f1_weighted": 0.889006, + "ap": 0.305241, + "ap_weighted": 0.305241 + } + ], + "main_score": 0.869159, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.2426159381866455, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/TextualismToolPlainLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/TextualismToolPlainLegalBenchClassification.json new file mode 100644 index 0000000000..dc76d48c5a --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/TextualismToolPlainLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "TextualismToolPlainLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.563636, + "f1": 0.55163, + "f1_weighted": 0.537846, + "ap": 0.472275, + "ap_weighted": 0.472275, + "scores_per_experiment": [ + { + "accuracy": 0.563636, + "f1": 0.55163, + "f1_weighted": 0.537846, + "ap": 0.472275, + "ap_weighted": 0.472275 + }, + { + "accuracy": 0.563636, + "f1": 0.55163, + "f1_weighted": 0.537846, + "ap": 0.472275, + "ap_weighted": 0.472275 + }, + { + "accuracy": 0.563636, + "f1": 0.55163, + "f1_weighted": 0.537846, + "ap": 0.472275, + "ap_weighted": 0.472275 + }, + { + "accuracy": 0.563636, + "f1": 0.55163, + "f1_weighted": 0.537846, + "ap": 0.472275, + "ap_weighted": 0.472275 + }, + { + "accuracy": 0.563636, + "f1": 0.55163, + "f1_weighted": 0.537846, + "ap": 0.472275, + "ap_weighted": 0.472275 + }, + { + "accuracy": 0.563636, + "f1": 0.55163, + "f1_weighted": 0.537846, + "ap": 0.472275, + "ap_weighted": 0.472275 + }, + { + "accuracy": 0.563636, + "f1": 0.55163, + "f1_weighted": 0.537846, + "ap": 0.472275, + "ap_weighted": 0.472275 + }, + { + "accuracy": 0.563636, + "f1": 0.55163, + "f1_weighted": 0.537846, + "ap": 0.472275, + "ap_weighted": 0.472275 + }, + { + "accuracy": 0.563636, + "f1": 0.55163, + "f1_weighted": 0.537846, + "ap": 0.472275, + "ap_weighted": 0.472275 + }, + { + "accuracy": 0.563636, + "f1": 0.55163, + "f1_weighted": 0.537846, + "ap": 0.472275, + "ap_weighted": 0.472275 + } + ], + "main_score": 0.563636, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.67960524559021, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/Touche2020.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/Touche2020.json deleted file mode 100644 index 02749d1d74..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/Touche2020.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "test": { - "evaluation_time": 2218.15, - "map_at_1": 0.02658, - "map_at_10": 0.10852, - "map_at_100": 0.16439, - "map_at_1000": 0.18023, - "map_at_3": 0.04969, - "map_at_5": 0.07058, - "ndcg_at_1": 0.27551, - "ndcg_at_10": 0.25891, - "ndcg_at_100": 0.36888, - "ndcg_at_1000": 0.48565, - "ndcg_at_3": 0.25311, - "ndcg_at_5": 0.25341, - "precision_at_1": 0.30612, - "precision_at_10": 0.2449, - "precision_at_100": 0.07673, - "precision_at_1000": 0.01541, - "precision_at_3": 0.26531, - "precision_at_5": 0.27347, - "recall_at_1": 0.02658, - "recall_at_10": 0.18234, - "recall_at_100": 0.4768, - "recall_at_1000": 0.83046, - "recall_at_3": 0.05969, - "recall_at_5": 0.10289 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "Touche2020", - "dataset_revision": "527b7d77e16e343303e68cb6af11d6e18b9f7b3b" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ToxicChatClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ToxicChatClassification.json new file mode 100644 index 0000000000..5085048a30 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/ToxicChatClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "3e0319203c7162b9c9f8015b594441f979c199bc", + "task_name": "ToxicChatClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.722337, + "f1": 0.60512, + "f1_weighted": 0.764039, + "ap": 0.231153, + "ap_weighted": 0.231153, + "scores_per_experiment": [ + { + "accuracy": 0.550687, + "f1": 0.481575, + "f1_weighted": 0.623378, + "ap": 0.167368, + "ap_weighted": 0.167368 + }, + { + "accuracy": 0.805842, + "f1": 0.650515, + "f1_weighted": 0.825058, + "ap": 0.241538, + "ap_weighted": 0.241538 + }, + { + "accuracy": 0.75945, + "f1": 0.646482, + "f1_weighted": 0.796191, + "ap": 0.271292, + "ap_weighted": 0.271292 + }, + { + "accuracy": 0.795533, + "f1": 0.671077, + "f1_weighted": 0.822649, + "ap": 0.285269, + "ap_weighted": 0.285269 + }, + { + "accuracy": 0.72079, + "f1": 0.576777, + "f1_weighted": 0.761725, + "ap": 0.18601, + "ap_weighted": 0.18601 + }, + { + "accuracy": 0.686426, + "f1": 0.582379, + "f1_weighted": 0.738539, + "ap": 0.218004, + "ap_weighted": 0.218004 + }, + { + "accuracy": 0.805842, + "f1": 0.650515, + "f1_weighted": 0.825058, + "ap": 0.241538, + "ap_weighted": 0.241538 + }, + { + "accuracy": 0.683849, + "f1": 0.584123, + "f1_weighted": 0.736686, + "ap": 0.223571, + "ap_weighted": 0.223571 + }, + { + "accuracy": 0.69244, + "f1": 0.592705, + "f1_weighted": 0.743693, + "ap": 0.231569, + "ap_weighted": 0.231569 + }, + { + "accuracy": 0.722509, + "f1": 0.615047, + "f1_weighted": 0.767415, + "ap": 0.24537, + "ap_weighted": 0.24537 + } + ], + "main_score": 0.722337, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.404684066772461, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/ToxicConversationsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/ToxicConversationsClassification.json index 8a9d31e0be..21db6e68c0 100644 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/ToxicConversationsClassification.json +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/ToxicConversationsClassification.json @@ -1,15 +1,95 @@ { - "test": { - "accuracy": 0.6660360000000001, - "accuracy_stderr": 0.044851273159186925, - "ap": 0.12610869387810816, - "ap_stderr": 0.0068727404045248715, - "evaluation_time": 1040.29, - "f1": 0.5142155209998112, - "f1_stderr": 0.025130875407843555, - "main_score": 0.12610869387810816 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "ToxicConversationsClassification", - "dataset_revision": "edfaf9da55d3dd50d43143d90c1ac476895ae6de" + "dataset_revision": "edfaf9da55d3dd50d43143d90c1ac476895ae6de", + "task_name": "ToxicConversationsClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.66582, + "f1": 0.498727, + "f1_weighted": 0.739936, + "ap": 0.108115, + "ap_weighted": 0.108115, + "scores_per_experiment": [ + { + "accuracy": 0.679688, + "f1": 0.498961, + "f1_weighted": 0.752272, + "ap": 0.101289, + "ap_weighted": 0.101289 + }, + { + "accuracy": 0.716309, + "f1": 0.527312, + "f1_weighted": 0.778918, + "ap": 0.114359, + "ap_weighted": 0.114359 + }, + { + "accuracy": 0.724121, + "f1": 0.525514, + "f1_weighted": 0.783928, + "ap": 0.109199, + "ap_weighted": 0.109199 + }, + { + "accuracy": 0.765137, + "f1": 0.561848, + "f1_weighted": 0.813081, + "ap": 0.130092, + "ap_weighted": 0.130092 + }, + { + "accuracy": 0.585449, + "f1": 0.45533, + "f1_weighted": 0.679432, + "ap": 0.099123, + "ap_weighted": 0.099123 + }, + { + "accuracy": 0.518555, + "f1": 0.418932, + "f1_weighted": 0.621467, + "ap": 0.094941, + "ap_weighted": 0.094941 + }, + { + "accuracy": 0.725586, + "f1": 0.521237, + "f1_weighted": 0.784539, + "ap": 0.104652, + "ap_weighted": 0.104652 + }, + { + "accuracy": 0.620117, + "f1": 0.481114, + "f1_weighted": 0.707191, + "ap": 0.110224, + "ap_weighted": 0.110224 + }, + { + "accuracy": 0.658691, + "f1": 0.495413, + "f1_weighted": 0.737037, + "ap": 0.106644, + "ap_weighted": 0.106644 + }, + { + "accuracy": 0.664551, + "f1": 0.501614, + "f1_weighted": 0.741497, + "ap": 0.110631, + "ap_weighted": 0.110631 + } + ], + "main_score": 0.66582, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 6.723692893981934, + "kg_co2_emissions": null } \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/TswanaNewsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/TswanaNewsClassification.json new file mode 100644 index 0000000000..dea0744090 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/TswanaNewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "061ca1525717eebaaa9bada240f6cbb31eb3aa87", + "task_name": "TswanaNewsClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.239836, + "f1": 0.23054, + "f1_weighted": 0.241804, + "scores_per_experiment": [ + { + "accuracy": 0.23614, + "f1": 0.220329, + "f1_weighted": 0.24529 + }, + { + "accuracy": 0.268994, + "f1": 0.264202, + "f1_weighted": 0.271215 + }, + { + "accuracy": 0.25462, + "f1": 0.234289, + "f1_weighted": 0.257466 + }, + { + "accuracy": 0.217659, + "f1": 0.200817, + "f1_weighted": 0.214256 + }, + { + "accuracy": 0.25462, + "f1": 0.227766, + "f1_weighted": 0.250457 + }, + { + "accuracy": 0.244353, + "f1": 0.239683, + "f1_weighted": 0.241195 + }, + { + "accuracy": 0.209446, + "f1": 0.215789, + "f1_weighted": 0.215879 + }, + { + "accuracy": 0.240246, + "f1": 0.23352, + "f1_weighted": 0.243878 + }, + { + "accuracy": 0.219713, + "f1": 0.217567, + "f1_weighted": 0.222208 + }, + { + "accuracy": 0.252567, + "f1": 0.251444, + "f1_weighted": 0.256196 + } + ], + "main_score": 0.239836, + "hf_subset": "default", + "languages": [ + "tsn-Latn" + ] + } + ] + }, + "evaluation_time": 14.453412771224976, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/TurkicClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/TurkicClassification.json new file mode 100644 index 0000000000..eb151696d7 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/TurkicClassification.json @@ -0,0 +1,197 @@ +{ + "dataset_revision": "db1a67c1bdd54fbb8536af026dc8596f00f9c41d", + "task_name": "TurkicClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.179834, + "f1": 0.137705, + "f1_weighted": 0.16934, + "scores_per_experiment": [ + { + "accuracy": 0.179199, + "f1": 0.129996, + "f1_weighted": 0.166698 + }, + { + "accuracy": 0.216309, + "f1": 0.148504, + "f1_weighted": 0.215254 + }, + { + "accuracy": 0.166504, + "f1": 0.129386, + "f1_weighted": 0.147065 + }, + { + "accuracy": 0.193848, + "f1": 0.1564, + "f1_weighted": 0.213238 + }, + { + "accuracy": 0.166992, + "f1": 0.138664, + "f1_weighted": 0.151306 + }, + { + "accuracy": 0.186035, + "f1": 0.145656, + "f1_weighted": 0.173989 + }, + { + "accuracy": 0.159668, + "f1": 0.135587, + "f1_weighted": 0.12755 + }, + { + "accuracy": 0.179688, + "f1": 0.136334, + "f1_weighted": 0.166379 + }, + { + "accuracy": 0.190918, + "f1": 0.137927, + "f1_weighted": 0.166088 + }, + { + "accuracy": 0.15918, + "f1": 0.118597, + "f1_weighted": 0.165836 + } + ], + "main_score": 0.179834, + "hf_subset": "ky", + "languages": [ + "kir-Cyrl" + ] + }, + { + "accuracy": 0.268213, + "f1": 0.20255, + "f1_weighted": 0.278541, + "scores_per_experiment": [ + { + "accuracy": 0.201172, + "f1": 0.205267, + "f1_weighted": 0.194035 + }, + { + "accuracy": 0.35791, + "f1": 0.225505, + "f1_weighted": 0.374039 + }, + { + "accuracy": 0.290039, + "f1": 0.205627, + "f1_weighted": 0.306715 + }, + { + "accuracy": 0.290039, + "f1": 0.2027, + "f1_weighted": 0.318867 + }, + { + "accuracy": 0.165039, + "f1": 0.166676, + "f1_weighted": 0.145186 + }, + { + "accuracy": 0.302734, + "f1": 0.247123, + "f1_weighted": 0.330717 + }, + { + "accuracy": 0.267578, + "f1": 0.191906, + "f1_weighted": 0.302291 + }, + { + "accuracy": 0.302246, + "f1": 0.205067, + "f1_weighted": 0.320519 + }, + { + "accuracy": 0.217285, + "f1": 0.18594, + "f1_weighted": 0.177422 + }, + { + "accuracy": 0.288086, + "f1": 0.189686, + "f1_weighted": 0.315615 + } + ], + "main_score": 0.268213, + "hf_subset": "kk", + "languages": [ + "kaz-Cyrl" + ] + }, + { + "accuracy": 0.093164, + "f1": 0.058131, + "f1_weighted": 0.096647, + "scores_per_experiment": [ + { + "accuracy": 0.097656, + "f1": 0.06318, + "f1_weighted": 0.105473 + }, + { + "accuracy": 0.070801, + "f1": 0.061248, + "f1_weighted": 0.056064 + }, + { + "accuracy": 0.094238, + "f1": 0.052619, + "f1_weighted": 0.097132 + }, + { + "accuracy": 0.103516, + "f1": 0.051605, + "f1_weighted": 0.11364 + }, + { + "accuracy": 0.058594, + "f1": 0.059282, + "f1_weighted": 0.034716 + }, + { + "accuracy": 0.094238, + "f1": 0.055249, + "f1_weighted": 0.10288 + }, + { + "accuracy": 0.09082, + "f1": 0.058601, + "f1_weighted": 0.093348 + }, + { + "accuracy": 0.100586, + "f1": 0.065336, + "f1_weighted": 0.108846 + }, + { + "accuracy": 0.098145, + "f1": 0.064941, + "f1_weighted": 0.105799 + }, + { + "accuracy": 0.123047, + "f1": 0.049247, + "f1_weighted": 0.148567 + } + ], + "main_score": 0.093164, + "hf_subset": "ba", + "languages": [ + "bak-Cyrl" + ] + } + ] + }, + "evaluation_time": 155.07717728614807, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/TurkishMovieSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/TurkishMovieSentimentClassification.json new file mode 100644 index 0000000000..0893965c2b --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/TurkishMovieSentimentClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "409a4415cce5f6bcfca6d5f3ca3c408211ca00b3", + "task_name": "TurkishMovieSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.54248, + "f1": 0.530593, + "f1_weighted": 0.530593, + "ap": 0.523864, + "ap_weighted": 0.523864, + "scores_per_experiment": [ + { + "accuracy": 0.50293, + "f1": 0.487799, + "f1_weighted": 0.487799, + "ap": 0.501478, + "ap_weighted": 0.501478 + }, + { + "accuracy": 0.559082, + "f1": 0.546204, + "f1_weighted": 0.546204, + "ap": 0.534805, + "ap_weighted": 0.534805 + }, + { + "accuracy": 0.560059, + "f1": 0.559403, + "f1_weighted": 0.559403, + "ap": 0.533378, + "ap_weighted": 0.533378 + }, + { + "accuracy": 0.539062, + "f1": 0.52723, + "f1_weighted": 0.52723, + "ap": 0.52069, + "ap_weighted": 0.52069 + }, + { + "accuracy": 0.522949, + "f1": 0.509016, + "f1_weighted": 0.509016, + "ap": 0.511869, + "ap_weighted": 0.511869 + }, + { + "accuracy": 0.493652, + "f1": 0.433974, + "f1_weighted": 0.433974, + "ap": 0.496941, + "ap_weighted": 0.496941 + }, + { + "accuracy": 0.553223, + "f1": 0.553007, + "f1_weighted": 0.553007, + "ap": 0.529574, + "ap_weighted": 0.529574 + }, + { + "accuracy": 0.568848, + "f1": 0.56713, + "f1_weighted": 0.56713, + "ap": 0.539847, + "ap_weighted": 0.539847 + }, + { + "accuracy": 0.558105, + "f1": 0.557512, + "f1_weighted": 0.557512, + "ap": 0.532696, + "ap_weighted": 0.532696 + }, + { + "accuracy": 0.566895, + "f1": 0.564652, + "f1_weighted": 0.564652, + "ap": 0.53736, + "ap_weighted": 0.53736 + } + ], + "main_score": 0.54248, + "hf_subset": "default", + "languages": [ + "tur-Latn" + ] + } + ] + }, + "evaluation_time": 4.185898303985596, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/TurkishProductSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/TurkishProductSentimentClassification.json new file mode 100644 index 0000000000..c9c778a257 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/TurkishProductSentimentClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "ad861e463abda351ff65ca5ac0cc5985afe9eb99", + "task_name": "TurkishProductSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.53725, + "f1": 0.531518, + "f1_weighted": 0.531518, + "ap": 0.520732, + "ap_weighted": 0.520732, + "scores_per_experiment": [ + { + "accuracy": 0.5, + "f1": 0.499622, + "f1_weighted": 0.499622, + "ap": 0.5, + "ap_weighted": 0.5 + }, + { + "accuracy": 0.51125, + "f1": 0.511158, + "f1_weighted": 0.511158, + "ap": 0.505755, + "ap_weighted": 0.505755 + }, + { + "accuracy": 0.51, + "f1": 0.509482, + "f1_weighted": 0.509482, + "ap": 0.505094, + "ap_weighted": 0.505094 + }, + { + "accuracy": 0.55125, + "f1": 0.548626, + "f1_weighted": 0.548626, + "ap": 0.528724, + "ap_weighted": 0.528724 + }, + { + "accuracy": 0.51875, + "f1": 0.49627, + "f1_weighted": 0.49627, + "ap": 0.509984, + "ap_weighted": 0.509984 + }, + { + "accuracy": 0.53125, + "f1": 0.515328, + "f1_weighted": 0.515328, + "ap": 0.516342, + "ap_weighted": 0.516342 + }, + { + "accuracy": 0.56375, + "f1": 0.563744, + "f1_weighted": 0.563744, + "ap": 0.53597, + "ap_weighted": 0.53597 + }, + { + "accuracy": 0.59375, + "f1": 0.59268, + "f1_weighted": 0.59268, + "ap": 0.554847, + "ap_weighted": 0.554847 + }, + { + "accuracy": 0.54, + "f1": 0.527524, + "f1_weighted": 0.527524, + "ap": 0.521208, + "ap_weighted": 0.521208 + }, + { + "accuracy": 0.5525, + "f1": 0.550745, + "f1_weighted": 0.550745, + "ap": 0.5294, + "ap_weighted": 0.5294 + } + ], + "main_score": 0.53725, + "hf_subset": "default", + "languages": [ + "tur-Latn" + ] + } + ] + }, + "evaluation_time": 3.482795000076294, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/TweetEmotionClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/TweetEmotionClassification.json new file mode 100644 index 0000000000..2f6b94ac34 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/TweetEmotionClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "0ded8ff72cc68cbb7bb5c01b0a9157982b73ddaf", + "task_name": "TweetEmotionClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.196143, + "f1": 0.174436, + "f1_weighted": 0.174634, + "scores_per_experiment": [ + { + "accuracy": 0.199219, + "f1": 0.176004, + "f1_weighted": 0.176898 + }, + { + "accuracy": 0.191406, + "f1": 0.178917, + "f1_weighted": 0.180593 + }, + { + "accuracy": 0.167969, + "f1": 0.163954, + "f1_weighted": 0.158655 + }, + { + "accuracy": 0.194824, + "f1": 0.164844, + "f1_weighted": 0.16683 + }, + { + "accuracy": 0.216797, + "f1": 0.185493, + "f1_weighted": 0.188131 + }, + { + "accuracy": 0.215332, + "f1": 0.192421, + "f1_weighted": 0.185956 + }, + { + "accuracy": 0.165039, + "f1": 0.138887, + "f1_weighted": 0.139254 + }, + { + "accuracy": 0.213379, + "f1": 0.182709, + "f1_weighted": 0.181739 + }, + { + "accuracy": 0.19873, + "f1": 0.176921, + "f1_weighted": 0.181695 + }, + { + "accuracy": 0.19873, + "f1": 0.18421, + "f1_weighted": 0.186586 + } + ], + "main_score": 0.196143, + "hf_subset": "default", + "languages": [ + "ara-Arab" + ] + } + ] + }, + "evaluation_time": 3.0842416286468506, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/TweetSarcasmClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/TweetSarcasmClassification.json new file mode 100644 index 0000000000..475d1d77d6 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/TweetSarcasmClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "557bf94ac6177cc442f42d0b09b6e4b76e8f47c9", + "task_name": "TweetSarcasmClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.603081, + "f1": 0.521199, + "f1_weighted": 0.649516, + "ap": 0.206412, + "ap_weighted": 0.206412, + "scores_per_experiment": [ + { + "accuracy": 0.679147, + "f1": 0.541874, + "f1_weighted": 0.710642, + "ap": 0.189438, + "ap_weighted": 0.189438 + }, + { + "accuracy": 0.537915, + "f1": 0.495621, + "f1_weighted": 0.593914, + "ap": 0.211753, + "ap_weighted": 0.211753 + }, + { + "accuracy": 0.563507, + "f1": 0.512084, + "f1_weighted": 0.618684, + "ap": 0.214511, + "ap_weighted": 0.214511 + }, + { + "accuracy": 0.527014, + "f1": 0.491994, + "f1_weighted": 0.581758, + "ap": 0.217593, + "ap_weighted": 0.217593 + }, + { + "accuracy": 0.57109, + "f1": 0.522514, + "f1_weighted": 0.625008, + "ap": 0.225252, + "ap_weighted": 0.225252 + }, + { + "accuracy": 0.578673, + "f1": 0.520488, + "f1_weighted": 0.6329, + "ap": 0.214513, + "ap_weighted": 0.214513 + }, + { + "accuracy": 0.654502, + "f1": 0.539336, + "f1_weighted": 0.694346, + "ap": 0.194723, + "ap_weighted": 0.194723 + }, + { + "accuracy": 0.696209, + "f1": 0.516223, + "f1_weighted": 0.714808, + "ap": 0.170594, + "ap_weighted": 0.170594 + }, + { + "accuracy": 0.599052, + "f1": 0.533683, + "f1_weighted": 0.651182, + "ap": 0.218168, + "ap_weighted": 0.218168 + }, + { + "accuracy": 0.623697, + "f1": 0.538173, + "f1_weighted": 0.671921, + "ap": 0.207579, + "ap_weighted": 0.207579 + } + ], + "main_score": 0.603081, + "hf_subset": "default", + "languages": [ + "ara-Arab" + ] + } + ] + }, + "evaluation_time": 2.9086320400238037, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/TweetSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/TweetSentimentClassification.json new file mode 100644 index 0000000000..47d5a8b187 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/TweetSentimentClassification.json @@ -0,0 +1,507 @@ +{ + "dataset_revision": "d522bb117c32f5e0207344f69f7075fc9941168b", + "task_name": "TweetSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.475781, + "f1": 0.470272, + "f1_weighted": 0.470366, + "scores_per_experiment": [ + { + "accuracy": 0.410156, + "f1": 0.394886, + "f1_weighted": 0.395119 + }, + { + "accuracy": 0.4375, + "f1": 0.433251, + "f1_weighted": 0.433501 + }, + { + "accuracy": 0.535156, + "f1": 0.531265, + "f1_weighted": 0.531621 + }, + { + "accuracy": 0.441406, + "f1": 0.437434, + "f1_weighted": 0.437411 + }, + { + "accuracy": 0.5625, + "f1": 0.559611, + "f1_weighted": 0.559453 + }, + { + "accuracy": 0.464844, + "f1": 0.460017, + "f1_weighted": 0.459793 + }, + { + "accuracy": 0.511719, + "f1": 0.507653, + "f1_weighted": 0.507771 + }, + { + "accuracy": 0.472656, + "f1": 0.458343, + "f1_weighted": 0.458636 + }, + { + "accuracy": 0.539062, + "f1": 0.536697, + "f1_weighted": 0.536876 + }, + { + "accuracy": 0.382812, + "f1": 0.383562, + "f1_weighted": 0.38348 + } + ], + "main_score": 0.475781, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + }, + { + "accuracy": 0.336719, + "f1": 0.294244, + "f1_weighted": 0.294099, + "scores_per_experiment": [ + { + "accuracy": 0.394531, + "f1": 0.315191, + "f1_weighted": 0.315812 + }, + { + "accuracy": 0.386719, + "f1": 0.385585, + "f1_weighted": 0.385312 + }, + { + "accuracy": 0.3125, + "f1": 0.225959, + "f1_weighted": 0.225076 + }, + { + "accuracy": 0.324219, + "f1": 0.254293, + "f1_weighted": 0.255134 + }, + { + "accuracy": 0.351562, + "f1": 0.354732, + "f1_weighted": 0.354661 + }, + { + "accuracy": 0.265625, + "f1": 0.249634, + "f1_weighted": 0.249715 + }, + { + "accuracy": 0.308594, + "f1": 0.293946, + "f1_weighted": 0.29362 + }, + { + "accuracy": 0.34375, + "f1": 0.302337, + "f1_weighted": 0.301586 + }, + { + "accuracy": 0.363281, + "f1": 0.275707, + "f1_weighted": 0.275151 + }, + { + "accuracy": 0.316406, + "f1": 0.285054, + "f1_weighted": 0.284923 + } + ], + "main_score": 0.336719, + "hf_subset": "arabic", + "languages": [ + "ara-Arab" + ] + }, + { + "accuracy": 0.389844, + "f1": 0.351493, + "f1_weighted": 0.351632, + "scores_per_experiment": [ + { + "accuracy": 0.328125, + "f1": 0.326338, + "f1_weighted": 0.325985 + }, + { + "accuracy": 0.398438, + "f1": 0.319767, + "f1_weighted": 0.31937 + }, + { + "accuracy": 0.417969, + "f1": 0.39596, + "f1_weighted": 0.396188 + }, + { + "accuracy": 0.390625, + "f1": 0.334271, + "f1_weighted": 0.334213 + }, + { + "accuracy": 0.324219, + "f1": 0.304908, + "f1_weighted": 0.305148 + }, + { + "accuracy": 0.390625, + "f1": 0.351383, + "f1_weighted": 0.350714 + }, + { + "accuracy": 0.503906, + "f1": 0.477925, + "f1_weighted": 0.47836 + }, + { + "accuracy": 0.429688, + "f1": 0.388308, + "f1_weighted": 0.388828 + }, + { + "accuracy": 0.394531, + "f1": 0.351943, + "f1_weighted": 0.352505 + }, + { + "accuracy": 0.320312, + "f1": 0.264127, + "f1_weighted": 0.265003 + } + ], + "main_score": 0.389844, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ] + }, + { + "accuracy": 0.403125, + "f1": 0.395774, + "f1_weighted": 0.395556, + "scores_per_experiment": [ + { + "accuracy": 0.34375, + "f1": 0.345283, + "f1_weighted": 0.345364 + }, + { + "accuracy": 0.394531, + "f1": 0.387552, + "f1_weighted": 0.38756 + }, + { + "accuracy": 0.417969, + "f1": 0.413378, + "f1_weighted": 0.412914 + }, + { + "accuracy": 0.386719, + "f1": 0.372675, + "f1_weighted": 0.372375 + }, + { + "accuracy": 0.453125, + "f1": 0.441753, + "f1_weighted": 0.441258 + }, + { + "accuracy": 0.429688, + "f1": 0.425142, + "f1_weighted": 0.424783 + }, + { + "accuracy": 0.457031, + "f1": 0.457121, + "f1_weighted": 0.457052 + }, + { + "accuracy": 0.441406, + "f1": 0.423591, + "f1_weighted": 0.42302 + }, + { + "accuracy": 0.332031, + "f1": 0.332109, + "f1_weighted": 0.332145 + }, + { + "accuracy": 0.375, + "f1": 0.359136, + "f1_weighted": 0.359091 + } + ], + "main_score": 0.403125, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "accuracy": 0.431641, + "f1": 0.428371, + "f1_weighted": 0.428333, + "scores_per_experiment": [ + { + "accuracy": 0.410156, + "f1": 0.402474, + "f1_weighted": 0.402618 + }, + { + "accuracy": 0.4375, + "f1": 0.431527, + "f1_weighted": 0.431229 + }, + { + "accuracy": 0.472656, + "f1": 0.469864, + "f1_weighted": 0.469631 + }, + { + "accuracy": 0.355469, + "f1": 0.351802, + "f1_weighted": 0.351806 + }, + { + "accuracy": 0.464844, + "f1": 0.459059, + "f1_weighted": 0.459357 + }, + { + "accuracy": 0.410156, + "f1": 0.408921, + "f1_weighted": 0.408541 + }, + { + "accuracy": 0.441406, + "f1": 0.440672, + "f1_weighted": 0.440492 + }, + { + "accuracy": 0.425781, + "f1": 0.42057, + "f1_weighted": 0.420659 + }, + { + "accuracy": 0.4375, + "f1": 0.436096, + "f1_weighted": 0.436081 + }, + { + "accuracy": 0.460938, + "f1": 0.462727, + "f1_weighted": 0.46291 + } + ], + "main_score": 0.431641, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "accuracy": 0.409375, + "f1": 0.397899, + "f1_weighted": 0.397663, + "scores_per_experiment": [ + { + "accuracy": 0.371094, + "f1": 0.364687, + "f1_weighted": 0.3648 + }, + { + "accuracy": 0.441406, + "f1": 0.431586, + "f1_weighted": 0.431119 + }, + { + "accuracy": 0.421875, + "f1": 0.420792, + "f1_weighted": 0.420503 + }, + { + "accuracy": 0.429688, + "f1": 0.398611, + "f1_weighted": 0.3979 + }, + { + "accuracy": 0.46875, + "f1": 0.455841, + "f1_weighted": 0.455216 + }, + { + "accuracy": 0.363281, + "f1": 0.35957, + "f1_weighted": 0.359673 + }, + { + "accuracy": 0.429688, + "f1": 0.423849, + "f1_weighted": 0.423828 + }, + { + "accuracy": 0.375, + "f1": 0.369773, + "f1_weighted": 0.36978 + }, + { + "accuracy": 0.382812, + "f1": 0.35402, + "f1_weighted": 0.353895 + }, + { + "accuracy": 0.410156, + "f1": 0.400256, + "f1_weighted": 0.399914 + } + ], + "main_score": 0.409375, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ] + }, + { + "accuracy": 0.340234, + "f1": 0.335847, + "f1_weighted": 0.335875, + "scores_per_experiment": [ + { + "accuracy": 0.351562, + "f1": 0.345842, + "f1_weighted": 0.345519 + }, + { + "accuracy": 0.316406, + "f1": 0.309621, + "f1_weighted": 0.30969 + }, + { + "accuracy": 0.3125, + "f1": 0.308891, + "f1_weighted": 0.309086 + }, + { + "accuracy": 0.363281, + "f1": 0.360776, + "f1_weighted": 0.360738 + }, + { + "accuracy": 0.34375, + "f1": 0.344165, + "f1_weighted": 0.344136 + }, + { + "accuracy": 0.320312, + "f1": 0.317635, + "f1_weighted": 0.317592 + }, + { + "accuracy": 0.382812, + "f1": 0.375459, + "f1_weighted": 0.375733 + }, + { + "accuracy": 0.347656, + "f1": 0.348034, + "f1_weighted": 0.348104 + }, + { + "accuracy": 0.324219, + "f1": 0.324071, + "f1_weighted": 0.32416 + }, + { + "accuracy": 0.339844, + "f1": 0.323972, + "f1_weighted": 0.323992 + } + ], + "main_score": 0.340234, + "hf_subset": "hindi", + "languages": [ + "hin-Deva" + ] + }, + { + "accuracy": 0.432031, + "f1": 0.411054, + "f1_weighted": 0.410775, + "scores_per_experiment": [ + { + "accuracy": 0.449219, + "f1": 0.433198, + "f1_weighted": 0.433246 + }, + { + "accuracy": 0.441406, + "f1": 0.399081, + "f1_weighted": 0.398139 + }, + { + "accuracy": 0.464844, + "f1": 0.458903, + "f1_weighted": 0.458653 + }, + { + "accuracy": 0.410156, + "f1": 0.380685, + "f1_weighted": 0.380862 + }, + { + "accuracy": 0.457031, + "f1": 0.442232, + "f1_weighted": 0.44199 + }, + { + "accuracy": 0.390625, + "f1": 0.354271, + "f1_weighted": 0.354135 + }, + { + "accuracy": 0.445312, + "f1": 0.445491, + "f1_weighted": 0.445313 + }, + { + "accuracy": 0.367188, + "f1": 0.357317, + "f1_weighted": 0.357443 + }, + { + "accuracy": 0.472656, + "f1": 0.434798, + "f1_weighted": 0.43406 + }, + { + "accuracy": 0.421875, + "f1": 0.404561, + "f1_weighted": 0.403914 + } + ], + "main_score": 0.432031, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 6.005335807800293, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/TweetSentimentExtractionClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/TweetSentimentExtractionClassification.json index 8be4ce1262..5852255506 100644 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/TweetSentimentExtractionClassification.json +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/TweetSentimentExtractionClassification.json @@ -1,13 +1,73 @@ { - "test": { - "accuracy": 0.5602150537634408, - "accuracy_stderr": 0.010283605517794947, - "evaluation_time": 32.29, - "f1": 0.5630529778288564, - "f1_stderr": 0.010445112661516681, - "main_score": 0.5602150537634408 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "TweetSentimentExtractionClassification", - "dataset_revision": "62146448f05be9e52a36b8ee9936447ea787eede" + "dataset_revision": "d604517c81ca91fe16a244d1248fc021f9ecee7a", + "task_name": "TweetSentimentExtractionClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.550113, + "f1": 0.553289, + "f1_weighted": 0.545274, + "scores_per_experiment": [ + { + "accuracy": 0.550934, + "f1": 0.553738, + "f1_weighted": 0.546054 + }, + { + "accuracy": 0.544992, + "f1": 0.548155, + "f1_weighted": 0.539215 + }, + { + "accuracy": 0.554046, + "f1": 0.557292, + "f1_weighted": 0.551086 + }, + { + "accuracy": 0.565365, + "f1": 0.570112, + "f1_weighted": 0.564005 + }, + { + "accuracy": 0.579513, + "f1": 0.583842, + "f1_weighted": 0.577581 + }, + { + "accuracy": 0.56678, + "f1": 0.568544, + "f1_weighted": 0.556838 + }, + { + "accuracy": 0.529145, + "f1": 0.529283, + "f1_weighted": 0.521481 + }, + { + "accuracy": 0.55631, + "f1": 0.559787, + "f1_weighted": 0.55051 + }, + { + "accuracy": 0.520939, + "f1": 0.525033, + "f1_weighted": 0.516111 + }, + { + "accuracy": 0.533107, + "f1": 0.537109, + "f1_weighted": 0.52986 + } + ], + "main_score": 0.550113, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 4.214621067047119, + "kg_co2_emissions": null } \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/TweetTopicSingleClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/TweetTopicSingleClassification.json new file mode 100644 index 0000000000..34812d5285 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/TweetTopicSingleClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "87b7a0d1c402dbb481db649569c556d9aa27ac05", + "task_name": "TweetTopicSingleClassification", + "mteb_version": "1.34.14", + "scores": { + "test_2021": [ + { + "accuracy": 0.59238, + "f1": 0.458507, + "f1_weighted": 0.633203, + "scores_per_experiment": [ + { + "accuracy": 0.587714, + "f1": 0.445926, + "f1_weighted": 0.634607 + }, + { + "accuracy": 0.588305, + "f1": 0.464936, + "f1_weighted": 0.621661 + }, + { + "accuracy": 0.626698, + "f1": 0.503333, + "f1_weighted": 0.657664 + }, + { + "accuracy": 0.578263, + "f1": 0.470078, + "f1_weighted": 0.610767 + }, + { + "accuracy": 0.613113, + "f1": 0.4857, + "f1_weighted": 0.659726 + }, + { + "accuracy": 0.604843, + "f1": 0.442997, + "f1_weighted": 0.639471 + }, + { + "accuracy": 0.57531, + "f1": 0.431741, + "f1_weighted": 0.632141 + }, + { + "accuracy": 0.640874, + "f1": 0.489833, + "f1_weighted": 0.68033 + }, + { + "accuracy": 0.509746, + "f1": 0.391205, + "f1_weighted": 0.549134 + }, + { + "accuracy": 0.598937, + "f1": 0.459317, + "f1_weighted": 0.646528 + } + ], + "main_score": 0.59238, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.411313533782959, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/TwentyNewsgroupsClustering.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/TwentyNewsgroupsClustering.json deleted file mode 100644 index ec67b301e0..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/TwentyNewsgroupsClustering.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "test": { - "evaluation_time": 30.57, - "v_measure": 0.46718954435107624, - "v_measure_std": 0.01752491994873024 - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "TwentyNewsgroupsClustering", - "dataset_revision": "091a54f9a36281ce7d6590ec8c75dd485e7e01d4" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/TwitterSemEval2015.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/TwitterSemEval2015.json deleted file mode 100644 index c8ddd84225..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/TwitterSemEval2015.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "test": { - "cos_sim": { - "accuracy": 0.853907134767837, - "accuracy_threshold": 0.8573521375656128, - "ap": 0.7222675077030647, - "f1": 0.6806487013778807, - "f1_threshold": 0.8156364560127258, - "precision": 0.6327363409657674, - "recall": 0.7364116094986808 - }, - "dot": { - "accuracy": 0.853907134767837, - "accuracy_threshold": 0.8573521375656128, - "ap": 0.7222675006771236, - "f1": 0.6806487013778807, - "f1_threshold": 0.8156364560127258, - "precision": 0.6327363409657674, - "recall": 0.7364116094986808 - }, - "euclidean": { - "accuracy": 0.853907134767837, - "accuracy_threshold": 0.5341306924819946, - "ap": 0.7222675376564271, - "f1": 0.6806487013778807, - "f1_threshold": 0.6072289943695068, - "precision": 0.6327363409657674, - "recall": 0.7364116094986808 - }, - "evaluation_time": 10.5, - "manhattan": { - "accuracy": 0.8540859510043511, - "accuracy_threshold": 12.074257850646973, - "ap": 0.7220275449321797, - "f1": 0.6808250945929453, - "f1_threshold": 13.386808395385742, - "precision": 0.6334317510788099, - "recall": 0.7358839050131926 - }, - "max": { - "accuracy": 0.8540859510043511, - "ap": 0.7222675376564271, - "f1": 0.6808250945929453 - } - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "TwitterSemEval2015", - "dataset_revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/TwitterURLCorpus.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/TwitterURLCorpus.json deleted file mode 100644 index 56e420f9a3..0000000000 --- a/results/sentence-transformers__gtr-t5-base/no_revision_available/TwitterURLCorpus.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "test": { - "cos_sim": { - "accuracy": 0.8866573524275235, - "accuracy_threshold": 0.800679624080658, - "ap": 0.8477048332956029, - "f1": 0.7737385487858077, - "f1_threshold": 0.7719563245773315, - "precision": 0.7329201101928374, - "recall": 0.819371727748691 - }, - "dot": { - "accuracy": 0.8866573524275235, - "accuracy_threshold": 0.8006796836853027, - "ap": 0.8477047124323303, - "f1": 0.7737385487858077, - "f1_threshold": 0.7719563841819763, - "precision": 0.7329201101928374, - "recall": 0.819371727748691 - }, - "euclidean": { - "accuracy": 0.8866573524275235, - "accuracy_threshold": 0.6313799619674683, - "ap": 0.8477045299031885, - "f1": 0.7737385487858077, - "f1_threshold": 0.6753424406051636, - "precision": 0.7329201101928374, - "recall": 0.819371727748691 - }, - "evaluation_time": 37.89, - "manhattan": { - "accuracy": 0.8866379477626422, - "accuracy_threshold": 13.86162281036377, - "ap": 0.8475078904725522, - "f1": 0.7741025355459034, - "f1_threshold": 14.698223114013672, - "precision": 0.7474370922646785, - "recall": 0.802740991684632 - }, - "max": { - "accuracy": 0.8866573524275235, - "ap": 0.8477048332956029, - "f1": 0.7741025355459034 - } - }, - "mteb_version": "0.0.2", - "mteb_dataset_name": "TwitterURLCorpus", - "dataset_revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf" -} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/UCCVCommonLawLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/UCCVCommonLawLegalBenchClassification.json new file mode 100644 index 0000000000..a9aa0c1c59 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/UCCVCommonLawLegalBenchClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "UCCVCommonLawLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.595745, + "f1": 0.591117, + "f1_weighted": 0.584639, + "ap": 0.499859, + "ap_weighted": 0.499859, + "scores_per_experiment": [ + { + "accuracy": 0.595745, + "f1": 0.591117, + "f1_weighted": 0.584639, + "ap": 0.499859, + "ap_weighted": 0.499859 + }, + { + "accuracy": 0.595745, + "f1": 0.591117, + "f1_weighted": 0.584639, + "ap": 0.499859, + "ap_weighted": 0.499859 + }, + { + "accuracy": 0.595745, + "f1": 0.591117, + "f1_weighted": 0.584639, + "ap": 0.499859, + "ap_weighted": 0.499859 + }, + { + "accuracy": 0.595745, + "f1": 0.591117, + "f1_weighted": 0.584639, + "ap": 0.499859, + "ap_weighted": 0.499859 + }, + { + "accuracy": 0.595745, + "f1": 0.591117, + "f1_weighted": 0.584639, + "ap": 0.499859, + "ap_weighted": 0.499859 + }, + { + "accuracy": 0.595745, + "f1": 0.591117, + "f1_weighted": 0.584639, + "ap": 0.499859, + "ap_weighted": 0.499859 + }, + { + "accuracy": 0.595745, + "f1": 0.591117, + "f1_weighted": 0.584639, + "ap": 0.499859, + "ap_weighted": 0.499859 + }, + { + "accuracy": 0.595745, + "f1": 0.591117, + "f1_weighted": 0.584639, + "ap": 0.499859, + "ap_weighted": 0.499859 + }, + { + "accuracy": 0.595745, + "f1": 0.591117, + "f1_weighted": 0.584639, + "ap": 0.499859, + "ap_weighted": 0.499859 + }, + { + "accuracy": 0.595745, + "f1": 0.591117, + "f1_weighted": 0.584639, + "ap": 0.499859, + "ap_weighted": 0.499859 + } + ], + "main_score": 0.595745, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.6309599876403809, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/UkrFormalityClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/UkrFormalityClassification.json new file mode 100644 index 0000000000..d9152468e3 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/UkrFormalityClassification.json @@ -0,0 +1,181 @@ +{ + "dataset_revision": "671d1e6bbf45a74ef21af351fd4ef7b32b7856f8", + "task_name": "UkrFormalityClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.52207, + "f1": 0.506013, + "f1_weighted": 0.506013, + "ap": 0.512421, + "ap_weighted": 0.512421, + "scores_per_experiment": [ + { + "accuracy": 0.525879, + "f1": 0.524431, + "f1_weighted": 0.524431, + "ap": 0.513692, + "ap_weighted": 0.513692 + }, + { + "accuracy": 0.512695, + "f1": 0.473619, + "f1_weighted": 0.473619, + "ap": 0.506452, + "ap_weighted": 0.506452 + }, + { + "accuracy": 0.538086, + "f1": 0.5188, + "f1_weighted": 0.5188, + "ap": 0.521462, + "ap_weighted": 0.521462 + }, + { + "accuracy": 0.510742, + "f1": 0.509753, + "f1_weighted": 0.509753, + "ap": 0.505477, + "ap_weighted": 0.505477 + }, + { + "accuracy": 0.524414, + "f1": 0.510607, + "f1_weighted": 0.510607, + "ap": 0.512653, + "ap_weighted": 0.512653 + }, + { + "accuracy": 0.452148, + "f1": 0.429737, + "f1_weighted": 0.429737, + "ap": 0.479868, + "ap_weighted": 0.479868 + }, + { + "accuracy": 0.561523, + "f1": 0.522423, + "f1_weighted": 0.522423, + "ap": 0.533169, + "ap_weighted": 0.533169 + }, + { + "accuracy": 0.531738, + "f1": 0.529633, + "f1_weighted": 0.529633, + "ap": 0.516758, + "ap_weighted": 0.516758 + }, + { + "accuracy": 0.537598, + "f1": 0.518193, + "f1_weighted": 0.518193, + "ap": 0.52116, + "ap_weighted": 0.52116 + }, + { + "accuracy": 0.525879, + "f1": 0.522931, + "f1_weighted": 0.522931, + "ap": 0.513518, + "ap_weighted": 0.513518 + } + ], + "main_score": 0.52207, + "hf_subset": "default", + "languages": [ + "ukr-Cyrl" + ] + } + ], + "test": [ + { + "accuracy": 0.520459, + "f1": 0.502172, + "f1_weighted": 0.504579, + "ap": 0.447154, + "ap_weighted": 0.447154, + "scores_per_experiment": [ + { + "accuracy": 0.537598, + "f1": 0.53371, + "f1_weighted": 0.539406, + "ap": 0.451702, + "ap_weighted": 0.451702 + }, + { + "accuracy": 0.475586, + "f1": 0.449966, + "f1_weighted": 0.434084, + "ap": 0.439943, + "ap_weighted": 0.439943 + }, + { + "accuracy": 0.555664, + "f1": 0.519479, + "f1_weighted": 0.537121, + "ap": 0.448592, + "ap_weighted": 0.448592 + }, + { + "accuracy": 0.506836, + "f1": 0.506384, + "f1_weighted": 0.508383, + "ap": 0.439121, + "ap_weighted": 0.439121 + }, + { + "accuracy": 0.505859, + "f1": 0.50166, + "f1_weighted": 0.49554, + "ap": 0.447294, + "ap_weighted": 0.447294 + }, + { + "accuracy": 0.475586, + "f1": 0.427772, + "f1_weighted": 0.449902, + "ap": 0.412126, + "ap_weighted": 0.412126 + }, + { + "accuracy": 0.51416, + "f1": 0.482441, + "f1_weighted": 0.4653, + "ap": 0.463462, + "ap_weighted": 0.463462 + }, + { + "accuracy": 0.536621, + "f1": 0.536621, + "f1_weighted": 0.536651, + "ap": 0.457969, + "ap_weighted": 0.457969 + }, + { + "accuracy": 0.564941, + "f1": 0.532045, + "f1_weighted": 0.548645, + "ap": 0.455138, + "ap_weighted": 0.455138 + }, + { + "accuracy": 0.531738, + "f1": 0.531644, + "f1_weighted": 0.530757, + "ap": 0.456198, + "ap_weighted": 0.456198 + } + ], + "main_score": 0.520459, + "hf_subset": "default", + "languages": [ + "ukr-Cyrl" + ] + } + ] + }, + "evaluation_time": 6.122084856033325, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/UnfairTOSLegalBenchClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/UnfairTOSLegalBenchClassification.json new file mode 100644 index 0000000000..9844877613 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/UnfairTOSLegalBenchClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "12ca3b695563788fead87a982ad1a068284413f4", + "task_name": "UnfairTOSLegalBenchClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.146484, + "f1": 0.142441, + "f1_weighted": 0.192226, + "scores_per_experiment": [ + { + "accuracy": 0.146484, + "f1": 0.142441, + "f1_weighted": 0.192226 + }, + { + "accuracy": 0.146484, + "f1": 0.142441, + "f1_weighted": 0.192226 + }, + { + "accuracy": 0.146484, + "f1": 0.142441, + "f1_weighted": 0.192226 + }, + { + "accuracy": 0.146484, + "f1": 0.142441, + "f1_weighted": 0.192226 + }, + { + "accuracy": 0.146484, + "f1": 0.142441, + "f1_weighted": 0.192226 + }, + { + "accuracy": 0.146484, + "f1": 0.142441, + "f1_weighted": 0.192226 + }, + { + "accuracy": 0.146484, + "f1": 0.142441, + "f1_weighted": 0.192226 + }, + { + "accuracy": 0.146484, + "f1": 0.142441, + "f1_weighted": 0.192226 + }, + { + "accuracy": 0.146484, + "f1": 0.142441, + "f1_weighted": 0.192226 + }, + { + "accuracy": 0.146484, + "f1": 0.142441, + "f1_weighted": 0.192226 + } + ], + "main_score": 0.146484, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.214916706085205, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/UrduRomanSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/UrduRomanSentimentClassification.json new file mode 100644 index 0000000000..9a9952e2f9 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/UrduRomanSentimentClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "566be6449bb30b9b9f2b59173391647fe0ca3224", + "task_name": "UrduRomanSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "train": [ + { + "accuracy": 0.409863, + "f1": 0.396522, + "f1_weighted": 0.409415, + "scores_per_experiment": [ + { + "accuracy": 0.473633, + "f1": 0.429667, + "f1_weighted": 0.456734 + }, + { + "accuracy": 0.414062, + "f1": 0.412533, + "f1_weighted": 0.413867 + }, + { + "accuracy": 0.42627, + "f1": 0.419281, + "f1_weighted": 0.431597 + }, + { + "accuracy": 0.411133, + "f1": 0.399139, + "f1_weighted": 0.411337 + }, + { + "accuracy": 0.404297, + "f1": 0.385494, + "f1_weighted": 0.403281 + }, + { + "accuracy": 0.402832, + "f1": 0.398796, + "f1_weighted": 0.406419 + }, + { + "accuracy": 0.396484, + "f1": 0.391959, + "f1_weighted": 0.402901 + }, + { + "accuracy": 0.387207, + "f1": 0.363531, + "f1_weighted": 0.381709 + }, + { + "accuracy": 0.388672, + "f1": 0.380846, + "f1_weighted": 0.393459 + }, + { + "accuracy": 0.394043, + "f1": 0.383974, + "f1_weighted": 0.39285 + } + ], + "main_score": 0.396522, + "hf_subset": "default", + "languages": [ + "urd-Latn" + ] + } + ] + }, + "evaluation_time": 3.803243398666382, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/VieStudentFeedbackClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/VieStudentFeedbackClassification.json new file mode 100644 index 0000000000..3bbbe74ac7 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/VieStudentFeedbackClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "7b56c6cb1c9c8523249f407044c838660df3811a", + "task_name": "VieStudentFeedbackClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.49834, + "f1": 0.41141, + "f1_weighted": 0.533055, + "scores_per_experiment": [ + { + "accuracy": 0.544922, + "f1": 0.450147, + "f1_weighted": 0.583783 + }, + { + "accuracy": 0.546875, + "f1": 0.442553, + "f1_weighted": 0.581927 + }, + { + "accuracy": 0.538574, + "f1": 0.437374, + "f1_weighted": 0.554671 + }, + { + "accuracy": 0.51416, + "f1": 0.437408, + "f1_weighted": 0.571894 + }, + { + "accuracy": 0.338867, + "f1": 0.310737, + "f1_weighted": 0.396102 + }, + { + "accuracy": 0.428223, + "f1": 0.388047, + "f1_weighted": 0.495397 + }, + { + "accuracy": 0.468262, + "f1": 0.391006, + "f1_weighted": 0.515553 + }, + { + "accuracy": 0.464844, + "f1": 0.354934, + "f1_weighted": 0.453292 + }, + { + "accuracy": 0.577148, + "f1": 0.449729, + "f1_weighted": 0.599828 + }, + { + "accuracy": 0.561523, + "f1": 0.452163, + "f1_weighted": 0.578101 + } + ], + "main_score": 0.49834, + "hf_subset": "default", + "languages": [ + "vie-Latn" + ] + } + ] + }, + "evaluation_time": 3.4378955364227295, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/WRIMEClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/WRIMEClassification.json new file mode 100644 index 0000000000..094345fda4 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/WRIMEClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "3fb7212c389d7818b8e6179e2cdac762f2e081d9", + "task_name": "WRIMEClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.197021, + "f1": 0.160201, + "f1_weighted": 0.179778, + "scores_per_experiment": [ + { + "accuracy": 0.208984, + "f1": 0.150505, + "f1_weighted": 0.199235 + }, + { + "accuracy": 0.15625, + "f1": 0.134106, + "f1_weighted": 0.143568 + }, + { + "accuracy": 0.225586, + "f1": 0.161804, + "f1_weighted": 0.175294 + }, + { + "accuracy": 0.223145, + "f1": 0.187253, + "f1_weighted": 0.211441 + }, + { + "accuracy": 0.147461, + "f1": 0.128923, + "f1_weighted": 0.128518 + }, + { + "accuracy": 0.165039, + "f1": 0.144472, + "f1_weighted": 0.157731 + }, + { + "accuracy": 0.222168, + "f1": 0.177365, + "f1_weighted": 0.182449 + }, + { + "accuracy": 0.247559, + "f1": 0.18647, + "f1_weighted": 0.239147 + }, + { + "accuracy": 0.229004, + "f1": 0.192805, + "f1_weighted": 0.215194 + }, + { + "accuracy": 0.14502, + "f1": 0.138307, + "f1_weighted": 0.145199 + } + ], + "main_score": 0.197021, + "hf_subset": "default", + "languages": [ + "jpn-Jpan" + ] + } + ] + }, + "evaluation_time": 2.9531404972076416, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/Waimai.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/Waimai.json new file mode 100644 index 0000000000..1c96847312 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/Waimai.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "339287def212450dcaa9df8c22bf93e9980c7023", + "task_name": "Waimai", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.5359, + "f1": 0.519632, + "f1_weighted": 0.537242, + "ap": 0.35912, + "ap_weighted": 0.35912, + "scores_per_experiment": [ + { + "accuracy": 0.478, + "f1": 0.477245, + "f1_weighted": 0.484198, + "ap": 0.338788, + "ap_weighted": 0.338788 + }, + { + "accuracy": 0.581, + "f1": 0.535493, + "f1_weighted": 0.586379, + "ap": 0.343622, + "ap_weighted": 0.343622 + }, + { + "accuracy": 0.452, + "f1": 0.438331, + "f1_weighted": 0.407664, + "ap": 0.363186, + "ap_weighted": 0.363186 + }, + { + "accuracy": 0.518, + "f1": 0.516373, + "f1_weighted": 0.506555, + "ap": 0.385175, + "ap_weighted": 0.385175 + }, + { + "accuracy": 0.524, + "f1": 0.506942, + "f1_weighted": 0.53904, + "ap": 0.335714, + "ap_weighted": 0.335714 + }, + { + "accuracy": 0.611, + "f1": 0.561826, + "f1_weighted": 0.613202, + "ap": 0.359344, + "ap_weighted": 0.359344 + }, + { + "accuracy": 0.55, + "f1": 0.549539, + "f1_weighted": 0.554584, + "ap": 0.387, + "ap_weighted": 0.387 + }, + { + "accuracy": 0.545, + "f1": 0.539324, + "f1_weighted": 0.557221, + "ap": 0.364896, + "ap_weighted": 0.364896 + }, + { + "accuracy": 0.533, + "f1": 0.5291, + "f1_weighted": 0.544099, + "ap": 0.361329, + "ap_weighted": 0.361329 + }, + { + "accuracy": 0.567, + "f1": 0.542143, + "f1_weighted": 0.579482, + "ap": 0.352145, + "ap_weighted": 0.352145 + } + ], + "main_score": 0.5359, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 1.5524041652679443, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaBioMetChemClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaBioMetChemClassification.json new file mode 100644 index 0000000000..43ae5d3734 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaBioMetChemClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "6ac491e5de9070c6dd434b31e76d3d379123dcff", + "task_name": "WikipediaBioMetChemClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.974151, + "f1": 0.9741, + "f1_weighted": 0.974162, + "ap": 0.955041, + "ap_weighted": 0.955041, + "scores_per_experiment": [ + { + "accuracy": 0.97389, + "f1": 0.973843, + "f1_weighted": 0.973904, + "ap": 0.953314, + "ap_weighted": 0.953314 + }, + { + "accuracy": 0.984334, + "f1": 0.984287, + "f1_weighted": 0.984334, + "ap": 0.974958, + "ap_weighted": 0.974958 + }, + { + "accuracy": 0.97302, + "f1": 0.972984, + "f1_weighted": 0.973038, + "ap": 0.949469, + "ap_weighted": 0.949469 + }, + { + "accuracy": 0.981723, + "f1": 0.981663, + "f1_weighted": 0.98172, + "ap": 0.972046, + "ap_weighted": 0.972046 + }, + { + "accuracy": 0.979112, + "f1": 0.979045, + "f1_weighted": 0.97911, + "ap": 0.967527, + "ap_weighted": 0.967527 + }, + { + "accuracy": 0.967798, + "f1": 0.967742, + "f1_weighted": 0.967816, + "ap": 0.943665, + "ap_weighted": 0.943665 + }, + { + "accuracy": 0.965187, + "f1": 0.965129, + "f1_weighted": 0.965207, + "ap": 0.939362, + "ap_weighted": 0.939362 + }, + { + "accuracy": 0.976501, + "f1": 0.976453, + "f1_weighted": 0.976512, + "ap": 0.958439, + "ap_weighted": 0.958439 + }, + { + "accuracy": 0.97302, + "f1": 0.972965, + "f1_weighted": 0.973032, + "ap": 0.953079, + "ap_weighted": 0.953079 + }, + { + "accuracy": 0.966928, + "f1": 0.966893, + "f1_weighted": 0.966952, + "ap": 0.938555, + "ap_weighted": 0.938555 + } + ], + "main_score": 0.974151, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 8.43142318725586, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaBiolumNeurochemClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaBiolumNeurochemClassification.json new file mode 100644 index 0000000000..20b7212954 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaBiolumNeurochemClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "2f68b7d34c2be896e46b14533573b366e59e5aae", + "task_name": "WikipediaBiolumNeurochemClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.95, + "f1": 0.947262, + "f1_weighted": 0.949937, + "ap": 0.903863, + "ap_weighted": 0.903863, + "scores_per_experiment": [ + { + "accuracy": 0.959184, + "f1": 0.957018, + "f1_weighted": 0.959184, + "ap": 0.917915, + "ap_weighted": 0.917915 + }, + { + "accuracy": 0.928571, + "f1": 0.924408, + "f1_weighted": 0.92839, + "ap": 0.863007, + "ap_weighted": 0.863007 + }, + { + "accuracy": 0.928571, + "f1": 0.925782, + "f1_weighted": 0.929012, + "ap": 0.852244, + "ap_weighted": 0.852244 + }, + { + "accuracy": 0.938776, + "f1": 0.934869, + "f1_weighted": 0.93845, + "ap": 0.885846, + "ap_weighted": 0.885846 + }, + { + "accuracy": 0.959184, + "f1": 0.95658, + "f1_weighted": 0.958967, + "ap": 0.92608, + "ap_weighted": 0.92608 + }, + { + "accuracy": 0.969388, + "f1": 0.967257, + "f1_weighted": 0.969132, + "ap": 0.951665, + "ap_weighted": 0.951665 + }, + { + "accuracy": 0.938776, + "f1": 0.937099, + "f1_weighted": 0.939404, + "ap": 0.863636, + "ap_weighted": 0.863636 + }, + { + "accuracy": 0.969388, + "f1": 0.967603, + "f1_weighted": 0.96931, + "ap": 0.942172, + "ap_weighted": 0.942172 + }, + { + "accuracy": 0.94898, + "f1": 0.945428, + "f1_weighted": 0.948553, + "ap": 0.909989, + "ap_weighted": 0.909989 + }, + { + "accuracy": 0.959184, + "f1": 0.95658, + "f1_weighted": 0.958967, + "ap": 0.92608, + "ap_weighted": 0.92608 + } + ], + "main_score": 0.95, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.3657026290893555, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaChemEngSpecialtiesClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaChemEngSpecialtiesClassification.json new file mode 100644 index 0000000000..b7a3a43b08 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaChemEngSpecialtiesClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f81a76a2fb690e5d5bd7a26dd07e85cdf8405dfb", + "task_name": "WikipediaChemEngSpecialtiesClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.591935, + "f1": 0.575071, + "f1_weighted": 0.596979, + "scores_per_experiment": [ + { + "accuracy": 0.629032, + "f1": 0.633653, + "f1_weighted": 0.628103 + }, + { + "accuracy": 0.637097, + "f1": 0.603938, + "f1_weighted": 0.637264 + }, + { + "accuracy": 0.612903, + "f1": 0.601049, + "f1_weighted": 0.629683 + }, + { + "accuracy": 0.637097, + "f1": 0.617031, + "f1_weighted": 0.643824 + }, + { + "accuracy": 0.540323, + "f1": 0.506353, + "f1_weighted": 0.531737 + }, + { + "accuracy": 0.516129, + "f1": 0.496751, + "f1_weighted": 0.518645 + }, + { + "accuracy": 0.612903, + "f1": 0.59962, + "f1_weighted": 0.623847 + }, + { + "accuracy": 0.580645, + "f1": 0.560432, + "f1_weighted": 0.584538 + }, + { + "accuracy": 0.580645, + "f1": 0.574781, + "f1_weighted": 0.586916 + }, + { + "accuracy": 0.572581, + "f1": 0.557099, + "f1_weighted": 0.585235 + } + ], + "main_score": 0.591935, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.473850965499878, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaChemFieldsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaChemFieldsClassification.json new file mode 100644 index 0000000000..9cfef4d7c5 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaChemFieldsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "a75fae77759acc115f015f2b856baa47776d733d", + "task_name": "WikipediaChemFieldsClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.443496, + "f1": 0.373365, + "f1_weighted": 0.468615, + "scores_per_experiment": [ + { + "accuracy": 0.415174, + "f1": 0.351286, + "f1_weighted": 0.442643 + }, + { + "accuracy": 0.442571, + "f1": 0.379319, + "f1_weighted": 0.476429 + }, + { + "accuracy": 0.479686, + "f1": 0.395006, + "f1_weighted": 0.497559 + }, + { + "accuracy": 0.442571, + "f1": 0.371675, + "f1_weighted": 0.473014 + }, + { + "accuracy": 0.476993, + "f1": 0.399764, + "f1_weighted": 0.505519 + }, + { + "accuracy": 0.47793, + "f1": 0.393637, + "f1_weighted": 0.508247 + }, + { + "accuracy": 0.441049, + "f1": 0.380354, + "f1_weighted": 0.46288 + }, + { + "accuracy": 0.369863, + "f1": 0.340118, + "f1_weighted": 0.38459 + }, + { + "accuracy": 0.421379, + "f1": 0.348677, + "f1_weighted": 0.451423 + }, + { + "accuracy": 0.467744, + "f1": 0.373816, + "f1_weighted": 0.483844 + } + ], + "main_score": 0.443496, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 50.7606942653656, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaChemistryTopicsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaChemistryTopicsClassification.json new file mode 100644 index 0000000000..1f4a95665e --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaChemistryTopicsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "d8fb355db2248f95df8ea410a43aa1db1ee96ba4", + "task_name": "WikipediaChemistryTopicsClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.801425, + "f1": 0.803345, + "f1_weighted": 0.798416, + "scores_per_experiment": [ + { + "accuracy": 0.790974, + "f1": 0.792167, + "f1_weighted": 0.78809 + }, + { + "accuracy": 0.833729, + "f1": 0.834764, + "f1_weighted": 0.832617 + }, + { + "accuracy": 0.7981, + "f1": 0.798902, + "f1_weighted": 0.79421 + }, + { + "accuracy": 0.800475, + "f1": 0.805258, + "f1_weighted": 0.799178 + }, + { + "accuracy": 0.793349, + "f1": 0.799657, + "f1_weighted": 0.789913 + }, + { + "accuracy": 0.795724, + "f1": 0.797998, + "f1_weighted": 0.793151 + }, + { + "accuracy": 0.821853, + "f1": 0.82444, + "f1_weighted": 0.818615 + }, + { + "accuracy": 0.807601, + "f1": 0.812179, + "f1_weighted": 0.804915 + }, + { + "accuracy": 0.779097, + "f1": 0.774273, + "f1_weighted": 0.773041 + }, + { + "accuracy": 0.793349, + "f1": 0.793815, + "f1_weighted": 0.790432 + } + ], + "main_score": 0.801425, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 7.5917840003967285, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaCompChemSpectroscopyClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaCompChemSpectroscopyClassification.json new file mode 100644 index 0000000000..ab71bab735 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaCompChemSpectroscopyClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "474d706a22b0451b5846d623aa4b4234ba5b0513", + "task_name": "WikipediaCompChemSpectroscopyClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.695475, + "f1": 0.691429, + "f1_weighted": 0.693895, + "ap": 0.688615, + "ap_weighted": 0.688615, + "scores_per_experiment": [ + { + "accuracy": 0.60181, + "f1": 0.600427, + "f1_weighted": 0.603086, + "ap": 0.616561, + "ap_weighted": 0.616561 + }, + { + "accuracy": 0.719457, + "f1": 0.719313, + "f1_weighted": 0.718595, + "ap": 0.726459, + "ap_weighted": 0.726459 + }, + { + "accuracy": 0.656109, + "f1": 0.652975, + "f1_weighted": 0.649245, + "ap": 0.681889, + "ap_weighted": 0.681889 + }, + { + "accuracy": 0.742081, + "f1": 0.7395, + "f1_weighted": 0.742433, + "ap": 0.721722, + "ap_weighted": 0.721722 + }, + { + "accuracy": 0.778281, + "f1": 0.7768, + "f1_weighted": 0.778856, + "ap": 0.759231, + "ap_weighted": 0.759231 + }, + { + "accuracy": 0.723982, + "f1": 0.707805, + "f1_weighted": 0.715582, + "ap": 0.68592, + "ap_weighted": 0.68592 + }, + { + "accuracy": 0.674208, + "f1": 0.672269, + "f1_weighted": 0.675121, + "ap": 0.668023, + "ap_weighted": 0.668023 + }, + { + "accuracy": 0.751131, + "f1": 0.743549, + "f1_weighted": 0.748537, + "ap": 0.716974, + "ap_weighted": 0.716974 + }, + { + "accuracy": 0.633484, + "f1": 0.627505, + "f1_weighted": 0.632844, + "ap": 0.631755, + "ap_weighted": 0.631755 + }, + { + "accuracy": 0.674208, + "f1": 0.674148, + "f1_weighted": 0.674648, + "ap": 0.677614, + "ap_weighted": 0.677614 + } + ], + "main_score": 0.695475, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.933551073074341, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaCryobiologySeparationClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaCryobiologySeparationClassification.json new file mode 100644 index 0000000000..6bc97bca5b --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaCryobiologySeparationClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "858633e882dadd1ec6a0d220f7549bcafd379236", + "task_name": "WikipediaCryobiologySeparationClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.888841, + "f1": 0.888836, + "f1_weighted": 0.888221, + "scores_per_experiment": [ + { + "accuracy": 0.901288, + "f1": 0.90252, + "f1_weighted": 0.901322 + }, + { + "accuracy": 0.888412, + "f1": 0.887818, + "f1_weighted": 0.888034 + }, + { + "accuracy": 0.927039, + "f1": 0.927079, + "f1_weighted": 0.926489 + }, + { + "accuracy": 0.879828, + "f1": 0.880579, + "f1_weighted": 0.880699 + }, + { + "accuracy": 0.901288, + "f1": 0.902906, + "f1_weighted": 0.900318 + }, + { + "accuracy": 0.871245, + "f1": 0.873358, + "f1_weighted": 0.869208 + }, + { + "accuracy": 0.88412, + "f1": 0.883788, + "f1_weighted": 0.884518 + }, + { + "accuracy": 0.888412, + "f1": 0.888903, + "f1_weighted": 0.886215 + }, + { + "accuracy": 0.888412, + "f1": 0.886379, + "f1_weighted": 0.886531 + }, + { + "accuracy": 0.858369, + "f1": 0.855032, + "f1_weighted": 0.858875 + } + ], + "main_score": 0.888841, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 4.459792375564575, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaCrystallographyAnalyticalClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaCrystallographyAnalyticalClassification.json new file mode 100644 index 0000000000..ad4ced6b4c --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaCrystallographyAnalyticalClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "740565a6a853aaed1114a13bdfd5fd46857b4f11", + "task_name": "WikipediaCrystallographyAnalyticalClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.889691, + "f1": 0.889195, + "f1_weighted": 0.889925, + "ap": 0.892774, + "ap_weighted": 0.892774, + "scores_per_experiment": [ + { + "accuracy": 0.879725, + "f1": 0.877639, + "f1_weighted": 0.87956, + "ap": 0.853601, + "ap_weighted": 0.853601 + }, + { + "accuracy": 0.90378, + "f1": 0.903524, + "f1_weighted": 0.904122, + "ap": 0.911382, + "ap_weighted": 0.911382 + }, + { + "accuracy": 0.907216, + "f1": 0.90647, + "f1_weighted": 0.907475, + "ap": 0.899438, + "ap_weighted": 0.899438 + }, + { + "accuracy": 0.910653, + "f1": 0.910347, + "f1_weighted": 0.910977, + "ap": 0.916769, + "ap_weighted": 0.916769 + }, + { + "accuracy": 0.896907, + "f1": 0.896259, + "f1_weighted": 0.897245, + "ap": 0.891407, + "ap_weighted": 0.891407 + }, + { + "accuracy": 0.859107, + "f1": 0.859107, + "f1_weighted": 0.859107, + "ap": 0.879482, + "ap_weighted": 0.879482 + }, + { + "accuracy": 0.865979, + "f1": 0.865922, + "f1_weighted": 0.866255, + "ap": 0.878719, + "ap_weighted": 0.878719 + }, + { + "accuracy": 0.886598, + "f1": 0.885946, + "f1_weighted": 0.886983, + "ap": 0.880674, + "ap_weighted": 0.880674 + }, + { + "accuracy": 0.883162, + "f1": 0.88305, + "f1_weighted": 0.883485, + "ap": 0.89522, + "ap_weighted": 0.89522 + }, + { + "accuracy": 0.90378, + "f1": 0.903688, + "f1_weighted": 0.904046, + "ap": 0.921049, + "ap_weighted": 0.921049 + } + ], + "main_score": 0.889691, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.5279951095581055, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaGreenhouseEnantiopureClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaGreenhouseEnantiopureClassification.json new file mode 100644 index 0000000000..0484e04bbf --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaGreenhouseEnantiopureClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "0cfc1a83b6ed832454e8f4f93f7a0e26208274d9", + "task_name": "WikipediaGreenhouseEnantiopureClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.961842, + "f1": 0.958466, + "f1_weighted": 0.961576, + "ap": 0.952381, + "ap_weighted": 0.952381, + "scores_per_experiment": [ + { + "accuracy": 0.95614, + "f1": 0.951858, + "f1_weighted": 0.955637, + "ap": 0.938642, + "ap_weighted": 0.938642 + }, + { + "accuracy": 0.964912, + "f1": 0.961911, + "f1_weighted": 0.964725, + "ap": 0.954906, + "ap_weighted": 0.954906 + }, + { + "accuracy": 0.973684, + "f1": 0.971863, + "f1_weighted": 0.973747, + "ap": 0.976073, + "ap_weighted": 0.976073 + }, + { + "accuracy": 0.969298, + "f1": 0.966582, + "f1_weighted": 0.969089, + "ap": 0.957453, + "ap_weighted": 0.957453 + }, + { + "accuracy": 0.947368, + "f1": 0.942867, + "f1_weighted": 0.947087, + "ap": 0.937214, + "ap_weighted": 0.937214 + }, + { + "accuracy": 0.973684, + "f1": 0.971583, + "f1_weighted": 0.973616, + "ap": 0.967866, + "ap_weighted": 0.967866 + }, + { + "accuracy": 0.960526, + "f1": 0.957263, + "f1_weighted": 0.960371, + "ap": 0.952359, + "ap_weighted": 0.952359 + }, + { + "accuracy": 0.97807, + "f1": 0.976496, + "f1_weighted": 0.978097, + "ap": 0.97863, + "ap_weighted": 0.97863 + }, + { + "accuracy": 0.942982, + "f1": 0.93647, + "f1_weighted": 0.941823, + "ap": 0.917197, + "ap_weighted": 0.917197 + }, + { + "accuracy": 0.951754, + "f1": 0.947766, + "f1_weighted": 0.951564, + "ap": 0.94347, + "ap_weighted": 0.94347 + } + ], + "main_score": 0.961842, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.970672845840454, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaIsotopesFissionClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaIsotopesFissionClassification.json new file mode 100644 index 0000000000..56871698d5 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaIsotopesFissionClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "897743346c7c794264f7dbfadc3978aa2895e8e2", + "task_name": "WikipediaIsotopesFissionClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.855952, + "f1": 0.847631, + "f1_weighted": 0.859092, + "ap": 0.700981, + "ap_weighted": 0.700981, + "scores_per_experiment": [ + { + "accuracy": 0.928571, + "f1": 0.923311, + "f1_weighted": 0.930006, + "ap": 0.823529, + "ap_weighted": 0.823529 + }, + { + "accuracy": 0.821429, + "f1": 0.815547, + "f1_weighted": 0.826526, + "ap": 0.646922, + "ap_weighted": 0.646922 + }, + { + "accuracy": 0.857143, + "f1": 0.848558, + "f1_weighted": 0.860577, + "ap": 0.694444, + "ap_weighted": 0.694444 + }, + { + "accuracy": 0.833333, + "f1": 0.828571, + "f1_weighted": 0.838095, + "ap": 0.666667, + "ap_weighted": 0.666667 + }, + { + "accuracy": 0.833333, + "f1": 0.828571, + "f1_weighted": 0.838095, + "ap": 0.666667, + "ap_weighted": 0.666667 + }, + { + "accuracy": 0.833333, + "f1": 0.821059, + "f1_weighted": 0.836681, + "ap": 0.652661, + "ap_weighted": 0.652661 + }, + { + "accuracy": 0.916667, + "f1": 0.901458, + "f1_weighted": 0.914362, + "ap": 0.822981, + "ap_weighted": 0.822981 + }, + { + "accuracy": 0.833333, + "f1": 0.823317, + "f1_weighted": 0.83734, + "ap": 0.655754, + "ap_weighted": 0.655754 + }, + { + "accuracy": 0.880952, + "f1": 0.87037, + "f1_weighted": 0.882716, + "ap": 0.733259, + "ap_weighted": 0.733259 + }, + { + "accuracy": 0.821429, + "f1": 0.815547, + "f1_weighted": 0.826526, + "ap": 0.646922, + "ap_weighted": 0.646922 + } + ], + "main_score": 0.855952, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.4473893642425537, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaLuminescenceClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaLuminescenceClassification.json new file mode 100644 index 0000000000..d8e13a2d08 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaLuminescenceClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "21c4dcebe2c5b36a35292e6441e7a10b59bf4896", + "task_name": "WikipediaLuminescenceClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.841463, + "f1": 0.834229, + "f1_weighted": 0.844108, + "ap": 0.688239, + "ap_weighted": 0.688239, + "scores_per_experiment": [ + { + "accuracy": 0.914634, + "f1": 0.909791, + "f1_weighted": 0.915909, + "ap": 0.807327, + "ap_weighted": 0.807327 + }, + { + "accuracy": 0.841463, + "f1": 0.835975, + "f1_weighted": 0.844757, + "ap": 0.685915, + "ap_weighted": 0.685915 + }, + { + "accuracy": 0.865854, + "f1": 0.854399, + "f1_weighted": 0.866352, + "ap": 0.723045, + "ap_weighted": 0.723045 + }, + { + "accuracy": 0.780488, + "f1": 0.773897, + "f1_weighted": 0.785195, + "ap": 0.601389, + "ap_weighted": 0.601389 + }, + { + "accuracy": 0.841463, + "f1": 0.838558, + "f1_weighted": 0.844897, + "ap": 0.690476, + "ap_weighted": 0.690476 + }, + { + "accuracy": 0.817073, + "f1": 0.801453, + "f1_weighted": 0.817752, + "ap": 0.641688, + "ap_weighted": 0.641688 + }, + { + "accuracy": 0.792683, + "f1": 0.785505, + "f1_weighted": 0.796989, + "ap": 0.615931, + "ap_weighted": 0.615931 + }, + { + "accuracy": 0.804878, + "f1": 0.795, + "f1_weighted": 0.808171, + "ap": 0.628463, + "ap_weighted": 0.628463 + }, + { + "accuracy": 0.890244, + "f1": 0.885315, + "f1_weighted": 0.892274, + "ap": 0.763153, + "ap_weighted": 0.763153 + }, + { + "accuracy": 0.865854, + "f1": 0.862395, + "f1_weighted": 0.86878, + "ap": 0.725, + "ap_weighted": 0.725 + } + ], + "main_score": 0.841463, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.202420473098755, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaOrganicInorganicClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaOrganicInorganicClassification.json new file mode 100644 index 0000000000..daa645a777 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaOrganicInorganicClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "96d1d9b37c4693f74c46c83d63a290573f78d511", + "task_name": "WikipediaOrganicInorganicClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.827376, + "f1": 0.82483, + "f1_weighted": 0.827059, + "ap": 0.731224, + "ap_weighted": 0.731224, + "scores_per_experiment": [ + { + "accuracy": 0.836502, + "f1": 0.828144, + "f1_weighted": 0.832899, + "ap": 0.763191, + "ap_weighted": 0.763191 + }, + { + "accuracy": 0.847909, + "f1": 0.846289, + "f1_weighted": 0.848269, + "ap": 0.754829, + "ap_weighted": 0.754829 + }, + { + "accuracy": 0.8327, + "f1": 0.830342, + "f1_weighted": 0.832852, + "ap": 0.736555, + "ap_weighted": 0.736555 + }, + { + "accuracy": 0.764259, + "f1": 0.764255, + "f1_weighted": 0.764368, + "ap": 0.648575, + "ap_weighted": 0.648575 + }, + { + "accuracy": 0.828897, + "f1": 0.82442, + "f1_weighted": 0.827938, + "ap": 0.737936, + "ap_weighted": 0.737936 + }, + { + "accuracy": 0.847909, + "f1": 0.846289, + "f1_weighted": 0.848269, + "ap": 0.754829, + "ap_weighted": 0.754829 + }, + { + "accuracy": 0.828897, + "f1": 0.823498, + "f1_weighted": 0.827371, + "ap": 0.740786, + "ap_weighted": 0.740786 + }, + { + "accuracy": 0.840304, + "f1": 0.839783, + "f1_weighted": 0.84093, + "ap": 0.739253, + "ap_weighted": 0.739253 + }, + { + "accuracy": 0.836502, + "f1": 0.835732, + "f1_weighted": 0.837143, + "ap": 0.735573, + "ap_weighted": 0.735573 + }, + { + "accuracy": 0.809886, + "f1": 0.809553, + "f1_weighted": 0.810552, + "ap": 0.700709, + "ap_weighted": 0.700709 + } + ], + "main_score": 0.827376, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.9607889652252197, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaSaltsSemiconductorsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaSaltsSemiconductorsClassification.json new file mode 100644 index 0000000000..dcaae1f1f2 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaSaltsSemiconductorsClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "9e5415a096012fa2d1f3a929952cf9859e4550e7", + "task_name": "WikipediaSaltsSemiconductorsClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.861616, + "f1": 0.86096, + "f1_weighted": 0.861016, + "ap": 0.807368, + "ap_weighted": 0.807368, + "scores_per_experiment": [ + { + "accuracy": 0.848485, + "f1": 0.846923, + "f1_weighted": 0.847079, + "ap": 0.779117, + "ap_weighted": 0.779117 + }, + { + "accuracy": 0.848485, + "f1": 0.848423, + "f1_weighted": 0.848392, + "ap": 0.806228, + "ap_weighted": 0.806228 + }, + { + "accuracy": 0.878788, + "f1": 0.878776, + "f1_weighted": 0.878788, + "ap": 0.835006, + "ap_weighted": 0.835006 + }, + { + "accuracy": 0.888889, + "f1": 0.888159, + "f1_weighted": 0.88825, + "ap": 0.828623, + "ap_weighted": 0.828623 + }, + { + "accuracy": 0.89899, + "f1": 0.898732, + "f1_weighted": 0.898783, + "ap": 0.848451, + "ap_weighted": 0.848451 + }, + { + "accuracy": 0.858586, + "f1": 0.857407, + "f1_weighted": 0.857538, + "ap": 0.792027, + "ap_weighted": 0.792027 + }, + { + "accuracy": 0.848485, + "f1": 0.846923, + "f1_weighted": 0.847079, + "ap": 0.779117, + "ap_weighted": 0.779117 + }, + { + "accuracy": 0.868687, + "f1": 0.868687, + "f1_weighted": 0.868687, + "ap": 0.825401, + "ap_weighted": 0.825401 + }, + { + "accuracy": 0.828283, + "f1": 0.82765, + "f1_weighted": 0.827544, + "ap": 0.79284, + "ap_weighted": 0.79284 + }, + { + "accuracy": 0.848485, + "f1": 0.847926, + "f1_weighted": 0.848019, + "ap": 0.786869, + "ap_weighted": 0.786869 + } + ], + "main_score": 0.861616, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.2611050605773926, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaSolidStateColloidalClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaSolidStateColloidalClassification.json new file mode 100644 index 0000000000..bd0708836e --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaSolidStateColloidalClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "7d8df44e588b6143d4856c781f72f919fa0599a7", + "task_name": "WikipediaSolidStateColloidalClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.724099, + "f1": 0.719892, + "f1_weighted": 0.720906, + "ap": 0.711773, + "ap_weighted": 0.711773, + "scores_per_experiment": [ + { + "accuracy": 0.722973, + "f1": 0.720098, + "f1_weighted": 0.717159, + "ap": 0.745735, + "ap_weighted": 0.745735 + }, + { + "accuracy": 0.601351, + "f1": 0.595139, + "f1_weighted": 0.589943, + "ap": 0.631544, + "ap_weighted": 0.631544 + }, + { + "accuracy": 0.797297, + "f1": 0.793266, + "f1_weighted": 0.796257, + "ap": 0.759915, + "ap_weighted": 0.759915 + }, + { + "accuracy": 0.777027, + "f1": 0.765851, + "f1_weighted": 0.771151, + "ap": 0.727537, + "ap_weighted": 0.727537 + }, + { + "accuracy": 0.677928, + "f1": 0.677848, + "f1_weighted": 0.677322, + "ap": 0.680096, + "ap_weighted": 0.680096 + }, + { + "accuracy": 0.765766, + "f1": 0.765723, + "f1_weighted": 0.766051, + "ap": 0.758061, + "ap_weighted": 0.758061 + }, + { + "accuracy": 0.740991, + "f1": 0.728565, + "f1_weighted": 0.734582, + "ap": 0.698287, + "ap_weighted": 0.698287 + }, + { + "accuracy": 0.738739, + "f1": 0.738654, + "f1_weighted": 0.739142, + "ap": 0.729945, + "ap_weighted": 0.729945 + }, + { + "accuracy": 0.668919, + "f1": 0.668917, + "f1_weighted": 0.66884, + "ap": 0.670004, + "ap_weighted": 0.670004 + }, + { + "accuracy": 0.75, + "f1": 0.744863, + "f1_weighted": 0.748614, + "ap": 0.716605, + "ap_weighted": 0.716605 + } + ], + "main_score": 0.724099, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 4.011911630630493, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaTheoreticalAppliedClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaTheoreticalAppliedClassification.json new file mode 100644 index 0000000000..c939ca7892 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/WikipediaTheoreticalAppliedClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "7896906653d31d7102a143d7f55d67cd688e3147", + "task_name": "WikipediaTheoreticalAppliedClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.606872, + "f1": 0.603674, + "f1_weighted": 0.605079, + "ap": 0.537275, + "ap_weighted": 0.537275, + "scores_per_experiment": [ + { + "accuracy": 0.656127, + "f1": 0.655112, + "f1_weighted": 0.656189, + "ap": 0.575282, + "ap_weighted": 0.575282 + }, + { + "accuracy": 0.598458, + "f1": 0.598167, + "f1_weighted": 0.598789, + "ap": 0.530761, + "ap_weighted": 0.530761 + }, + { + "accuracy": 0.63359, + "f1": 0.627699, + "f1_weighted": 0.630396, + "ap": 0.556782, + "ap_weighted": 0.556782 + }, + { + "accuracy": 0.623993, + "f1": 0.615821, + "f1_weighted": 0.619048, + "ap": 0.548882, + "ap_weighted": 0.548882 + }, + { + "accuracy": 0.598458, + "f1": 0.59812, + "f1_weighted": 0.59745, + "ap": 0.531954, + "ap_weighted": 0.531954 + }, + { + "accuracy": 0.626735, + "f1": 0.626644, + "f1_weighted": 0.62698, + "ap": 0.55176, + "ap_weighted": 0.55176 + }, + { + "accuracy": 0.601457, + "f1": 0.601421, + "f1_weighted": 0.601202, + "ap": 0.533631, + "ap_weighted": 0.533631 + }, + { + "accuracy": 0.56024, + "f1": 0.557972, + "f1_weighted": 0.559795, + "ap": 0.503968, + "ap_weighted": 0.503968 + }, + { + "accuracy": 0.577721, + "f1": 0.577229, + "f1_weighted": 0.578059, + "ap": 0.516343, + "ap_weighted": 0.516343 + }, + { + "accuracy": 0.591945, + "f1": 0.578557, + "f1_weighted": 0.582882, + "ap": 0.52339, + "ap_weighted": 0.52339 + } + ], + "main_score": 0.606872, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 63.82870292663574, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/WisesightSentimentClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/WisesightSentimentClassification.json new file mode 100644 index 0000000000..9acc3854aa --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/WisesightSentimentClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "14aa5773afa135ba835cc5179bbc4a63657a42ae", + "task_name": "WisesightSentimentClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.279004, + "f1": 0.233548, + "f1_weighted": 0.276181, + "scores_per_experiment": [ + { + "accuracy": 0.260742, + "f1": 0.222157, + "f1_weighted": 0.209149 + }, + { + "accuracy": 0.283691, + "f1": 0.256911, + "f1_weighted": 0.276454 + }, + { + "accuracy": 0.32959, + "f1": 0.237919, + "f1_weighted": 0.363552 + }, + { + "accuracy": 0.347168, + "f1": 0.272023, + "f1_weighted": 0.374104 + }, + { + "accuracy": 0.272949, + "f1": 0.213997, + "f1_weighted": 0.312723 + }, + { + "accuracy": 0.260742, + "f1": 0.225368, + "f1_weighted": 0.291417 + }, + { + "accuracy": 0.213379, + "f1": 0.201975, + "f1_weighted": 0.174918 + }, + { + "accuracy": 0.273926, + "f1": 0.228995, + "f1_weighted": 0.219901 + }, + { + "accuracy": 0.292969, + "f1": 0.253429, + "f1_weighted": 0.278915 + }, + { + "accuracy": 0.254883, + "f1": 0.222707, + "f1_weighted": 0.26068 + } + ], + "main_score": 0.233548, + "hf_subset": "default", + "languages": [ + "tha-Thai" + ] + } + ] + }, + "evaluation_time": 3.510394334793091, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/YahooAnswersTopicsClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/YahooAnswersTopicsClassification.json new file mode 100644 index 0000000000..74ba351c0b --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/YahooAnswersTopicsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "78fccffa043240c80e17a6b1da724f5a1057e8e5", + "task_name": "YahooAnswersTopicsClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.531152, + "f1": 0.523875, + "f1_weighted": 0.523973, + "scores_per_experiment": [ + { + "accuracy": 0.530273, + "f1": 0.521625, + "f1_weighted": 0.521713 + }, + { + "accuracy": 0.546387, + "f1": 0.541317, + "f1_weighted": 0.5414 + }, + { + "accuracy": 0.519531, + "f1": 0.510264, + "f1_weighted": 0.510389 + }, + { + "accuracy": 0.523438, + "f1": 0.515427, + "f1_weighted": 0.515509 + }, + { + "accuracy": 0.519531, + "f1": 0.514148, + "f1_weighted": 0.514256 + }, + { + "accuracy": 0.53125, + "f1": 0.526254, + "f1_weighted": 0.526292 + }, + { + "accuracy": 0.541016, + "f1": 0.538889, + "f1_weighted": 0.539012 + }, + { + "accuracy": 0.528809, + "f1": 0.516464, + "f1_weighted": 0.51659 + }, + { + "accuracy": 0.552734, + "f1": 0.544466, + "f1_weighted": 0.544556 + }, + { + "accuracy": 0.518555, + "f1": 0.509899, + "f1_weighted": 0.51001 + } + ], + "main_score": 0.531152, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 23.603565454483032, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/YelpReviewFullClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/YelpReviewFullClassification.json new file mode 100644 index 0000000000..fb95500063 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/YelpReviewFullClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "c1f9ee939b7d05667af864ee1cb066393154bf85", + "task_name": "YelpReviewFullClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.468799, + "f1": 0.460714, + "f1_weighted": 0.460682, + "scores_per_experiment": [ + { + "accuracy": 0.462402, + "f1": 0.454581, + "f1_weighted": 0.454534 + }, + { + "accuracy": 0.464355, + "f1": 0.456287, + "f1_weighted": 0.456267 + }, + { + "accuracy": 0.483887, + "f1": 0.477891, + "f1_weighted": 0.477842 + }, + { + "accuracy": 0.462891, + "f1": 0.453983, + "f1_weighted": 0.453941 + }, + { + "accuracy": 0.486328, + "f1": 0.476395, + "f1_weighted": 0.476368 + }, + { + "accuracy": 0.470703, + "f1": 0.461763, + "f1_weighted": 0.461727 + }, + { + "accuracy": 0.45166, + "f1": 0.441565, + "f1_weighted": 0.441523 + }, + { + "accuracy": 0.468262, + "f1": 0.457463, + "f1_weighted": 0.457435 + }, + { + "accuracy": 0.46875, + "f1": 0.464434, + "f1_weighted": 0.464414 + }, + { + "accuracy": 0.46875, + "f1": 0.46278, + "f1_weighted": 0.46277 + } + ], + "main_score": 0.468799, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 58.09208106994629, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/YueOpenriceReviewClassification.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/YueOpenriceReviewClassification.json new file mode 100644 index 0000000000..2c6be75ed9 --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/YueOpenriceReviewClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "1300d045cf983bac23faadf3aa12a619624769da", + "task_name": "YueOpenriceReviewClassification", + "mteb_version": "1.34.14", + "scores": { + "test": [ + { + "accuracy": 0.22998, + "f1": 0.187292, + "f1_weighted": 0.255896, + "scores_per_experiment": [ + { + "accuracy": 0.217285, + "f1": 0.19132, + "f1_weighted": 0.244795 + }, + { + "accuracy": 0.196777, + "f1": 0.178944, + "f1_weighted": 0.228978 + }, + { + "accuracy": 0.300293, + "f1": 0.214256, + "f1_weighted": 0.324204 + }, + { + "accuracy": 0.196289, + "f1": 0.179033, + "f1_weighted": 0.203512 + }, + { + "accuracy": 0.230469, + "f1": 0.179028, + "f1_weighted": 0.269426 + }, + { + "accuracy": 0.212402, + "f1": 0.182599, + "f1_weighted": 0.232937 + }, + { + "accuracy": 0.236328, + "f1": 0.190792, + "f1_weighted": 0.264053 + }, + { + "accuracy": 0.186035, + "f1": 0.167101, + "f1_weighted": 0.202676 + }, + { + "accuracy": 0.270996, + "f1": 0.19813, + "f1_weighted": 0.303828 + }, + { + "accuracy": 0.25293, + "f1": 0.191716, + "f1_weighted": 0.284552 + } + ], + "main_score": 0.22998, + "hf_subset": "default", + "languages": [ + "yue-Hant" + ] + } + ] + }, + "evaluation_time": 7.461560487747192, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__gtr-t5-base/no_revision_available/model_meta.json b/results/sentence-transformers__gtr-t5-base/no_revision_available/model_meta.json new file mode 100644 index 0000000000..9462d7c16a --- /dev/null +++ b/results/sentence-transformers__gtr-t5-base/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "sentence-transformers/gtr-t5-base", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage_mb": null, "max_tokens": null, "embed_dim": null, "license": "apache-2.0", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["PyTorch", "Sentence Transformers"], "reference": null, "similarity_fn_name": null, "use_instructions": null, "training_datasets": null, "adapted_from": null, "superseded_by": null, "modalities": ["text"], "loader": null} \ No newline at end of file