diff --git a/results/BAAI__bge-base-en-v1.5/a5beb1e3e68b9ab74eb54cfd186867f64f240e1a/BuiltBenchClusteringP2P.json b/results/BAAI__bge-base-en-v1.5/a5beb1e3e68b9ab74eb54cfd186867f64f240e1a/BuiltBenchClusteringP2P.json new file mode 100644 index 000000000..6ffd883b1 --- /dev/null +++ b/results/BAAI__bge-base-en-v1.5/a5beb1e3e68b9ab74eb54cfd186867f64f240e1a/BuiltBenchClusteringP2P.json @@ -0,0 +1,42 @@ +{ + "dataset_revision": "919bb71053e9de62a68998161ce4f0cee8f786fb", + "task_name": "BuiltBenchClusteringP2P", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.517757, + "v_measure_std": 0.158396, + "v_measures": [ + 0.388885, + 0.517974, + 0.461875, + 0.342251, + 0.389993, + 0.517478, + 0.549504, + 0.523524, + 0.732158, + 0.08921, + 0.635176, + 0.799711, + 0.633224, + 0.46089, + 0.409979, + 0.495442, + 0.531105, + 0.582254, + 0.498222, + 0.796294 + ], + "main_score": 0.517757, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 30.256697416305542, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/BAAI__bge-base-en-v1.5/a5beb1e3e68b9ab74eb54cfd186867f64f240e1a/BuiltBenchClusteringS2S.json b/results/BAAI__bge-base-en-v1.5/a5beb1e3e68b9ab74eb54cfd186867f64f240e1a/BuiltBenchClusteringS2S.json new file mode 100644 index 000000000..8787c0606 --- /dev/null +++ b/results/BAAI__bge-base-en-v1.5/a5beb1e3e68b9ab74eb54cfd186867f64f240e1a/BuiltBenchClusteringS2S.json @@ -0,0 +1,40 @@ +{ + "dataset_revision": "1aaeb2ece89ea0a8c64e215c95c4cfaf7e891149", + "task_name": "BuiltBenchClusteringS2S", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.430005, + "v_measure_std": 0.103333, + "v_measures": [ + 0.461463, + 0.547957, + 0.400753, + 0.367827, + 0.288828, + 0.443991, + 0.424877, + 0.560499, + 0.389418, + 0.440851, + 0.470388, + 0.608706, + 0.54972, + 0.48226, + 0.241198, + 0.287907, + 0.278523, + 0.49492 + ], + "main_score": 0.430005, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 5.229267597198486, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/BAAI__bge-base-en-v1.5/a5beb1e3e68b9ab74eb54cfd186867f64f240e1a/BuiltBenchReranking.json b/results/BAAI__bge-base-en-v1.5/a5beb1e3e68b9ab74eb54cfd186867f64f240e1a/BuiltBenchReranking.json new file mode 100644 index 000000000..2776173d2 --- /dev/null +++ b/results/BAAI__bge-base-en-v1.5/a5beb1e3e68b9ab74eb54cfd186867f64f240e1a/BuiltBenchReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "fd33b0b3454deb256be06a57e8147b32ba078ff9", + "task_name": "BuiltBenchReranking", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "map": 0.633789, + "mrr": 0.800539, + "nAUC_map_max": 0.385108, + "nAUC_map_std": 0.61852, + "nAUC_map_diff1": 0.033195, + "nAUC_mrr_max": 0.368781, + "nAUC_mrr_std": 0.489594, + "nAUC_mrr_diff1": 0.137947, + "main_score": 0.633789, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 18.513643503189087, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/BAAI__bge-base-en-v1.5/a5beb1e3e68b9ab74eb54cfd186867f64f240e1a/BuiltBenchRetrieval.json b/results/BAAI__bge-base-en-v1.5/a5beb1e3e68b9ab74eb54cfd186867f64f240e1a/BuiltBenchRetrieval.json new file mode 100644 index 000000000..e2a6b76c7 --- /dev/null +++ b/results/BAAI__bge-base-en-v1.5/a5beb1e3e68b9ab74eb54cfd186867f64f240e1a/BuiltBenchRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ae611238a58dae85f3130563fe9f9e995444a8d6", + "task_name": "BuiltBenchRetrieval", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.56886, + "ndcg_at_3": 0.58011, + "ndcg_at_5": 0.59022, + "ndcg_at_10": 0.61648, + "ndcg_at_20": 0.6495, + "ndcg_at_100": 0.70197, + "ndcg_at_1000": 0.72556, + "map_at_1": 0.28683, + "map_at_3": 0.40405, + "map_at_5": 0.44666, + "map_at_10": 0.49715, + "map_at_20": 0.53351, + "map_at_100": 0.56928, + "map_at_1000": 0.57555, + "recall_at_1": 0.28683, + "recall_at_3": 0.44487, + "recall_at_5": 0.52117, + "recall_at_10": 0.63682, + "recall_at_20": 0.7531, + "recall_at_100": 0.91812, + "recall_at_1000": 0.99875, + "precision_at_1": 0.56886, + "precision_at_3": 0.41018, + "precision_at_5": 0.33832, + "precision_at_10": 0.25, + "precision_at_20": 0.17979, + "precision_at_100": 0.06186, + "precision_at_1000": 0.0082, + "mrr_at_1": 0.568862, + "mrr_at_3": 0.641717, + "mrr_at_5": 0.655639, + "mrr_at_10": 0.664422, + "mrr_at_20": 0.668248, + "mrr_at_100": 0.669901, + "mrr_at_1000": 0.670052, + "nauc_ndcg_at_1_max": 0.657615, + "nauc_ndcg_at_1_std": -0.042279, + "nauc_ndcg_at_1_diff1": 0.460099, + "nauc_ndcg_at_3_max": 0.614084, + "nauc_ndcg_at_3_std": -0.076575, + "nauc_ndcg_at_3_diff1": 0.39682, + "nauc_ndcg_at_5_max": 0.610741, + "nauc_ndcg_at_5_std": -0.092122, + "nauc_ndcg_at_5_diff1": 0.417567, + "nauc_ndcg_at_10_max": 0.624393, + "nauc_ndcg_at_10_std": -0.077477, + "nauc_ndcg_at_10_diff1": 0.40211, + "nauc_ndcg_at_20_max": 0.634024, + "nauc_ndcg_at_20_std": -0.047231, + "nauc_ndcg_at_20_diff1": 0.416927, + "nauc_ndcg_at_100_max": 0.641933, + "nauc_ndcg_at_100_std": 0.002091, + "nauc_ndcg_at_100_diff1": 0.421078, + "nauc_ndcg_at_1000_max": 0.62616, + "nauc_ndcg_at_1000_std": -0.029943, + "nauc_ndcg_at_1000_diff1": 0.408794, + "nauc_map_at_1_max": 0.46745, + "nauc_map_at_1_std": -0.181416, + "nauc_map_at_1_diff1": 0.54144, + "nauc_map_at_3_max": 0.551756, + "nauc_map_at_3_std": -0.170018, + "nauc_map_at_3_diff1": 0.494867, + "nauc_map_at_5_max": 0.575965, + "nauc_map_at_5_std": -0.156605, + "nauc_map_at_5_diff1": 0.483961, + "nauc_map_at_10_max": 0.600074, + "nauc_map_at_10_std": -0.126368, + "nauc_map_at_10_diff1": 0.453, + "nauc_map_at_20_max": 0.619331, + "nauc_map_at_20_std": -0.100292, + "nauc_map_at_20_diff1": 0.447259, + "nauc_map_at_100_max": 0.632346, + "nauc_map_at_100_std": -0.061719, + "nauc_map_at_100_diff1": 0.434954, + "nauc_map_at_1000_max": 0.627738, + "nauc_map_at_1000_std": -0.063215, + "nauc_map_at_1000_diff1": 0.430271, + "nauc_recall_at_1_max": 0.46745, + "nauc_recall_at_1_std": -0.181416, + "nauc_recall_at_1_diff1": 0.54144, + "nauc_recall_at_3_max": 0.506602, + "nauc_recall_at_3_std": -0.184331, + "nauc_recall_at_3_diff1": 0.45311, + "nauc_recall_at_5_max": 0.52077, + "nauc_recall_at_5_std": -0.176895, + "nauc_recall_at_5_diff1": 0.447712, + "nauc_recall_at_10_max": 0.515653, + "nauc_recall_at_10_std": -0.124786, + "nauc_recall_at_10_diff1": 0.371061, + "nauc_recall_at_20_max": 0.534895, + "nauc_recall_at_20_std": -0.040026, + "nauc_recall_at_20_diff1": 0.368425, + "nauc_recall_at_100_max": 0.639816, + "nauc_recall_at_100_std": 0.308272, + "nauc_recall_at_100_diff1": 0.395279, + "nauc_recall_at_1000_max": 0.896105, + "nauc_recall_at_1000_std": 0.781033, + "nauc_recall_at_1000_diff1": 0.267489, + "nauc_precision_at_1_max": 0.657615, + "nauc_precision_at_1_std": -0.042279, + "nauc_precision_at_1_diff1": 0.460099, + "nauc_precision_at_3_max": 0.364233, + "nauc_precision_at_3_std": 0.074636, + "nauc_precision_at_3_diff1": 0.02891, + "nauc_precision_at_5_max": 0.236649, + "nauc_precision_at_5_std": 0.108568, + "nauc_precision_at_5_diff1": -0.066509, + "nauc_precision_at_10_max": 0.122793, + "nauc_precision_at_10_std": 0.158997, + "nauc_precision_at_10_diff1": -0.175302, + "nauc_precision_at_20_max": 0.029816, + "nauc_precision_at_20_std": 0.199538, + "nauc_precision_at_20_diff1": -0.201348, + "nauc_precision_at_100_max": -0.116511, + "nauc_precision_at_100_std": 0.214691, + "nauc_precision_at_100_diff1": -0.239711, + "nauc_precision_at_1000_max": -0.224998, + "nauc_precision_at_1000_std": 0.082654, + "nauc_precision_at_1000_diff1": -0.27228, + "nauc_mrr_at_1_max": 0.657615, + "nauc_mrr_at_1_std": -0.042279, + "nauc_mrr_at_1_diff1": 0.460099, + "nauc_mrr_at_3_max": 0.647651, + "nauc_mrr_at_3_std": -0.031469, + "nauc_mrr_at_3_diff1": 0.421722, + "nauc_mrr_at_5_max": 0.653044, + "nauc_mrr_at_5_std": -0.03276, + "nauc_mrr_at_5_diff1": 0.427097, + "nauc_mrr_at_10_max": 0.651649, + "nauc_mrr_at_10_std": -0.025527, + "nauc_mrr_at_10_diff1": 0.425706, + "nauc_mrr_at_20_max": 0.650775, + "nauc_mrr_at_20_std": -0.024984, + "nauc_mrr_at_20_diff1": 0.425565, + "nauc_mrr_at_100_max": 0.651302, + "nauc_mrr_at_100_std": -0.026192, + "nauc_mrr_at_100_diff1": 0.42767, + "nauc_mrr_at_1000_max": 0.651396, + "nauc_mrr_at_1000_std": -0.02623, + "nauc_mrr_at_1000_diff1": 0.42765, + "main_score": 0.61648, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 21.189420461654663, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/BAAI__bge-base-en-v1.5/a5beb1e3e68b9ab74eb54cfd186867f64f240e1a/model_meta.json b/results/BAAI__bge-base-en-v1.5/a5beb1e3e68b9ab74eb54cfd186867f64f240e1a/model_meta.json index 13dbc26e2..7e6d4cdba 100644 --- a/results/BAAI__bge-base-en-v1.5/a5beb1e3e68b9ab74eb54cfd186867f64f240e1a/model_meta.json +++ b/results/BAAI__bge-base-en-v1.5/a5beb1e3e68b9ab74eb54cfd186867f64f240e1a/model_meta.json @@ -1 +1 @@ -{"name": "BAAI/bge-base-en-v1.5", "revision": "a5beb1e3e68b9ab74eb54cfd186867f64f240e1a", "release_date": "2023-09-11", "languages": ["eng_Latn"], "n_parameters": 438000000, "max_tokens": 512.0, "embed_dim": 768, "license": "mit", "open_weights": true, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/BAAI/bge-base-en-v1.5", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": {"NQ": ["test"], "NQHardNegatives": ["test"], "AmazonReviewsClassification": ["validation", "test"], "MLQARetrieval": ["validation", "test"]}, "adapted_from": null, "superseded_by": null, "loader": "sentence_transformers_loader"} \ No newline at end of file +{"name": "BAAI/bge-base-en-v1.5", "revision": "a5beb1e3e68b9ab74eb54cfd186867f64f240e1a", "release_date": "2023-09-11", "languages": ["eng_Latn"], "n_parameters": 109000000, "max_tokens": 512.0, "embed_dim": 768, "license": "mit", "open_weights": true, "public_training_code": null, "public_training_data": "https://data.baai.ac.cn/details/BAAI-MTP", "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/BAAI/bge-base-en-v1.5", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": {"NQ": ["test"], "NQ-NL": ["test"], "NQHardNegatives": ["test"], "AmazonReviewsClassification": ["validation", "test"], "MLQARetrieval": ["validation", "test"]}, "adapted_from": null, "superseded_by": null, "modalities": ["text"], "loader": "sentence_transformers_loader"} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/BuiltBenchClusteringP2P.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/BuiltBenchClusteringP2P.json new file mode 100644 index 000000000..9459eecbc --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/BuiltBenchClusteringP2P.json @@ -0,0 +1,42 @@ +{ + "dataset_revision": "919bb71053e9de62a68998161ce4f0cee8f786fb", + "task_name": "BuiltBenchClusteringP2P", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.524083, + "v_measure_std": 0.131864, + "v_measures": [ + 0.363593, + 0.502413, + 0.45769, + 0.468297, + 0.386542, + 0.404564, + 0.623353, + 0.675208, + 0.706091, + 0.386532, + 0.490226, + 0.799711, + 0.537188, + 0.424919, + 0.35574, + 0.510183, + 0.73475, + 0.403612, + 0.603789, + 0.647252 + ], + "main_score": 0.524083, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 94.42164897918701, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/BuiltBenchClusteringS2S.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/BuiltBenchClusteringS2S.json new file mode 100644 index 000000000..e235ec269 --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/BuiltBenchClusteringS2S.json @@ -0,0 +1,40 @@ +{ + "dataset_revision": "1aaeb2ece89ea0a8c64e215c95c4cfaf7e891149", + "task_name": "BuiltBenchClusteringS2S", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.466924, + "v_measure_std": 0.126581, + "v_measures": [ + 0.301491, + 0.545938, + 0.460741, + 0.528819, + 0.341869, + 0.437427, + 0.380774, + 0.589232, + 0.499215, + 0.655186, + 0.575787, + 0.741065, + 0.538618, + 0.473277, + 0.253986, + 0.32315, + 0.348168, + 0.409886 + ], + "main_score": 0.466924, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 14.017496347427368, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/BuiltBenchReranking.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/BuiltBenchReranking.json new file mode 100644 index 000000000..e6802445a --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/BuiltBenchReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "fd33b0b3454deb256be06a57e8147b32ba078ff9", + "task_name": "BuiltBenchReranking", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "map": 0.654691, + "mrr": 0.851443, + "nAUC_map_max": 0.415067, + "nAUC_map_std": 0.442405, + "nAUC_map_diff1": 0.053061, + "nAUC_mrr_max": 0.301693, + "nAUC_mrr_std": 0.138653, + "nAUC_mrr_diff1": 0.11613, + "main_score": 0.654691, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 59.27251124382019, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/BuiltBenchRetrieval.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/BuiltBenchRetrieval.json new file mode 100644 index 000000000..162feb70a --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/BuiltBenchRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ae611238a58dae85f3130563fe9f9e995444a8d6", + "task_name": "BuiltBenchRetrieval", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.60479, + "ndcg_at_3": 0.60583, + "ndcg_at_5": 0.62223, + "ndcg_at_10": 0.64857, + "ndcg_at_20": 0.67528, + "ndcg_at_100": 0.72426, + "ndcg_at_1000": 0.74479, + "map_at_1": 0.29428, + "map_at_3": 0.41934, + "map_at_5": 0.47049, + "map_at_10": 0.52303, + "map_at_20": 0.55881, + "map_at_100": 0.59539, + "map_at_1000": 0.60156, + "recall_at_1": 0.29428, + "recall_at_3": 0.46678, + "recall_at_5": 0.5585, + "recall_at_10": 0.66605, + "recall_at_20": 0.76703, + "recall_at_100": 0.92691, + "recall_at_1000": 0.99742, + "precision_at_1": 0.60479, + "precision_at_3": 0.42914, + "precision_at_5": 0.35868, + "precision_at_10": 0.26886, + "precision_at_20": 0.18802, + "precision_at_100": 0.06383, + "precision_at_1000": 0.00819, + "mrr_at_1": 0.60479, + "mrr_at_3": 0.669661, + "mrr_at_5": 0.683433, + "mrr_at_10": 0.692926, + "mrr_at_20": 0.696181, + "mrr_at_100": 0.697653, + "mrr_at_1000": 0.697744, + "nauc_ndcg_at_1_max": 0.599305, + "nauc_ndcg_at_1_std": -0.04036, + "nauc_ndcg_at_1_diff1": 0.479754, + "nauc_ndcg_at_3_max": 0.568179, + "nauc_ndcg_at_3_std": -0.05839, + "nauc_ndcg_at_3_diff1": 0.423663, + "nauc_ndcg_at_5_max": 0.567487, + "nauc_ndcg_at_5_std": -0.069642, + "nauc_ndcg_at_5_diff1": 0.424045, + "nauc_ndcg_at_10_max": 0.588065, + "nauc_ndcg_at_10_std": -0.036427, + "nauc_ndcg_at_10_diff1": 0.434286, + "nauc_ndcg_at_20_max": 0.602274, + "nauc_ndcg_at_20_std": -0.001325, + "nauc_ndcg_at_20_diff1": 0.42464, + "nauc_ndcg_at_100_max": 0.600154, + "nauc_ndcg_at_100_std": 0.022748, + "nauc_ndcg_at_100_diff1": 0.427486, + "nauc_ndcg_at_1000_max": 0.590961, + "nauc_ndcg_at_1000_std": -0.004749, + "nauc_ndcg_at_1000_diff1": 0.424436, + "nauc_map_at_1_max": 0.420692, + "nauc_map_at_1_std": -0.166349, + "nauc_map_at_1_diff1": 0.562895, + "nauc_map_at_3_max": 0.481369, + "nauc_map_at_3_std": -0.150308, + "nauc_map_at_3_diff1": 0.503783, + "nauc_map_at_5_max": 0.523794, + "nauc_map_at_5_std": -0.131382, + "nauc_map_at_5_diff1": 0.486395, + "nauc_map_at_10_max": 0.555321, + "nauc_map_at_10_std": -0.097244, + "nauc_map_at_10_diff1": 0.468395, + "nauc_map_at_20_max": 0.573508, + "nauc_map_at_20_std": -0.063843, + "nauc_map_at_20_diff1": 0.448707, + "nauc_map_at_100_max": 0.590798, + "nauc_map_at_100_std": -0.026107, + "nauc_map_at_100_diff1": 0.433538, + "nauc_map_at_1000_max": 0.587422, + "nauc_map_at_1000_std": -0.026679, + "nauc_map_at_1000_diff1": 0.430607, + "nauc_recall_at_1_max": 0.420692, + "nauc_recall_at_1_std": -0.166349, + "nauc_recall_at_1_diff1": 0.562895, + "nauc_recall_at_3_max": 0.44097, + "nauc_recall_at_3_std": -0.141405, + "nauc_recall_at_3_diff1": 0.470031, + "nauc_recall_at_5_max": 0.463625, + "nauc_recall_at_5_std": -0.135154, + "nauc_recall_at_5_diff1": 0.446144, + "nauc_recall_at_10_max": 0.495685, + "nauc_recall_at_10_std": -0.075233, + "nauc_recall_at_10_diff1": 0.424931, + "nauc_recall_at_20_max": 0.513597, + "nauc_recall_at_20_std": 0.01953, + "nauc_recall_at_20_diff1": 0.382207, + "nauc_recall_at_100_max": 0.579129, + "nauc_recall_at_100_std": 0.324678, + "nauc_recall_at_100_diff1": 0.343282, + "nauc_recall_at_1000_max": 0.252225, + "nauc_recall_at_1000_std": 0.532362, + "nauc_recall_at_1000_diff1": -0.153418, + "nauc_precision_at_1_max": 0.599305, + "nauc_precision_at_1_std": -0.04036, + "nauc_precision_at_1_diff1": 0.479754, + "nauc_precision_at_3_max": 0.315553, + "nauc_precision_at_3_std": 0.065471, + "nauc_precision_at_3_diff1": 0.009594, + "nauc_precision_at_5_max": 0.22782, + "nauc_precision_at_5_std": 0.114951, + "nauc_precision_at_5_diff1": -0.101153, + "nauc_precision_at_10_max": 0.102695, + "nauc_precision_at_10_std": 0.173394, + "nauc_precision_at_10_diff1": -0.183023, + "nauc_precision_at_20_max": 0.026041, + "nauc_precision_at_20_std": 0.219388, + "nauc_precision_at_20_diff1": -0.239652, + "nauc_precision_at_100_max": -0.097067, + "nauc_precision_at_100_std": 0.200324, + "nauc_precision_at_100_diff1": -0.24746, + "nauc_precision_at_1000_max": -0.173043, + "nauc_precision_at_1000_std": 0.098357, + "nauc_precision_at_1000_diff1": -0.243375, + "nauc_mrr_at_1_max": 0.599305, + "nauc_mrr_at_1_std": -0.04036, + "nauc_mrr_at_1_diff1": 0.479754, + "nauc_mrr_at_3_max": 0.597312, + "nauc_mrr_at_3_std": -0.043115, + "nauc_mrr_at_3_diff1": 0.459792, + "nauc_mrr_at_5_max": 0.598591, + "nauc_mrr_at_5_std": -0.047761, + "nauc_mrr_at_5_diff1": 0.460074, + "nauc_mrr_at_10_max": 0.59693, + "nauc_mrr_at_10_std": -0.04229, + "nauc_mrr_at_10_diff1": 0.461776, + "nauc_mrr_at_20_max": 0.597614, + "nauc_mrr_at_20_std": -0.036767, + "nauc_mrr_at_20_diff1": 0.461103, + "nauc_mrr_at_100_max": 0.597735, + "nauc_mrr_at_100_std": -0.03837, + "nauc_mrr_at_100_diff1": 0.461926, + "nauc_mrr_at_1000_max": 0.597631, + "nauc_mrr_at_1000_std": -0.038689, + "nauc_mrr_at_1000_diff1": 0.461797, + "main_score": 0.64857, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 70.80637097358704, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/model_meta.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/model_meta.json index 4ae7419b8..9240e39d6 100644 --- a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/model_meta.json +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/model_meta.json @@ -1 +1 @@ -{"name": "BAAI/bge-large-en-v1.5", "revision": "d4aa6901d3a41ba39fb536a557fa166f842b0e09", "release_date": "2023-09-12", "languages": ["eng_Latn"], "n_parameters": 1340000000, "max_tokens": 512.0, "embed_dim": 1024, "license": "mit", "open_weights": true, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/BAAI/bge-large-en-v1.5", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": {"NQ": ["test"], "NQHardNegatives": ["test"], "AmazonReviewsClassification": ["validation", "test"], "MLQARetrieval": ["validation", "test"]}, "adapted_from": null, "superseded_by": null, "loader": "sentence_transformers_loader"} \ No newline at end of file +{"name": "BAAI/bge-large-en-v1.5", "revision": "d4aa6901d3a41ba39fb536a557fa166f842b0e09", "release_date": "2023-09-12", "languages": ["eng_Latn"], "n_parameters": 335000000, "max_tokens": 512.0, "embed_dim": 1024, "license": "mit", "open_weights": true, "public_training_code": null, "public_training_data": "https://data.baai.ac.cn/details/BAAI-MTP", "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/BAAI/bge-large-en-v1.5", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": {"NQ": ["test"], "NQ-NL": ["test"], "NQHardNegatives": ["test"], "AmazonReviewsClassification": ["validation", "test"], "MLQARetrieval": ["validation", "test"]}, "adapted_from": null, "superseded_by": null, "modalities": ["text"], "loader": "sentence_transformers_loader"} \ No newline at end of file diff --git a/results/NovaSearch__stella_en_400M_v5/32b4baf84d02a1b1beb2df8952e875232e8ebe1d/BuiltBenchClusteringP2P.json b/results/NovaSearch__stella_en_400M_v5/32b4baf84d02a1b1beb2df8952e875232e8ebe1d/BuiltBenchClusteringP2P.json new file mode 100644 index 000000000..517f60ea4 --- /dev/null +++ b/results/NovaSearch__stella_en_400M_v5/32b4baf84d02a1b1beb2df8952e875232e8ebe1d/BuiltBenchClusteringP2P.json @@ -0,0 +1,42 @@ +{ + "dataset_revision": "919bb71053e9de62a68998161ce4f0cee8f786fb", + "task_name": "BuiltBenchClusteringP2P", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.516507, + "v_measure_std": 0.147702, + "v_measures": [ + 0.642151, + 0.345022, + 0.492214, + 0.410793, + 0.358103, + 0.350398, + 0.580694, + 0.483005, + 0.743892, + 0.840635, + 0.515998, + 0.352908, + 0.505906, + 0.449813, + 0.322875, + 0.550648, + 0.661874, + 0.438988, + 0.507968, + 0.776264 + ], + "main_score": 0.516507, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 88.02034330368042, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/NovaSearch__stella_en_400M_v5/32b4baf84d02a1b1beb2df8952e875232e8ebe1d/BuiltBenchClusteringS2S.json b/results/NovaSearch__stella_en_400M_v5/32b4baf84d02a1b1beb2df8952e875232e8ebe1d/BuiltBenchClusteringS2S.json new file mode 100644 index 000000000..eaf195b27 --- /dev/null +++ b/results/NovaSearch__stella_en_400M_v5/32b4baf84d02a1b1beb2df8952e875232e8ebe1d/BuiltBenchClusteringS2S.json @@ -0,0 +1,40 @@ +{ + "dataset_revision": "1aaeb2ece89ea0a8c64e215c95c4cfaf7e891149", + "task_name": "BuiltBenchClusteringS2S", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.485392, + "v_measure_std": 0.095719, + "v_measures": [ + 0.480659, + 0.494427, + 0.420161, + 0.631061, + 0.448772, + 0.480785, + 0.536366, + 0.673037, + 0.405552, + 0.645963, + 0.443345, + 0.388061, + 0.520328, + 0.528824, + 0.271167, + 0.399984, + 0.468882, + 0.499684 + ], + "main_score": 0.485392, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 13.831450939178467, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/NovaSearch__stella_en_400M_v5/32b4baf84d02a1b1beb2df8952e875232e8ebe1d/BuiltBenchReranking.json b/results/NovaSearch__stella_en_400M_v5/32b4baf84d02a1b1beb2df8952e875232e8ebe1d/BuiltBenchReranking.json new file mode 100644 index 000000000..dab53f7f2 --- /dev/null +++ b/results/NovaSearch__stella_en_400M_v5/32b4baf84d02a1b1beb2df8952e875232e8ebe1d/BuiltBenchReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "fd33b0b3454deb256be06a57e8147b32ba078ff9", + "task_name": "BuiltBenchReranking", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "map": 0.698114, + "mrr": 0.8251, + "nAUC_map_max": 0.339404, + "nAUC_map_std": 0.630568, + "nAUC_map_diff1": 0.015098, + "nAUC_mrr_max": 0.279707, + "nAUC_mrr_std": 0.510832, + "nAUC_mrr_diff1": 0.152868, + "main_score": 0.698114, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 67.28903293609619, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/NovaSearch__stella_en_400M_v5/32b4baf84d02a1b1beb2df8952e875232e8ebe1d/BuiltBenchRetrieval.json b/results/NovaSearch__stella_en_400M_v5/32b4baf84d02a1b1beb2df8952e875232e8ebe1d/BuiltBenchRetrieval.json new file mode 100644 index 000000000..4c6d5b0c4 --- /dev/null +++ b/results/NovaSearch__stella_en_400M_v5/32b4baf84d02a1b1beb2df8952e875232e8ebe1d/BuiltBenchRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ae611238a58dae85f3130563fe9f9e995444a8d6", + "task_name": "BuiltBenchRetrieval", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.65868, + "ndcg_at_3": 0.66689, + "ndcg_at_5": 0.66978, + "ndcg_at_10": 0.69913, + "ndcg_at_20": 0.72807, + "ndcg_at_100": 0.77163, + "ndcg_at_1000": 0.78698, + "map_at_1": 0.32579, + "map_at_3": 0.46918, + "map_at_5": 0.51512, + "map_at_10": 0.57495, + "map_at_20": 0.61629, + "map_at_100": 0.6546, + "map_at_1000": 0.66034, + "recall_at_1": 0.32579, + "recall_at_3": 0.5141, + "recall_at_5": 0.57969, + "recall_at_10": 0.69961, + "recall_at_20": 0.80739, + "recall_at_100": 0.94906, + "recall_at_1000": 0.99935, + "precision_at_1": 0.65868, + "precision_at_3": 0.47006, + "precision_at_5": 0.38323, + "precision_at_10": 0.28982, + "precision_at_20": 0.20479, + "precision_at_100": 0.06698, + "precision_at_1000": 0.00826, + "mrr_at_1": 0.658683, + "mrr_at_3": 0.726048, + "mrr_at_5": 0.733683, + "mrr_at_10": 0.74419, + "mrr_at_20": 0.746801, + "mrr_at_100": 0.747573, + "mrr_at_1000": 0.747627, + "nauc_ndcg_at_1_max": 0.527835, + "nauc_ndcg_at_1_std": -0.165613, + "nauc_ndcg_at_1_diff1": 0.501346, + "nauc_ndcg_at_3_max": 0.537176, + "nauc_ndcg_at_3_std": -0.124118, + "nauc_ndcg_at_3_diff1": 0.46073, + "nauc_ndcg_at_5_max": 0.526192, + "nauc_ndcg_at_5_std": -0.100703, + "nauc_ndcg_at_5_diff1": 0.43644, + "nauc_ndcg_at_10_max": 0.542564, + "nauc_ndcg_at_10_std": -0.07654, + "nauc_ndcg_at_10_diff1": 0.451731, + "nauc_ndcg_at_20_max": 0.564254, + "nauc_ndcg_at_20_std": -0.067744, + "nauc_ndcg_at_20_diff1": 0.469985, + "nauc_ndcg_at_100_max": 0.558826, + "nauc_ndcg_at_100_std": -0.011117, + "nauc_ndcg_at_100_diff1": 0.458071, + "nauc_ndcg_at_1000_max": 0.556131, + "nauc_ndcg_at_1000_std": -0.033074, + "nauc_ndcg_at_1000_diff1": 0.452649, + "nauc_map_at_1_max": 0.42748, + "nauc_map_at_1_std": -0.16154, + "nauc_map_at_1_diff1": 0.596479, + "nauc_map_at_3_max": 0.499698, + "nauc_map_at_3_std": -0.169451, + "nauc_map_at_3_diff1": 0.548765, + "nauc_map_at_5_max": 0.518061, + "nauc_map_at_5_std": -0.154854, + "nauc_map_at_5_diff1": 0.517524, + "nauc_map_at_10_max": 0.536386, + "nauc_map_at_10_std": -0.137637, + "nauc_map_at_10_diff1": 0.502239, + "nauc_map_at_20_max": 0.546231, + "nauc_map_at_20_std": -0.120573, + "nauc_map_at_20_diff1": 0.489438, + "nauc_map_at_100_max": 0.558952, + "nauc_map_at_100_std": -0.05665, + "nauc_map_at_100_diff1": 0.472405, + "nauc_map_at_1000_max": 0.553335, + "nauc_map_at_1000_std": -0.05506, + "nauc_map_at_1000_diff1": 0.465799, + "nauc_recall_at_1_max": 0.42748, + "nauc_recall_at_1_std": -0.16154, + "nauc_recall_at_1_diff1": 0.596479, + "nauc_recall_at_3_max": 0.456957, + "nauc_recall_at_3_std": -0.17506, + "nauc_recall_at_3_diff1": 0.502009, + "nauc_recall_at_5_max": 0.462532, + "nauc_recall_at_5_std": -0.147746, + "nauc_recall_at_5_diff1": 0.452516, + "nauc_recall_at_10_max": 0.466771, + "nauc_recall_at_10_std": -0.147111, + "nauc_recall_at_10_diff1": 0.430271, + "nauc_recall_at_20_max": 0.489112, + "nauc_recall_at_20_std": -0.143661, + "nauc_recall_at_20_diff1": 0.458784, + "nauc_recall_at_100_max": 0.50772, + "nauc_recall_at_100_std": 0.163523, + "nauc_recall_at_100_diff1": 0.480972, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 0.734758, + "nauc_recall_at_1000_diff1": 1.0, + "nauc_precision_at_1_max": 0.527835, + "nauc_precision_at_1_std": -0.165613, + "nauc_precision_at_1_diff1": 0.501346, + "nauc_precision_at_3_max": 0.221107, + "nauc_precision_at_3_std": 0.050085, + "nauc_precision_at_3_diff1": -0.035127, + "nauc_precision_at_5_max": 0.100918, + "nauc_precision_at_5_std": 0.114991, + "nauc_precision_at_5_diff1": -0.169522, + "nauc_precision_at_10_max": -0.018706, + "nauc_precision_at_10_std": 0.159591, + "nauc_precision_at_10_diff1": -0.237397, + "nauc_precision_at_20_max": -0.08686, + "nauc_precision_at_20_std": 0.192179, + "nauc_precision_at_20_diff1": -0.273046, + "nauc_precision_at_100_max": -0.166757, + "nauc_precision_at_100_std": 0.262619, + "nauc_precision_at_100_diff1": -0.304443, + "nauc_precision_at_1000_max": -0.227402, + "nauc_precision_at_1000_std": 0.185501, + "nauc_precision_at_1000_diff1": -0.327131, + "nauc_mrr_at_1_max": 0.527835, + "nauc_mrr_at_1_std": -0.165613, + "nauc_mrr_at_1_diff1": 0.501346, + "nauc_mrr_at_3_max": 0.545194, + "nauc_mrr_at_3_std": -0.120976, + "nauc_mrr_at_3_diff1": 0.478928, + "nauc_mrr_at_5_max": 0.547414, + "nauc_mrr_at_5_std": -0.109271, + "nauc_mrr_at_5_diff1": 0.47691, + "nauc_mrr_at_10_max": 0.542014, + "nauc_mrr_at_10_std": -0.119428, + "nauc_mrr_at_10_diff1": 0.475768, + "nauc_mrr_at_20_max": 0.543596, + "nauc_mrr_at_20_std": -0.118665, + "nauc_mrr_at_20_diff1": 0.479752, + "nauc_mrr_at_100_max": 0.543521, + "nauc_mrr_at_100_std": -0.119641, + "nauc_mrr_at_100_diff1": 0.479855, + "nauc_mrr_at_1000_max": 0.543498, + "nauc_mrr_at_1000_std": -0.119623, + "nauc_mrr_at_1000_diff1": 0.479767, + "main_score": 0.69913, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 90.18572783470154, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/NovaSearch__stella_en_400M_v5/32b4baf84d02a1b1beb2df8952e875232e8ebe1d/model_meta.json b/results/NovaSearch__stella_en_400M_v5/32b4baf84d02a1b1beb2df8952e875232e8ebe1d/model_meta.json new file mode 100644 index 000000000..8ceb709bb --- /dev/null +++ b/results/NovaSearch__stella_en_400M_v5/32b4baf84d02a1b1beb2df8952e875232e8ebe1d/model_meta.json @@ -0,0 +1 @@ +{"name": "NovaSearch/stella_en_400M_v5", "revision": "32b4baf84d02a1b1beb2df8952e875232e8ebe1d", "release_date": null, "languages": [], "n_parameters": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["Sentence Transformers"], "reference": null, "similarity_fn_name": "cosine", "use_instructions": null, "training_datasets": null, "adapted_from": null, "superseded_by": null, "modalities": ["text"], "loader": null} \ No newline at end of file diff --git a/results/Snowflake__snowflake-arctic-embed-m-v2.0/0d1661ceed1cb456c85726749d5be61ebb30d4f1/BuiltBenchClusteringP2P.json b/results/Snowflake__snowflake-arctic-embed-m-v2.0/0d1661ceed1cb456c85726749d5be61ebb30d4f1/BuiltBenchClusteringP2P.json new file mode 100644 index 000000000..c221c3507 --- /dev/null +++ b/results/Snowflake__snowflake-arctic-embed-m-v2.0/0d1661ceed1cb456c85726749d5be61ebb30d4f1/BuiltBenchClusteringP2P.json @@ -0,0 +1,42 @@ +{ + "dataset_revision": "919bb71053e9de62a68998161ce4f0cee8f786fb", + "task_name": "BuiltBenchClusteringP2P", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.493651, + "v_measure_std": 0.158052, + "v_measures": [ + 0.368569, + 0.496148, + 0.663635, + 0.45923, + 0.346884, + 0.403417, + 0.660002, + 0.433343, + 0.834636, + 0.123753, + 0.434052, + 0.362101, + 0.506822, + 0.503945, + 0.454222, + 0.6298, + 0.764047, + 0.390053, + 0.437295, + 0.601066 + ], + "main_score": 0.493651, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 40.115692138671875, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Snowflake__snowflake-arctic-embed-m-v2.0/0d1661ceed1cb456c85726749d5be61ebb30d4f1/BuiltBenchClusteringS2S.json b/results/Snowflake__snowflake-arctic-embed-m-v2.0/0d1661ceed1cb456c85726749d5be61ebb30d4f1/BuiltBenchClusteringS2S.json new file mode 100644 index 000000000..ca94d382c --- /dev/null +++ b/results/Snowflake__snowflake-arctic-embed-m-v2.0/0d1661ceed1cb456c85726749d5be61ebb30d4f1/BuiltBenchClusteringS2S.json @@ -0,0 +1,40 @@ +{ + "dataset_revision": "1aaeb2ece89ea0a8c64e215c95c4cfaf7e891149", + "task_name": "BuiltBenchClusteringS2S", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.380859, + "v_measure_std": 0.104405, + "v_measures": [ + 0.334703, + 0.440613, + 0.297332, + 0.465048, + 0.283268, + 0.270439, + 0.283762, + 0.628805, + 0.475892, + 0.464182, + 0.345746, + 0.341301, + 0.511834, + 0.467031, + 0.2381, + 0.311724, + 0.266826, + 0.42885 + ], + "main_score": 0.380859, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 5.650991916656494, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Snowflake__snowflake-arctic-embed-m-v2.0/0d1661ceed1cb456c85726749d5be61ebb30d4f1/BuiltBenchReranking.json b/results/Snowflake__snowflake-arctic-embed-m-v2.0/0d1661ceed1cb456c85726749d5be61ebb30d4f1/BuiltBenchReranking.json new file mode 100644 index 000000000..1c33f0167 --- /dev/null +++ b/results/Snowflake__snowflake-arctic-embed-m-v2.0/0d1661ceed1cb456c85726749d5be61ebb30d4f1/BuiltBenchReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "fd33b0b3454deb256be06a57e8147b32ba078ff9", + "task_name": "BuiltBenchReranking", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "map": 0.658457, + "mrr": 0.817052, + "nAUC_map_max": 0.347203, + "nAUC_map_std": 0.634482, + "nAUC_map_diff1": 0.019492, + "nAUC_mrr_max": 0.277545, + "nAUC_mrr_std": 0.387159, + "nAUC_mrr_diff1": 0.257989, + "main_score": 0.658457, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 23.288203716278076, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Snowflake__snowflake-arctic-embed-m-v2.0/0d1661ceed1cb456c85726749d5be61ebb30d4f1/BuiltBenchRetrieval.json b/results/Snowflake__snowflake-arctic-embed-m-v2.0/0d1661ceed1cb456c85726749d5be61ebb30d4f1/BuiltBenchRetrieval.json new file mode 100644 index 000000000..4a65b8894 --- /dev/null +++ b/results/Snowflake__snowflake-arctic-embed-m-v2.0/0d1661ceed1cb456c85726749d5be61ebb30d4f1/BuiltBenchRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ae611238a58dae85f3130563fe9f9e995444a8d6", + "task_name": "BuiltBenchRetrieval", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.61976, + "ndcg_at_3": 0.61388, + "ndcg_at_5": 0.6276, + "ndcg_at_10": 0.65022, + "ndcg_at_20": 0.67216, + "ndcg_at_100": 0.72379, + "ndcg_at_1000": 0.74875, + "map_at_1": 0.31668, + "map_at_3": 0.43361, + "map_at_5": 0.48248, + "map_at_10": 0.53234, + "map_at_20": 0.56575, + "map_at_100": 0.60241, + "map_at_1000": 0.60908, + "recall_at_1": 0.31668, + "recall_at_3": 0.47384, + "recall_at_5": 0.5507, + "recall_at_10": 0.65163, + "recall_at_20": 0.73949, + "recall_at_100": 0.90478, + "recall_at_1000": 0.99766, + "precision_at_1": 0.61976, + "precision_at_3": 0.42116, + "precision_at_5": 0.35389, + "precision_at_10": 0.26108, + "precision_at_20": 0.18249, + "precision_at_100": 0.06228, + "precision_at_1000": 0.00823, + "mrr_at_1": 0.61976, + "mrr_at_3": 0.686627, + "mrr_at_5": 0.697255, + "mrr_at_10": 0.703038, + "mrr_at_20": 0.705962, + "mrr_at_100": 0.707788, + "mrr_at_1000": 0.707932, + "nauc_ndcg_at_1_max": 0.610611, + "nauc_ndcg_at_1_std": 0.048774, + "nauc_ndcg_at_1_diff1": 0.559202, + "nauc_ndcg_at_3_max": 0.590231, + "nauc_ndcg_at_3_std": 0.02657, + "nauc_ndcg_at_3_diff1": 0.507558, + "nauc_ndcg_at_5_max": 0.588762, + "nauc_ndcg_at_5_std": 0.017243, + "nauc_ndcg_at_5_diff1": 0.508107, + "nauc_ndcg_at_10_max": 0.615228, + "nauc_ndcg_at_10_std": 0.0347, + "nauc_ndcg_at_10_diff1": 0.500781, + "nauc_ndcg_at_20_max": 0.628677, + "nauc_ndcg_at_20_std": 0.069285, + "nauc_ndcg_at_20_diff1": 0.498023, + "nauc_ndcg_at_100_max": 0.613312, + "nauc_ndcg_at_100_std": 0.087878, + "nauc_ndcg_at_100_diff1": 0.503867, + "nauc_ndcg_at_1000_max": 0.602957, + "nauc_ndcg_at_1000_std": 0.070026, + "nauc_ndcg_at_1000_diff1": 0.498731, + "nauc_map_at_1_max": 0.42862, + "nauc_map_at_1_std": -0.102014, + "nauc_map_at_1_diff1": 0.609341, + "nauc_map_at_3_max": 0.52974, + "nauc_map_at_3_std": -0.072454, + "nauc_map_at_3_diff1": 0.595412, + "nauc_map_at_5_max": 0.567365, + "nauc_map_at_5_std": -0.0534, + "nauc_map_at_5_diff1": 0.575118, + "nauc_map_at_10_max": 0.591829, + "nauc_map_at_10_std": -0.03599, + "nauc_map_at_10_diff1": 0.548333, + "nauc_map_at_20_max": 0.599526, + "nauc_map_at_20_std": -0.003117, + "nauc_map_at_20_diff1": 0.527039, + "nauc_map_at_100_max": 0.606318, + "nauc_map_at_100_std": 0.040461, + "nauc_map_at_100_diff1": 0.51321, + "nauc_map_at_1000_max": 0.602479, + "nauc_map_at_1000_std": 0.040215, + "nauc_map_at_1000_diff1": 0.507042, + "nauc_recall_at_1_max": 0.42862, + "nauc_recall_at_1_std": -0.102014, + "nauc_recall_at_1_diff1": 0.609341, + "nauc_recall_at_3_max": 0.499252, + "nauc_recall_at_3_std": -0.066284, + "nauc_recall_at_3_diff1": 0.562471, + "nauc_recall_at_5_max": 0.52734, + "nauc_recall_at_5_std": -0.056347, + "nauc_recall_at_5_diff1": 0.529201, + "nauc_recall_at_10_max": 0.568503, + "nauc_recall_at_10_std": -0.038815, + "nauc_recall_at_10_diff1": 0.48454, + "nauc_recall_at_20_max": 0.602691, + "nauc_recall_at_20_std": 0.067578, + "nauc_recall_at_20_diff1": 0.447301, + "nauc_recall_at_100_max": 0.618527, + "nauc_recall_at_100_std": 0.313447, + "nauc_recall_at_100_diff1": 0.440581, + "nauc_recall_at_1000_max": 0.895308, + "nauc_recall_at_1000_std": 0.850053, + "nauc_recall_at_1000_diff1": 0.620358, + "nauc_precision_at_1_max": 0.610611, + "nauc_precision_at_1_std": 0.048774, + "nauc_precision_at_1_diff1": 0.559202, + "nauc_precision_at_3_max": 0.335713, + "nauc_precision_at_3_std": 0.132501, + "nauc_precision_at_3_diff1": 0.046162, + "nauc_precision_at_5_max": 0.205044, + "nauc_precision_at_5_std": 0.139389, + "nauc_precision_at_5_diff1": -0.093785, + "nauc_precision_at_10_max": 0.072458, + "nauc_precision_at_10_std": 0.167213, + "nauc_precision_at_10_diff1": -0.20954, + "nauc_precision_at_20_max": -0.028484, + "nauc_precision_at_20_std": 0.203141, + "nauc_precision_at_20_diff1": -0.272846, + "nauc_precision_at_100_max": -0.160325, + "nauc_precision_at_100_std": 0.194947, + "nauc_precision_at_100_diff1": -0.321683, + "nauc_precision_at_1000_max": -0.23693, + "nauc_precision_at_1000_std": 0.108784, + "nauc_precision_at_1000_diff1": -0.348205, + "nauc_mrr_at_1_max": 0.610611, + "nauc_mrr_at_1_std": 0.048774, + "nauc_mrr_at_1_diff1": 0.559202, + "nauc_mrr_at_3_max": 0.609134, + "nauc_mrr_at_3_std": 0.050655, + "nauc_mrr_at_3_diff1": 0.546418, + "nauc_mrr_at_5_max": 0.609323, + "nauc_mrr_at_5_std": 0.052277, + "nauc_mrr_at_5_diff1": 0.5486, + "nauc_mrr_at_10_max": 0.612671, + "nauc_mrr_at_10_std": 0.054758, + "nauc_mrr_at_10_diff1": 0.548588, + "nauc_mrr_at_20_max": 0.612034, + "nauc_mrr_at_20_std": 0.056974, + "nauc_mrr_at_20_diff1": 0.547546, + "nauc_mrr_at_100_max": 0.611239, + "nauc_mrr_at_100_std": 0.055021, + "nauc_mrr_at_100_diff1": 0.548289, + "nauc_mrr_at_1000_max": 0.611276, + "nauc_mrr_at_1000_std": 0.054971, + "nauc_mrr_at_1000_diff1": 0.548288, + "main_score": 0.65022, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 25.708516359329224, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Snowflake__snowflake-arctic-embed-m-v2.0/0d1661ceed1cb456c85726749d5be61ebb30d4f1/model_meta.json b/results/Snowflake__snowflake-arctic-embed-m-v2.0/0d1661ceed1cb456c85726749d5be61ebb30d4f1/model_meta.json new file mode 100644 index 000000000..69b4df93c --- /dev/null +++ b/results/Snowflake__snowflake-arctic-embed-m-v2.0/0d1661ceed1cb456c85726749d5be61ebb30d4f1/model_meta.json @@ -0,0 +1 @@ +{"name": "Snowflake/snowflake-arctic-embed-m-v2.0", "revision": "0d1661ceed1cb456c85726749d5be61ebb30d4f1", "release_date": null, "languages": [], "n_parameters": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["Sentence Transformers"], "reference": null, "similarity_fn_name": "cosine", "use_instructions": null, "training_datasets": null, "adapted_from": null, "superseded_by": null, "modalities": ["text"], "loader": null} \ No newline at end of file diff --git a/results/Snowflake__snowflake-arctic-embed-m-v2.0/f2a7d59d80dfda5b1d14f096f3ce88bb6bf9ebdc/model_meta.json b/results/Snowflake__snowflake-arctic-embed-m-v2.0/f2a7d59d80dfda5b1d14f096f3ce88bb6bf9ebdc/model_meta.json index e0ac3c614..fe6995e58 100644 --- a/results/Snowflake__snowflake-arctic-embed-m-v2.0/f2a7d59d80dfda5b1d14f096f3ce88bb6bf9ebdc/model_meta.json +++ b/results/Snowflake__snowflake-arctic-embed-m-v2.0/f2a7d59d80dfda5b1d14f096f3ce88bb6bf9ebdc/model_meta.json @@ -1 +1 @@ -{"name": "Snowflake/snowflake-arctic-embed-m-v2.0", "revision": "f2a7d59d80dfda5b1d14f096f3ce88bb6bf9ebdc", "release_date": "2024-12-04", "languages": ["afr_Latn", "ara_Arab", "aze_Latn", "bel_Cyrl", "bul_Cyrl", "ben_Beng", "cat_Latn", "ceb_Latn", "ces_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "ell_Grek", "eng_Latn", "spa_Latn", "est_Latn", "eus_Latn", "fas_Arab", "fin_Latn", "fra_Latn", "glg_Latn", "guj_Gujr", "heb_Hebr", "hin_Deva", "hrv_Latn", "hat_Latn", "hun_Latn", "hye_Armn", "ind_Latn", "isl_Latn", "ita_Latn", "jpn_Jpan", "jav_Latn", "kat_Geor", "kaz_Cyrl", "khm_Khmr", "kan_Knda", "kor_Hang", "kir_Cyrl", "lao_Laoo", "lit_Latn", "lav_Latn", "mkd_Cyrl", "mal_Mlym", "mon_Cyrl", "mar_Deva", "msa_Latn", "mya_Mymr", "nep_Deva", "nld_Latn", "pan_Guru", "pol_Latn", "por_Latn", "que_Latn", "ron_Latn", "rus_Cyrl", "sin_Sinh", "slk_Latn", "slv_Latn", "som_Latn", "sqi_Latn", "srp_Cyrl", "swe_Latn", "swa_Latn", "tam_Taml", "tel_Telu", "tha_Thai", "tgl_Latn", "tur_Latn", "ukr_Cyrl", "urd_Arab", "vie_Latn", "yor_Latn", "zho_Hans"], "n_parameters": 305000000, "max_tokens": 8192.0, "embed_dim": 768, "license": "apache-2.0", "open_weights": true, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/Snowflake/snowflake-arctic-embed-m-v2.0", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": {"NQ": ["test"], "NQHardNegatives": ["test"], "HotPotQA": ["test"], "HotPotQAHardNegatives": ["test"], "HotPotQA-PL": ["test"], "FEVER": ["test"], "FEVERHardNegatives": ["test"]}, "adapted_from": "Alibaba-NLP/gte-multilingual-base", "superseded_by": null, "loader": "sentence_transformers_loader"} \ No newline at end of file +{"name": "Snowflake/snowflake-arctic-embed-m-v2.0", "revision": "f2a7d59d80dfda5b1d14f096f3ce88bb6bf9ebdc", "release_date": "2024-12-04", "languages": ["afr_Latn", "ara_Arab", "aze_Latn", "bel_Cyrl", "bul_Cyrl", "ben_Beng", "cat_Latn", "ceb_Latn", "ces_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "ell_Grek", "eng_Latn", "spa_Latn", "est_Latn", "eus_Latn", "fas_Arab", "fin_Latn", "fra_Latn", "glg_Latn", "guj_Gujr", "heb_Hebr", "hin_Deva", "hrv_Latn", "hat_Latn", "hun_Latn", "hye_Armn", "ind_Latn", "isl_Latn", "ita_Latn", "jpn_Jpan", "jav_Latn", "kat_Geor", "kaz_Cyrl", "khm_Khmr", "kan_Knda", "kor_Hang", "kir_Cyrl", "lao_Laoo", "lit_Latn", "lav_Latn", "mkd_Cyrl", "mal_Mlym", "mon_Cyrl", "mar_Deva", "msa_Latn", "mya_Mymr", "nep_Deva", "nld_Latn", "pan_Guru", "pol_Latn", "por_Latn", "que_Latn", "ron_Latn", "rus_Cyrl", "sin_Sinh", "slk_Latn", "slv_Latn", "som_Latn", "sqi_Latn", "srp_Cyrl", "swe_Latn", "swa_Latn", "tam_Taml", "tel_Telu", "tha_Thai", "tgl_Latn", "tur_Latn", "ukr_Cyrl", "urd_Arab", "vie_Latn", "yor_Latn", "zho_Hans"], "n_parameters": 305000000, "max_tokens": 8192.0, "embed_dim": 768, "license": "apache-2.0", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/Snowflake/snowflake-arctic-embed-m-v2.0", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": {"NQ": ["test"], "NQHardNegatives": ["test"], "HotPotQA": ["test"], "HotPotQAHardNegatives": ["test"], "HotPotQA-PL": ["test"], "FEVER": ["test"], "FEVERHardNegatives": ["test"]}, "adapted_from": "Alibaba-NLP/gte-multilingual-base", "superseded_by": null, "modalities": ["text"], "loader": "sentence_transformers_loader"} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/model_meta.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/model_meta.json index 16d655b80..1dd2fae2c 100644 --- a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/model_meta.json +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/model_meta.json @@ -1 +1 @@ -{"name": "WhereIsAI/UAE-Large-V1", "revision": "369c368f70f16a613f19f5598d4f12d9f44235d4", "release_date": "2023-12-04", "languages": ["eng_Latn"], "n_parameters": 335000, "max_tokens": 512.0, "embed_dim": 1024, "license": "mit", "open_weights": true, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/WhereIsAI/UAE-Large-V1", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": {"MNLI": [], "NLI": [], "SNLI": []}, "adapted_from": null, "superseded_by": null, "loader": "UAEWrapper"} \ No newline at end of file +{"name": "WhereIsAI/UAE-Large-V1", "revision": "369c368f70f16a613f19f5598d4f12d9f44235d4", "release_date": "2023-12-04", "languages": ["eng_Latn"], "n_parameters": 335000000, "max_tokens": 512.0, "embed_dim": 1024, "license": "mit", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/WhereIsAI/UAE-Large-V1", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": {"MNLI": [], "NLI": [], "SNLI": []}, "adapted_from": null, "superseded_by": null, "modalities": ["text"], "loader": "UAEWrapper"} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/f4264cd240f4e46a527f9f57a70cda6c2a12d248/BuiltBenchClusteringP2P.json b/results/WhereIsAI__UAE-Large-V1/f4264cd240f4e46a527f9f57a70cda6c2a12d248/BuiltBenchClusteringP2P.json new file mode 100644 index 000000000..0172c80df --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/f4264cd240f4e46a527f9f57a70cda6c2a12d248/BuiltBenchClusteringP2P.json @@ -0,0 +1,42 @@ +{ + "dataset_revision": "919bb71053e9de62a68998161ce4f0cee8f786fb", + "task_name": "BuiltBenchClusteringP2P", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.495252, + "v_measure_std": 0.114853, + "v_measures": [ + 0.404638, + 0.538664, + 0.535445, + 0.500171, + 0.409133, + 0.414723, + 0.581433, + 0.411615, + 0.736861, + 0.715567, + 0.458579, + 0.346875, + 0.611692, + 0.36096, + 0.380714, + 0.447584, + 0.569721, + 0.41237, + 0.409515, + 0.65878 + ], + "main_score": 0.495252, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 99.49797821044922, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/f4264cd240f4e46a527f9f57a70cda6c2a12d248/BuiltBenchClusteringS2S.json b/results/WhereIsAI__UAE-Large-V1/f4264cd240f4e46a527f9f57a70cda6c2a12d248/BuiltBenchClusteringS2S.json new file mode 100644 index 000000000..c9582671f --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/f4264cd240f4e46a527f9f57a70cda6c2a12d248/BuiltBenchClusteringS2S.json @@ -0,0 +1,40 @@ +{ + "dataset_revision": "1aaeb2ece89ea0a8c64e215c95c4cfaf7e891149", + "task_name": "BuiltBenchClusteringS2S", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.454516, + "v_measure_std": 0.118165, + "v_measures": [ + 0.315193, + 0.449456, + 0.427348, + 0.460202, + 0.363196, + 0.543411, + 0.370027, + 0.424889, + 0.519691, + 0.645066, + 0.575787, + 0.740878, + 0.445108, + 0.465929, + 0.242924, + 0.325197, + 0.374313, + 0.492678 + ], + "main_score": 0.454516, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 15.841771125793457, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/f4264cd240f4e46a527f9f57a70cda6c2a12d248/BuiltBenchReranking.json b/results/WhereIsAI__UAE-Large-V1/f4264cd240f4e46a527f9f57a70cda6c2a12d248/BuiltBenchReranking.json new file mode 100644 index 000000000..fbfd8329e --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/f4264cd240f4e46a527f9f57a70cda6c2a12d248/BuiltBenchReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "fd33b0b3454deb256be06a57e8147b32ba078ff9", + "task_name": "BuiltBenchReranking", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "map": 0.685326, + "mrr": 0.860482, + "nAUC_map_max": 0.396932, + "nAUC_map_std": 0.419806, + "nAUC_map_diff1": 0.057667, + "nAUC_mrr_max": 0.307459, + "nAUC_mrr_std": 0.164059, + "nAUC_mrr_diff1": 0.095558, + "main_score": 0.685326, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 63.21856665611267, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/f4264cd240f4e46a527f9f57a70cda6c2a12d248/BuiltBenchRetrieval.json b/results/WhereIsAI__UAE-Large-V1/f4264cd240f4e46a527f9f57a70cda6c2a12d248/BuiltBenchRetrieval.json new file mode 100644 index 000000000..7d4b98c5e --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/f4264cd240f4e46a527f9f57a70cda6c2a12d248/BuiltBenchRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ae611238a58dae85f3130563fe9f9e995444a8d6", + "task_name": "BuiltBenchRetrieval", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.62575, + "ndcg_at_3": 0.62827, + "ndcg_at_5": 0.63605, + "ndcg_at_10": 0.6642, + "ndcg_at_20": 0.69061, + "ndcg_at_100": 0.73907, + "ndcg_at_1000": 0.7578, + "map_at_1": 0.30095, + "map_at_3": 0.43523, + "map_at_5": 0.48446, + "map_at_10": 0.54033, + "map_at_20": 0.57701, + "map_at_100": 0.61434, + "map_at_1000": 0.62024, + "recall_at_1": 0.30095, + "recall_at_3": 0.48313, + "recall_at_5": 0.56199, + "recall_at_10": 0.67871, + "recall_at_20": 0.77589, + "recall_at_100": 0.93368, + "recall_at_1000": 0.99876, + "precision_at_1": 0.62575, + "precision_at_3": 0.44511, + "precision_at_5": 0.36587, + "precision_at_10": 0.27305, + "precision_at_20": 0.19237, + "precision_at_100": 0.06488, + "precision_at_1000": 0.0082, + "mrr_at_1": 0.625749, + "mrr_at_3": 0.69012, + "mrr_at_5": 0.698952, + "mrr_at_10": 0.708635, + "mrr_at_20": 0.711667, + "mrr_at_100": 0.713221, + "mrr_at_1000": 0.713312, + "nauc_ndcg_at_1_max": 0.553124, + "nauc_ndcg_at_1_std": -0.152324, + "nauc_ndcg_at_1_diff1": 0.461243, + "nauc_ndcg_at_3_max": 0.536789, + "nauc_ndcg_at_3_std": -0.155071, + "nauc_ndcg_at_3_diff1": 0.431587, + "nauc_ndcg_at_5_max": 0.535988, + "nauc_ndcg_at_5_std": -0.16593, + "nauc_ndcg_at_5_diff1": 0.423759, + "nauc_ndcg_at_10_max": 0.56407, + "nauc_ndcg_at_10_std": -0.126958, + "nauc_ndcg_at_10_diff1": 0.418181, + "nauc_ndcg_at_20_max": 0.583176, + "nauc_ndcg_at_20_std": -0.096102, + "nauc_ndcg_at_20_diff1": 0.413823, + "nauc_ndcg_at_100_max": 0.569742, + "nauc_ndcg_at_100_std": -0.065246, + "nauc_ndcg_at_100_diff1": 0.419494, + "nauc_ndcg_at_1000_max": 0.561335, + "nauc_ndcg_at_1000_std": -0.098469, + "nauc_ndcg_at_1000_diff1": 0.421221, + "nauc_map_at_1_max": 0.413421, + "nauc_map_at_1_std": -0.225079, + "nauc_map_at_1_diff1": 0.566989, + "nauc_map_at_3_max": 0.472218, + "nauc_map_at_3_std": -0.227714, + "nauc_map_at_3_diff1": 0.520783, + "nauc_map_at_5_max": 0.506683, + "nauc_map_at_5_std": -0.214513, + "nauc_map_at_5_diff1": 0.501612, + "nauc_map_at_10_max": 0.538902, + "nauc_map_at_10_std": -0.184429, + "nauc_map_at_10_diff1": 0.478835, + "nauc_map_at_20_max": 0.558252, + "nauc_map_at_20_std": -0.154043, + "nauc_map_at_20_diff1": 0.457956, + "nauc_map_at_100_max": 0.569734, + "nauc_map_at_100_std": -0.115905, + "nauc_map_at_100_diff1": 0.43757, + "nauc_map_at_1000_max": 0.567401, + "nauc_map_at_1000_std": -0.117284, + "nauc_map_at_1000_diff1": 0.434799, + "nauc_recall_at_1_max": 0.413421, + "nauc_recall_at_1_std": -0.225079, + "nauc_recall_at_1_diff1": 0.566989, + "nauc_recall_at_3_max": 0.428319, + "nauc_recall_at_3_std": -0.212393, + "nauc_recall_at_3_diff1": 0.46809, + "nauc_recall_at_5_max": 0.458722, + "nauc_recall_at_5_std": -0.202688, + "nauc_recall_at_5_diff1": 0.448762, + "nauc_recall_at_10_max": 0.492089, + "nauc_recall_at_10_std": -0.15505, + "nauc_recall_at_10_diff1": 0.41439, + "nauc_recall_at_20_max": 0.522936, + "nauc_recall_at_20_std": -0.059767, + "nauc_recall_at_20_diff1": 0.35762, + "nauc_recall_at_100_max": 0.569034, + "nauc_recall_at_100_std": 0.331512, + "nauc_recall_at_100_diff1": 0.316901, + "nauc_recall_at_1000_max": 0.145142, + "nauc_recall_at_1000_std": 0.791605, + "nauc_recall_at_1000_diff1": -0.081607, + "nauc_precision_at_1_max": 0.553124, + "nauc_precision_at_1_std": -0.152324, + "nauc_precision_at_1_diff1": 0.461243, + "nauc_precision_at_3_max": 0.274199, + "nauc_precision_at_3_std": 0.021431, + "nauc_precision_at_3_diff1": 0.008402, + "nauc_precision_at_5_max": 0.186933, + "nauc_precision_at_5_std": 0.075818, + "nauc_precision_at_5_diff1": -0.11313, + "nauc_precision_at_10_max": 0.072035, + "nauc_precision_at_10_std": 0.164813, + "nauc_precision_at_10_diff1": -0.221412, + "nauc_precision_at_20_max": 0.001304, + "nauc_precision_at_20_std": 0.209759, + "nauc_precision_at_20_diff1": -0.270813, + "nauc_precision_at_100_max": -0.110118, + "nauc_precision_at_100_std": 0.213833, + "nauc_precision_at_100_diff1": -0.277998, + "nauc_precision_at_1000_max": -0.17023, + "nauc_precision_at_1000_std": 0.110298, + "nauc_precision_at_1000_diff1": -0.264185, + "nauc_mrr_at_1_max": 0.553124, + "nauc_mrr_at_1_std": -0.152324, + "nauc_mrr_at_1_diff1": 0.461243, + "nauc_mrr_at_3_max": 0.551942, + "nauc_mrr_at_3_std": -0.143472, + "nauc_mrr_at_3_diff1": 0.443942, + "nauc_mrr_at_5_max": 0.550314, + "nauc_mrr_at_5_std": -0.147817, + "nauc_mrr_at_5_diff1": 0.445379, + "nauc_mrr_at_10_max": 0.552575, + "nauc_mrr_at_10_std": -0.140042, + "nauc_mrr_at_10_diff1": 0.442453, + "nauc_mrr_at_20_max": 0.55202, + "nauc_mrr_at_20_std": -0.137339, + "nauc_mrr_at_20_diff1": 0.442877, + "nauc_mrr_at_100_max": 0.552242, + "nauc_mrr_at_100_std": -0.138171, + "nauc_mrr_at_100_diff1": 0.444768, + "nauc_mrr_at_1000_max": 0.552099, + "nauc_mrr_at_1000_std": -0.138562, + "nauc_mrr_at_1000_diff1": 0.444601, + "main_score": 0.6642, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 100.32028198242188, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/f4264cd240f4e46a527f9f57a70cda6c2a12d248/model_meta.json b/results/WhereIsAI__UAE-Large-V1/f4264cd240f4e46a527f9f57a70cda6c2a12d248/model_meta.json new file mode 100644 index 000000000..d75377c79 --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/f4264cd240f4e46a527f9f57a70cda6c2a12d248/model_meta.json @@ -0,0 +1 @@ +{"name": "WhereIsAI/UAE-Large-V1", "revision": "f4264cd240f4e46a527f9f57a70cda6c2a12d248", "release_date": null, "languages": [], "n_parameters": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["Sentence Transformers"], "reference": null, "similarity_fn_name": "cosine", "use_instructions": null, "training_datasets": null, "adapted_from": null, "superseded_by": null, "modalities": ["text"], "loader": null} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/BuiltBenchClusteringP2P.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/BuiltBenchClusteringP2P.json new file mode 100644 index 000000000..920eaa948 --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/BuiltBenchClusteringP2P.json @@ -0,0 +1,42 @@ +{ + "dataset_revision": "919bb71053e9de62a68998161ce4f0cee8f786fb", + "task_name": "BuiltBenchClusteringP2P", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.499609, + "v_measure_std": 0.141955, + "v_measures": [ + 0.442107, + 0.511765, + 0.571154, + 0.550088, + 0.452445, + 0.473042, + 0.43122, + 0.468903, + 0.718894, + 0.1193, + 0.550024, + 0.799711, + 0.605426, + 0.411249, + 0.357857, + 0.44525, + 0.486472, + 0.43225, + 0.445542, + 0.71948 + ], + "main_score": 0.499609, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 43.72559976577759, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/BuiltBenchClusteringS2S.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/BuiltBenchClusteringS2S.json new file mode 100644 index 000000000..8071b78bd --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/BuiltBenchClusteringS2S.json @@ -0,0 +1,40 @@ +{ + "dataset_revision": "1aaeb2ece89ea0a8c64e215c95c4cfaf7e891149", + "task_name": "BuiltBenchClusteringS2S", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.464265, + "v_measure_std": 0.11438, + "v_measures": [ + 0.561357, + 0.481918, + 0.513714, + 0.381318, + 0.452458, + 0.424086, + 0.298472, + 0.471612, + 0.411202, + 0.628274, + 0.589339, + 0.741065, + 0.509937, + 0.408049, + 0.259533, + 0.340411, + 0.413189, + 0.470842 + ], + "main_score": 0.464265, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 5.52687406539917, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/BuiltBenchReranking.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/BuiltBenchReranking.json new file mode 100644 index 000000000..d3dc4647c --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/BuiltBenchReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "fd33b0b3454deb256be06a57e8147b32ba078ff9", + "task_name": "BuiltBenchReranking", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "map": 0.657474, + "mrr": 0.795564, + "nAUC_map_max": 0.384589, + "nAUC_map_std": 0.658021, + "nAUC_map_diff1": 0.007765, + "nAUC_mrr_max": 0.350717, + "nAUC_mrr_std": 0.536495, + "nAUC_mrr_diff1": 0.140772, + "main_score": 0.657474, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 20.830808877944946, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/BuiltBenchRetrieval.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/BuiltBenchRetrieval.json new file mode 100644 index 000000000..7536d52e8 --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/BuiltBenchRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ae611238a58dae85f3130563fe9f9e995444a8d6", + "task_name": "BuiltBenchRetrieval", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.58683, + "ndcg_at_3": 0.59249, + "ndcg_at_5": 0.60017, + "ndcg_at_10": 0.62782, + "ndcg_at_20": 0.65965, + "ndcg_at_100": 0.7145, + "ndcg_at_1000": 0.73457, + "map_at_1": 0.28895, + "map_at_3": 0.4138, + "map_at_5": 0.45762, + "map_at_10": 0.50915, + "map_at_20": 0.54557, + "map_at_100": 0.58378, + "map_at_1000": 0.58966, + "recall_at_1": 0.28895, + "recall_at_3": 0.45837, + "recall_at_5": 0.52876, + "recall_at_10": 0.64247, + "recall_at_20": 0.75777, + "recall_at_100": 0.93213, + "recall_at_1000": 0.9988, + "precision_at_1": 0.58683, + "precision_at_3": 0.41816, + "precision_at_5": 0.34491, + "precision_at_10": 0.25778, + "precision_at_20": 0.18398, + "precision_at_100": 0.06368, + "precision_at_1000": 0.00822, + "mrr_at_1": 0.586826, + "mrr_at_3": 0.651198, + "mrr_at_5": 0.664072, + "mrr_at_10": 0.673546, + "mrr_at_20": 0.678447, + "mrr_at_100": 0.680029, + "mrr_at_1000": 0.68012, + "nauc_ndcg_at_1_max": 0.62462, + "nauc_ndcg_at_1_std": -0.067142, + "nauc_ndcg_at_1_diff1": 0.455621, + "nauc_ndcg_at_3_max": 0.572341, + "nauc_ndcg_at_3_std": -0.101374, + "nauc_ndcg_at_3_diff1": 0.412943, + "nauc_ndcg_at_5_max": 0.575507, + "nauc_ndcg_at_5_std": -0.097517, + "nauc_ndcg_at_5_diff1": 0.416309, + "nauc_ndcg_at_10_max": 0.593817, + "nauc_ndcg_at_10_std": -0.084952, + "nauc_ndcg_at_10_diff1": 0.407991, + "nauc_ndcg_at_20_max": 0.606773, + "nauc_ndcg_at_20_std": -0.065671, + "nauc_ndcg_at_20_diff1": 0.413687, + "nauc_ndcg_at_100_max": 0.605709, + "nauc_ndcg_at_100_std": -0.016086, + "nauc_ndcg_at_100_diff1": 0.413031, + "nauc_ndcg_at_1000_max": 0.592016, + "nauc_ndcg_at_1000_std": -0.037528, + "nauc_ndcg_at_1000_diff1": 0.404493, + "nauc_map_at_1_max": 0.479039, + "nauc_map_at_1_std": -0.181236, + "nauc_map_at_1_diff1": 0.552155, + "nauc_map_at_3_max": 0.528738, + "nauc_map_at_3_std": -0.17087, + "nauc_map_at_3_diff1": 0.500048, + "nauc_map_at_5_max": 0.556277, + "nauc_map_at_5_std": -0.157284, + "nauc_map_at_5_diff1": 0.481839, + "nauc_map_at_10_max": 0.579469, + "nauc_map_at_10_std": -0.129728, + "nauc_map_at_10_diff1": 0.460287, + "nauc_map_at_20_max": 0.595512, + "nauc_map_at_20_std": -0.105173, + "nauc_map_at_20_diff1": 0.448557, + "nauc_map_at_100_max": 0.599658, + "nauc_map_at_100_std": -0.065257, + "nauc_map_at_100_diff1": 0.434083, + "nauc_map_at_1000_max": 0.596014, + "nauc_map_at_1000_std": -0.065566, + "nauc_map_at_1000_diff1": 0.427813, + "nauc_recall_at_1_max": 0.479039, + "nauc_recall_at_1_std": -0.181236, + "nauc_recall_at_1_diff1": 0.552155, + "nauc_recall_at_3_max": 0.465063, + "nauc_recall_at_3_std": -0.193887, + "nauc_recall_at_3_diff1": 0.442091, + "nauc_recall_at_5_max": 0.501087, + "nauc_recall_at_5_std": -0.176074, + "nauc_recall_at_5_diff1": 0.434141, + "nauc_recall_at_10_max": 0.512645, + "nauc_recall_at_10_std": -0.168839, + "nauc_recall_at_10_diff1": 0.38751, + "nauc_recall_at_20_max": 0.534183, + "nauc_recall_at_20_std": -0.102337, + "nauc_recall_at_20_diff1": 0.360225, + "nauc_recall_at_100_max": 0.616596, + "nauc_recall_at_100_std": 0.215859, + "nauc_recall_at_100_diff1": 0.398533, + "nauc_recall_at_1000_max": 0.90687, + "nauc_recall_at_1000_std": 0.801312, + "nauc_recall_at_1000_diff1": 0.680085, + "nauc_precision_at_1_max": 0.62462, + "nauc_precision_at_1_std": -0.067142, + "nauc_precision_at_1_diff1": 0.455621, + "nauc_precision_at_3_max": 0.298731, + "nauc_precision_at_3_std": 0.040584, + "nauc_precision_at_3_diff1": 0.041033, + "nauc_precision_at_5_max": 0.186909, + "nauc_precision_at_5_std": 0.098692, + "nauc_precision_at_5_diff1": -0.075901, + "nauc_precision_at_10_max": 0.073697, + "nauc_precision_at_10_std": 0.159174, + "nauc_precision_at_10_diff1": -0.17656, + "nauc_precision_at_20_max": -0.014398, + "nauc_precision_at_20_std": 0.186869, + "nauc_precision_at_20_diff1": -0.221676, + "nauc_precision_at_100_max": -0.151115, + "nauc_precision_at_100_std": 0.212761, + "nauc_precision_at_100_diff1": -0.275524, + "nauc_precision_at_1000_max": -0.233659, + "nauc_precision_at_1000_std": 0.096996, + "nauc_precision_at_1000_diff1": -0.310535, + "nauc_mrr_at_1_max": 0.62462, + "nauc_mrr_at_1_std": -0.067142, + "nauc_mrr_at_1_diff1": 0.455621, + "nauc_mrr_at_3_max": 0.615056, + "nauc_mrr_at_3_std": -0.06073, + "nauc_mrr_at_3_diff1": 0.437494, + "nauc_mrr_at_5_max": 0.615354, + "nauc_mrr_at_5_std": -0.052991, + "nauc_mrr_at_5_diff1": 0.435164, + "nauc_mrr_at_10_max": 0.617831, + "nauc_mrr_at_10_std": -0.052387, + "nauc_mrr_at_10_diff1": 0.436208, + "nauc_mrr_at_20_max": 0.615893, + "nauc_mrr_at_20_std": -0.053123, + "nauc_mrr_at_20_diff1": 0.435246, + "nauc_mrr_at_100_max": 0.616476, + "nauc_mrr_at_100_std": -0.053192, + "nauc_mrr_at_100_diff1": 0.437377, + "nauc_mrr_at_1000_max": 0.616442, + "nauc_mrr_at_1000_std": -0.053095, + "nauc_mrr_at_1000_diff1": 0.437262, + "main_score": 0.62782, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 22.773422479629517, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/model_meta.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/model_meta.json index 14d39878a..b407730ff 100644 --- a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/model_meta.json +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/model_meta.json @@ -1 +1 @@ -{"name": "avsolatorio/GIST-Embedding-v0", "revision": "bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c", "release_date": "2024-01-31", "languages": ["eng_Latn"], "n_parameters": 109482240, "max_tokens": 512.0, "embed_dim": 768, "license": "mit", "open_weights": true, "public_training_code": null, "framework": ["PyTorch", "Sentence Transformers"], "reference": "https://huggingface.co/avsolatorio/GIST-Embedding-v0", "similarity_fn_name": "cosine", "use_instructions": null, "training_datasets": null, "adapted_from": null, "superseded_by": null, "loader": null} \ No newline at end of file +{"name": "avsolatorio/GIST-Embedding-v0", "revision": "bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c", "release_date": "2024-01-31", "languages": ["eng_Latn"], "n_parameters": 109482240, "max_tokens": 512.0, "embed_dim": 768, "license": "mit", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["PyTorch", "Sentence Transformers"], "reference": "https://huggingface.co/avsolatorio/GIST-Embedding-v0", "similarity_fn_name": "cosine", "use_instructions": null, "training_datasets": null, "adapted_from": null, "superseded_by": null, "modalities": ["text"], "loader": null} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/BuiltBenchClusteringP2P.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/BuiltBenchClusteringP2P.json new file mode 100644 index 000000000..d8937ecca --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/BuiltBenchClusteringP2P.json @@ -0,0 +1,42 @@ +{ + "dataset_revision": "919bb71053e9de62a68998161ce4f0cee8f786fb", + "task_name": "BuiltBenchClusteringP2P", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.479121, + "v_measure_std": 0.087825, + "v_measures": [ + 0.367224, + 0.622439, + 0.46868, + 0.488691, + 0.401202, + 0.485713, + 0.368538, + 0.497778, + 0.618962, + 0.58535, + 0.448143, + 0.346875, + 0.589643, + 0.433486, + 0.377827, + 0.392122, + 0.512267, + 0.475304, + 0.482431, + 0.619736 + ], + "main_score": 0.479121, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 107.0368058681488, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/BuiltBenchClusteringS2S.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/BuiltBenchClusteringS2S.json new file mode 100644 index 000000000..c24e28551 --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/BuiltBenchClusteringS2S.json @@ -0,0 +1,40 @@ +{ + "dataset_revision": "1aaeb2ece89ea0a8c64e215c95c4cfaf7e891149", + "task_name": "BuiltBenchClusteringS2S", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.479673, + "v_measure_std": 0.131759, + "v_measures": [ + 0.448192, + 0.568439, + 0.286369, + 0.657252, + 0.358804, + 0.528404, + 0.417137, + 0.492416, + 0.615605, + 0.632342, + 0.575787, + 0.741065, + 0.394496, + 0.477026, + 0.225925, + 0.383663, + 0.364464, + 0.466727 + ], + "main_score": 0.479673, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 15.74053144454956, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/BuiltBenchReranking.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/BuiltBenchReranking.json new file mode 100644 index 000000000..0ccd601b1 --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/BuiltBenchReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "fd33b0b3454deb256be06a57e8147b32ba078ff9", + "task_name": "BuiltBenchReranking", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "map": 0.680336, + "mrr": 0.841833, + "nAUC_map_max": 0.383605, + "nAUC_map_std": 0.468615, + "nAUC_map_diff1": 0.005051, + "nAUC_mrr_max": 0.30361, + "nAUC_mrr_std": 0.218201, + "nAUC_mrr_diff1": 0.1166, + "main_score": 0.680336, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 64.0937888622284, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/BuiltBenchRetrieval.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/BuiltBenchRetrieval.json new file mode 100644 index 000000000..2f255e2c8 --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/BuiltBenchRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ae611238a58dae85f3130563fe9f9e995444a8d6", + "task_name": "BuiltBenchRetrieval", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.62575, + "ndcg_at_3": 0.63915, + "ndcg_at_5": 0.6443, + "ndcg_at_10": 0.67057, + "ndcg_at_20": 0.69486, + "ndcg_at_100": 0.74297, + "ndcg_at_1000": 0.76245, + "map_at_1": 0.3021, + "map_at_3": 0.44459, + "map_at_5": 0.4913, + "map_at_10": 0.54794, + "map_at_20": 0.58264, + "map_at_100": 0.62135, + "map_at_1000": 0.62716, + "recall_at_1": 0.3021, + "recall_at_3": 0.49647, + "recall_at_5": 0.56729, + "recall_at_10": 0.67825, + "recall_at_20": 0.77805, + "recall_at_100": 0.9307, + "recall_at_1000": 0.99944, + "precision_at_1": 0.62575, + "precision_at_3": 0.45509, + "precision_at_5": 0.37066, + "precision_at_10": 0.27784, + "precision_at_20": 0.19207, + "precision_at_100": 0.06518, + "precision_at_1000": 0.00823, + "mrr_at_1": 0.625749, + "mrr_at_3": 0.693114, + "mrr_at_5": 0.703293, + "mrr_at_10": 0.709761, + "mrr_at_20": 0.713687, + "mrr_at_100": 0.714843, + "mrr_at_1000": 0.714996, + "nauc_ndcg_at_1_max": 0.531432, + "nauc_ndcg_at_1_std": -0.04085, + "nauc_ndcg_at_1_diff1": 0.419739, + "nauc_ndcg_at_3_max": 0.51428, + "nauc_ndcg_at_3_std": -0.09522, + "nauc_ndcg_at_3_diff1": 0.373659, + "nauc_ndcg_at_5_max": 0.523662, + "nauc_ndcg_at_5_std": -0.091405, + "nauc_ndcg_at_5_diff1": 0.391451, + "nauc_ndcg_at_10_max": 0.561152, + "nauc_ndcg_at_10_std": -0.033336, + "nauc_ndcg_at_10_diff1": 0.386995, + "nauc_ndcg_at_20_max": 0.577367, + "nauc_ndcg_at_20_std": -0.014151, + "nauc_ndcg_at_20_diff1": 0.382947, + "nauc_ndcg_at_100_max": 0.55659, + "nauc_ndcg_at_100_std": 0.012935, + "nauc_ndcg_at_100_diff1": 0.374016, + "nauc_ndcg_at_1000_max": 0.553945, + "nauc_ndcg_at_1000_std": -0.016982, + "nauc_ndcg_at_1000_diff1": 0.377553, + "nauc_map_at_1_max": 0.4208, + "nauc_map_at_1_std": -0.185565, + "nauc_map_at_1_diff1": 0.556411, + "nauc_map_at_3_max": 0.466804, + "nauc_map_at_3_std": -0.175822, + "nauc_map_at_3_diff1": 0.502811, + "nauc_map_at_5_max": 0.505895, + "nauc_map_at_5_std": -0.154865, + "nauc_map_at_5_diff1": 0.491905, + "nauc_map_at_10_max": 0.539591, + "nauc_map_at_10_std": -0.106939, + "nauc_map_at_10_diff1": 0.461097, + "nauc_map_at_20_max": 0.554402, + "nauc_map_at_20_std": -0.081188, + "nauc_map_at_20_diff1": 0.435462, + "nauc_map_at_100_max": 0.556175, + "nauc_map_at_100_std": -0.04046, + "nauc_map_at_100_diff1": 0.400918, + "nauc_map_at_1000_max": 0.554593, + "nauc_map_at_1000_std": -0.041122, + "nauc_map_at_1000_diff1": 0.398481, + "nauc_recall_at_1_max": 0.4208, + "nauc_recall_at_1_std": -0.185565, + "nauc_recall_at_1_diff1": 0.556411, + "nauc_recall_at_3_max": 0.418931, + "nauc_recall_at_3_std": -0.176201, + "nauc_recall_at_3_diff1": 0.445027, + "nauc_recall_at_5_max": 0.465098, + "nauc_recall_at_5_std": -0.152949, + "nauc_recall_at_5_diff1": 0.457695, + "nauc_recall_at_10_max": 0.520309, + "nauc_recall_at_10_std": -0.056516, + "nauc_recall_at_10_diff1": 0.424722, + "nauc_recall_at_20_max": 0.542489, + "nauc_recall_at_20_std": 0.006262, + "nauc_recall_at_20_diff1": 0.377273, + "nauc_recall_at_100_max": 0.514278, + "nauc_recall_at_100_std": 0.344186, + "nauc_recall_at_100_diff1": 0.235497, + "nauc_recall_at_1000_max": -0.046945, + "nauc_recall_at_1000_std": 0.756666, + "nauc_recall_at_1000_diff1": -0.02328, + "nauc_precision_at_1_max": 0.531432, + "nauc_precision_at_1_std": -0.04085, + "nauc_precision_at_1_diff1": 0.419739, + "nauc_precision_at_3_max": 0.227548, + "nauc_precision_at_3_std": 0.045785, + "nauc_precision_at_3_diff1": -0.054027, + "nauc_precision_at_5_max": 0.152209, + "nauc_precision_at_5_std": 0.106662, + "nauc_precision_at_5_diff1": -0.145604, + "nauc_precision_at_10_max": 0.045203, + "nauc_precision_at_10_std": 0.186459, + "nauc_precision_at_10_diff1": -0.258081, + "nauc_precision_at_20_max": -0.02797, + "nauc_precision_at_20_std": 0.211588, + "nauc_precision_at_20_diff1": -0.311151, + "nauc_precision_at_100_max": -0.131654, + "nauc_precision_at_100_std": 0.213136, + "nauc_precision_at_100_diff1": -0.321153, + "nauc_precision_at_1000_max": -0.177883, + "nauc_precision_at_1000_std": 0.111124, + "nauc_precision_at_1000_diff1": -0.294537, + "nauc_mrr_at_1_max": 0.531432, + "nauc_mrr_at_1_std": -0.04085, + "nauc_mrr_at_1_diff1": 0.419739, + "nauc_mrr_at_3_max": 0.540075, + "nauc_mrr_at_3_std": -0.037062, + "nauc_mrr_at_3_diff1": 0.386981, + "nauc_mrr_at_5_max": 0.544177, + "nauc_mrr_at_5_std": -0.032308, + "nauc_mrr_at_5_diff1": 0.39113, + "nauc_mrr_at_10_max": 0.545088, + "nauc_mrr_at_10_std": -0.027182, + "nauc_mrr_at_10_diff1": 0.394573, + "nauc_mrr_at_20_max": 0.545227, + "nauc_mrr_at_20_std": -0.026443, + "nauc_mrr_at_20_diff1": 0.394919, + "nauc_mrr_at_100_max": 0.545393, + "nauc_mrr_at_100_std": -0.026581, + "nauc_mrr_at_100_diff1": 0.395573, + "nauc_mrr_at_1000_max": 0.545264, + "nauc_mrr_at_1000_std": -0.027017, + "nauc_mrr_at_1000_diff1": 0.395741, + "main_score": 0.67057, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 72.72722625732422, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/model_meta.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/model_meta.json index b38f04821..748d80015 100644 --- a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/model_meta.json +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/model_meta.json @@ -1 +1 @@ -{"name": "avsolatorio/GIST-large-Embedding-v0", "revision": "7831200e2f7819b994490c091cf3258a2b821f0c", "release_date": "2024-02-14", "languages": ["eng_Latn"], "n_parameters": 335141888, "max_tokens": 512.0, "embed_dim": 1024, "license": "mit", "open_weights": true, "public_training_code": null, "framework": ["PyTorch", "Sentence Transformers"], "reference": "https://huggingface.co/avsolatorio/GIST-large-Embedding-v0", "similarity_fn_name": "cosine", "use_instructions": null, "training_datasets": null, "adapted_from": null, "superseded_by": null, "loader": null} \ No newline at end of file +{"name": "avsolatorio/GIST-large-Embedding-v0", "revision": "7831200e2f7819b994490c091cf3258a2b821f0c", "release_date": "2024-02-14", "languages": ["eng_Latn"], "n_parameters": 335141888, "max_tokens": 512.0, "embed_dim": 1024, "license": "mit", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["PyTorch", "Sentence Transformers"], "reference": "https://huggingface.co/avsolatorio/GIST-large-Embedding-v0", "similarity_fn_name": "cosine", "use_instructions": null, "training_datasets": null, "adapted_from": null, "superseded_by": null, "modalities": ["text"], "loader": null} \ No newline at end of file diff --git a/results/intfloat__e5-large-v2/b322e09026e4ea05f42beadf4d661fb4e101d311/BuiltBenchClusteringP2P.json b/results/intfloat__e5-large-v2/b322e09026e4ea05f42beadf4d661fb4e101d311/BuiltBenchClusteringP2P.json new file mode 100644 index 000000000..2c3c6d6c3 --- /dev/null +++ b/results/intfloat__e5-large-v2/b322e09026e4ea05f42beadf4d661fb4e101d311/BuiltBenchClusteringP2P.json @@ -0,0 +1,42 @@ +{ + "dataset_revision": "919bb71053e9de62a68998161ce4f0cee8f786fb", + "task_name": "BuiltBenchClusteringP2P", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.494497, + "v_measure_std": 0.138226, + "v_measures": [ + 0.373464, + 0.468404, + 0.457841, + 0.428981, + 0.362404, + 0.40716, + 0.465831, + 0.528102, + 0.636094, + 0.945911, + 0.438281, + 0.436577, + 0.524842, + 0.417565, + 0.39871, + 0.449854, + 0.738042, + 0.420359, + 0.393346, + 0.598169 + ], + "main_score": 0.494497, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 115.16386437416077, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/intfloat__e5-large-v2/b322e09026e4ea05f42beadf4d661fb4e101d311/BuiltBenchClusteringS2S.json b/results/intfloat__e5-large-v2/b322e09026e4ea05f42beadf4d661fb4e101d311/BuiltBenchClusteringS2S.json new file mode 100644 index 000000000..3b0c2eaf8 --- /dev/null +++ b/results/intfloat__e5-large-v2/b322e09026e4ea05f42beadf4d661fb4e101d311/BuiltBenchClusteringS2S.json @@ -0,0 +1,40 @@ +{ + "dataset_revision": "1aaeb2ece89ea0a8c64e215c95c4cfaf7e891149", + "task_name": "BuiltBenchClusteringS2S", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.421101, + "v_measure_std": 0.111749, + "v_measures": [ + 0.35576, + 0.664944, + 0.442887, + 0.538046, + 0.347608, + 0.32859, + 0.321035, + 0.341899, + 0.362586, + 0.570139, + 0.511879, + 0.37728, + 0.572604, + 0.406961, + 0.253889, + 0.339228, + 0.314918, + 0.529572 + ], + "main_score": 0.421101, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 15.965330362319946, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/intfloat__e5-large-v2/b322e09026e4ea05f42beadf4d661fb4e101d311/BuiltBenchReranking.json b/results/intfloat__e5-large-v2/b322e09026e4ea05f42beadf4d661fb4e101d311/BuiltBenchReranking.json new file mode 100644 index 000000000..399fdc1db --- /dev/null +++ b/results/intfloat__e5-large-v2/b322e09026e4ea05f42beadf4d661fb4e101d311/BuiltBenchReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "fd33b0b3454deb256be06a57e8147b32ba078ff9", + "task_name": "BuiltBenchReranking", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "map": 0.645789, + "mrr": 0.796894, + "nAUC_map_max": 0.296548, + "nAUC_map_std": 0.629646, + "nAUC_map_diff1": 0.075253, + "nAUC_mrr_max": 0.204346, + "nAUC_mrr_std": 0.465675, + "nAUC_mrr_diff1": 0.147721, + "main_score": 0.645789, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 60.41311073303223, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/intfloat__e5-large-v2/b322e09026e4ea05f42beadf4d661fb4e101d311/BuiltBenchRetrieval.json b/results/intfloat__e5-large-v2/b322e09026e4ea05f42beadf4d661fb4e101d311/BuiltBenchRetrieval.json new file mode 100644 index 000000000..9f821f719 --- /dev/null +++ b/results/intfloat__e5-large-v2/b322e09026e4ea05f42beadf4d661fb4e101d311/BuiltBenchRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ae611238a58dae85f3130563fe9f9e995444a8d6", + "task_name": "BuiltBenchRetrieval", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.60778, + "ndcg_at_3": 0.60696, + "ndcg_at_5": 0.61824, + "ndcg_at_10": 0.64632, + "ndcg_at_20": 0.66891, + "ndcg_at_100": 0.71935, + "ndcg_at_1000": 0.74204, + "map_at_1": 0.30441, + "map_at_3": 0.42134, + "map_at_5": 0.46797, + "map_at_10": 0.52117, + "map_at_20": 0.5539, + "map_at_100": 0.58991, + "map_at_1000": 0.59625, + "recall_at_1": 0.30441, + "recall_at_3": 0.46366, + "recall_at_5": 0.54622, + "recall_at_10": 0.65978, + "recall_at_20": 0.75384, + "recall_at_100": 0.91505, + "recall_at_1000": 0.99411, + "precision_at_1": 0.60778, + "precision_at_3": 0.42415, + "precision_at_5": 0.3503, + "precision_at_10": 0.26377, + "precision_at_20": 0.18204, + "precision_at_100": 0.06254, + "precision_at_1000": 0.00816, + "mrr_at_1": 0.607784, + "mrr_at_3": 0.676148, + "mrr_at_5": 0.69007, + "mrr_at_10": 0.699739, + "mrr_at_20": 0.702589, + "mrr_at_100": 0.703808, + "mrr_at_1000": 0.703962, + "nauc_ndcg_at_1_max": 0.534045, + "nauc_ndcg_at_1_std": -0.023658, + "nauc_ndcg_at_1_diff1": 0.453446, + "nauc_ndcg_at_3_max": 0.513482, + "nauc_ndcg_at_3_std": 0.002944, + "nauc_ndcg_at_3_diff1": 0.419438, + "nauc_ndcg_at_5_max": 0.524899, + "nauc_ndcg_at_5_std": 0.01267, + "nauc_ndcg_at_5_diff1": 0.419074, + "nauc_ndcg_at_10_max": 0.535071, + "nauc_ndcg_at_10_std": 0.035706, + "nauc_ndcg_at_10_diff1": 0.434345, + "nauc_ndcg_at_20_max": 0.558122, + "nauc_ndcg_at_20_std": 0.05637, + "nauc_ndcg_at_20_diff1": 0.448541, + "nauc_ndcg_at_100_max": 0.562264, + "nauc_ndcg_at_100_std": 0.101283, + "nauc_ndcg_at_100_diff1": 0.420439, + "nauc_ndcg_at_1000_max": 0.551626, + "nauc_ndcg_at_1000_std": 0.08016, + "nauc_ndcg_at_1000_diff1": 0.432627, + "nauc_map_at_1_max": 0.394754, + "nauc_map_at_1_std": -0.116952, + "nauc_map_at_1_diff1": 0.576709, + "nauc_map_at_3_max": 0.461218, + "nauc_map_at_3_std": -0.086919, + "nauc_map_at_3_diff1": 0.523889, + "nauc_map_at_5_max": 0.503118, + "nauc_map_at_5_std": -0.059517, + "nauc_map_at_5_diff1": 0.499743, + "nauc_map_at_10_max": 0.517927, + "nauc_map_at_10_std": -0.035828, + "nauc_map_at_10_diff1": 0.483042, + "nauc_map_at_20_max": 0.536622, + "nauc_map_at_20_std": -0.009299, + "nauc_map_at_20_diff1": 0.472391, + "nauc_map_at_100_max": 0.547438, + "nauc_map_at_100_std": 0.033609, + "nauc_map_at_100_diff1": 0.454312, + "nauc_map_at_1000_max": 0.543891, + "nauc_map_at_1000_std": 0.036524, + "nauc_map_at_1000_diff1": 0.453778, + "nauc_recall_at_1_max": 0.394754, + "nauc_recall_at_1_std": -0.116952, + "nauc_recall_at_1_diff1": 0.576709, + "nauc_recall_at_3_max": 0.423234, + "nauc_recall_at_3_std": -0.090789, + "nauc_recall_at_3_diff1": 0.479938, + "nauc_recall_at_5_max": 0.458032, + "nauc_recall_at_5_std": -0.04838, + "nauc_recall_at_5_diff1": 0.42149, + "nauc_recall_at_10_max": 0.452825, + "nauc_recall_at_10_std": -0.003369, + "nauc_recall_at_10_diff1": 0.406894, + "nauc_recall_at_20_max": 0.514549, + "nauc_recall_at_20_std": 0.064726, + "nauc_recall_at_20_diff1": 0.412604, + "nauc_recall_at_100_max": 0.586511, + "nauc_recall_at_100_std": 0.363432, + "nauc_recall_at_100_diff1": 0.187894, + "nauc_recall_at_1000_max": 0.921547, + "nauc_recall_at_1000_std": 0.816773, + "nauc_recall_at_1000_diff1": 0.569514, + "nauc_precision_at_1_max": 0.534045, + "nauc_precision_at_1_std": -0.023658, + "nauc_precision_at_1_diff1": 0.453446, + "nauc_precision_at_3_max": 0.298537, + "nauc_precision_at_3_std": 0.145957, + "nauc_precision_at_3_diff1": -0.001597, + "nauc_precision_at_5_max": 0.2094, + "nauc_precision_at_5_std": 0.176532, + "nauc_precision_at_5_diff1": -0.112884, + "nauc_precision_at_10_max": 0.072056, + "nauc_precision_at_10_std": 0.181115, + "nauc_precision_at_10_diff1": -0.191554, + "nauc_precision_at_20_max": 0.004889, + "nauc_precision_at_20_std": 0.198854, + "nauc_precision_at_20_diff1": -0.223929, + "nauc_precision_at_100_max": -0.118563, + "nauc_precision_at_100_std": 0.210237, + "nauc_precision_at_100_diff1": -0.257261, + "nauc_precision_at_1000_max": -0.193372, + "nauc_precision_at_1000_std": 0.127344, + "nauc_precision_at_1000_diff1": -0.230863, + "nauc_mrr_at_1_max": 0.534045, + "nauc_mrr_at_1_std": -0.023658, + "nauc_mrr_at_1_diff1": 0.453446, + "nauc_mrr_at_3_max": 0.551258, + "nauc_mrr_at_3_std": 0.027438, + "nauc_mrr_at_3_diff1": 0.443809, + "nauc_mrr_at_5_max": 0.551497, + "nauc_mrr_at_5_std": 0.03394, + "nauc_mrr_at_5_diff1": 0.437685, + "nauc_mrr_at_10_max": 0.548744, + "nauc_mrr_at_10_std": 0.03226, + "nauc_mrr_at_10_diff1": 0.438889, + "nauc_mrr_at_20_max": 0.54953, + "nauc_mrr_at_20_std": 0.032038, + "nauc_mrr_at_20_diff1": 0.43965, + "nauc_mrr_at_100_max": 0.549359, + "nauc_mrr_at_100_std": 0.031004, + "nauc_mrr_at_100_diff1": 0.438776, + "nauc_mrr_at_1000_max": 0.549313, + "nauc_mrr_at_1000_std": 0.030643, + "nauc_mrr_at_1000_diff1": 0.439114, + "main_score": 0.64632, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 74.12310862541199, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/intfloat__e5-large-v2/b322e09026e4ea05f42beadf4d661fb4e101d311/model_meta.json b/results/intfloat__e5-large-v2/b322e09026e4ea05f42beadf4d661fb4e101d311/model_meta.json index 8f5b60ea6..5d1b1622a 100644 --- a/results/intfloat__e5-large-v2/b322e09026e4ea05f42beadf4d661fb4e101d311/model_meta.json +++ b/results/intfloat__e5-large-v2/b322e09026e4ea05f42beadf4d661fb4e101d311/model_meta.json @@ -1 +1 @@ -{"name": "intfloat/e5-large-v2", "revision": "b322e09026e4ea05f42beadf4d661fb4e101d311", "release_date": "2024-02-08", "languages": ["eng_Latn"], "n_parameters": 335000000, "max_tokens": 514.0, "embed_dim": 1024, "license": "mit", "open_weights": true, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/intfloat/e5-large-v2", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": {"MSMARCO": ["train"], "MSMARCOHardNegatives": ["train"], "NanoMSMARCORetrieval": ["train"], "MSMARCO-PL": ["train"], "NQ": ["train"], "NQHardNegatives": ["train"], "NanoNQRetrieval": ["train"], "NQ-PL": ["train"]}, "adapted_from": null, "superseded_by": null, "loader": "sentence_transformers_loader"} \ No newline at end of file +{"name": "intfloat/e5-large-v2", "revision": "b322e09026e4ea05f42beadf4d661fb4e101d311", "release_date": "2024-02-08", "languages": ["eng_Latn"], "n_parameters": 335000000, "max_tokens": 514.0, "embed_dim": 1024, "license": "mit", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/intfloat/e5-large-v2", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": {"MSMARCO": ["train"], "MSMARCOHardNegatives": ["train"], "NanoMSMARCORetrieval": ["train"], "MSMARCO-PL": ["train"], "mMARCO-NL": ["train"], "NQ": ["train"], "NQHardNegatives": ["train"], "NanoNQRetrieval": ["train"], "NQ-PL": ["train"], "NQ-NL": ["train"]}, "adapted_from": null, "superseded_by": null, "modalities": ["text"], "loader": "sentence_transformers_loader"} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-large-instruct/c9e87c786ffac96aeaeb42863276930883923ecb/BuiltBenchClusteringP2P.json b/results/intfloat__multilingual-e5-large-instruct/c9e87c786ffac96aeaeb42863276930883923ecb/BuiltBenchClusteringP2P.json new file mode 100644 index 000000000..12bc8acda --- /dev/null +++ b/results/intfloat__multilingual-e5-large-instruct/c9e87c786ffac96aeaeb42863276930883923ecb/BuiltBenchClusteringP2P.json @@ -0,0 +1,42 @@ +{ + "dataset_revision": "919bb71053e9de62a68998161ce4f0cee8f786fb", + "task_name": "BuiltBenchClusteringP2P", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.528174, + "v_measure_std": 0.149352, + "v_measures": [ + 0.354067, + 0.474886, + 0.483955, + 0.427402, + 0.375624, + 0.339462, + 0.382339, + 0.691649, + 0.814355, + 0.840635, + 0.604176, + 0.336176, + 0.59319, + 0.429986, + 0.488789, + 0.548958, + 0.635859, + 0.537493, + 0.473189, + 0.731288 + ], + "main_score": 0.528174, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 118.47219109535217, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-large-instruct/c9e87c786ffac96aeaeb42863276930883923ecb/BuiltBenchClusteringS2S.json b/results/intfloat__multilingual-e5-large-instruct/c9e87c786ffac96aeaeb42863276930883923ecb/BuiltBenchClusteringS2S.json new file mode 100644 index 000000000..cee40f177 --- /dev/null +++ b/results/intfloat__multilingual-e5-large-instruct/c9e87c786ffac96aeaeb42863276930883923ecb/BuiltBenchClusteringS2S.json @@ -0,0 +1,40 @@ +{ + "dataset_revision": "1aaeb2ece89ea0a8c64e215c95c4cfaf7e891149", + "task_name": "BuiltBenchClusteringS2S", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.480142, + "v_measure_std": 0.08465, + "v_measures": [ + 0.363104, + 0.609166, + 0.485797, + 0.548093, + 0.455281, + 0.508768, + 0.494266, + 0.537421, + 0.541662, + 0.601812, + 0.478376, + 0.47132, + 0.500542, + 0.491058, + 0.276968, + 0.312511, + 0.467953, + 0.498469 + ], + "main_score": 0.480142, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 17.057082653045654, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-large-instruct/c9e87c786ffac96aeaeb42863276930883923ecb/BuiltBenchReranking.json b/results/intfloat__multilingual-e5-large-instruct/c9e87c786ffac96aeaeb42863276930883923ecb/BuiltBenchReranking.json new file mode 100644 index 000000000..4b28709e6 --- /dev/null +++ b/results/intfloat__multilingual-e5-large-instruct/c9e87c786ffac96aeaeb42863276930883923ecb/BuiltBenchReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "fd33b0b3454deb256be06a57e8147b32ba078ff9", + "task_name": "BuiltBenchReranking", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "map": 0.658987, + "mrr": 0.820608, + "nAUC_map_max": 0.269419, + "nAUC_map_std": 0.528005, + "nAUC_map_diff1": -0.026623, + "nAUC_mrr_max": 0.23615, + "nAUC_mrr_std": 0.448529, + "nAUC_mrr_diff1": 0.189537, + "main_score": 0.658987, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 70.66350626945496, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-large-instruct/c9e87c786ffac96aeaeb42863276930883923ecb/BuiltBenchRetrieval.json b/results/intfloat__multilingual-e5-large-instruct/c9e87c786ffac96aeaeb42863276930883923ecb/BuiltBenchRetrieval.json new file mode 100644 index 000000000..819339bd4 --- /dev/null +++ b/results/intfloat__multilingual-e5-large-instruct/c9e87c786ffac96aeaeb42863276930883923ecb/BuiltBenchRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ae611238a58dae85f3130563fe9f9e995444a8d6", + "task_name": "BuiltBenchRetrieval", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.59281, + "ndcg_at_3": 0.60937, + "ndcg_at_5": 0.62747, + "ndcg_at_10": 0.64555, + "ndcg_at_20": 0.67011, + "ndcg_at_100": 0.71849, + "ndcg_at_1000": 0.74186, + "map_at_1": 0.28665, + "map_at_3": 0.41957, + "map_at_5": 0.47354, + "map_at_10": 0.52313, + "map_at_20": 0.56023, + "map_at_100": 0.59711, + "map_at_1000": 0.60285, + "recall_at_1": 0.28665, + "recall_at_3": 0.46146, + "recall_at_5": 0.56312, + "recall_at_10": 0.66053, + "recall_at_20": 0.75097, + "recall_at_100": 0.90994, + "recall_at_1000": 0.99505, + "precision_at_1": 0.59281, + "precision_at_3": 0.44012, + "precision_at_5": 0.36886, + "precision_at_10": 0.26737, + "precision_at_20": 0.18862, + "precision_at_100": 0.06251, + "precision_at_1000": 0.00815, + "mrr_at_1": 0.592814, + "mrr_at_3": 0.664172, + "mrr_at_5": 0.683782, + "mrr_at_10": 0.690667, + "mrr_at_20": 0.691882, + "mrr_at_100": 0.693865, + "mrr_at_1000": 0.694046, + "nauc_ndcg_at_1_max": 0.476243, + "nauc_ndcg_at_1_std": 0.0733, + "nauc_ndcg_at_1_diff1": 0.444054, + "nauc_ndcg_at_3_max": 0.46404, + "nauc_ndcg_at_3_std": 0.051669, + "nauc_ndcg_at_3_diff1": 0.411764, + "nauc_ndcg_at_5_max": 0.456827, + "nauc_ndcg_at_5_std": 0.04522, + "nauc_ndcg_at_5_diff1": 0.429039, + "nauc_ndcg_at_10_max": 0.449275, + "nauc_ndcg_at_10_std": 0.049436, + "nauc_ndcg_at_10_diff1": 0.426188, + "nauc_ndcg_at_20_max": 0.474076, + "nauc_ndcg_at_20_std": 0.092597, + "nauc_ndcg_at_20_diff1": 0.425437, + "nauc_ndcg_at_100_max": 0.471929, + "nauc_ndcg_at_100_std": 0.127594, + "nauc_ndcg_at_100_diff1": 0.42562, + "nauc_ndcg_at_1000_max": 0.477398, + "nauc_ndcg_at_1000_std": 0.114361, + "nauc_ndcg_at_1000_diff1": 0.419925, + "nauc_map_at_1_max": 0.337841, + "nauc_map_at_1_std": -0.167151, + "nauc_map_at_1_diff1": 0.567115, + "nauc_map_at_3_max": 0.395084, + "nauc_map_at_3_std": -0.12971, + "nauc_map_at_3_diff1": 0.480586, + "nauc_map_at_5_max": 0.426664, + "nauc_map_at_5_std": -0.100149, + "nauc_map_at_5_diff1": 0.467078, + "nauc_map_at_10_max": 0.424844, + "nauc_map_at_10_std": -0.063959, + "nauc_map_at_10_diff1": 0.453186, + "nauc_map_at_20_max": 0.441215, + "nauc_map_at_20_std": -0.010204, + "nauc_map_at_20_diff1": 0.444886, + "nauc_map_at_100_max": 0.460701, + "nauc_map_at_100_std": 0.045366, + "nauc_map_at_100_diff1": 0.439229, + "nauc_map_at_1000_max": 0.462787, + "nauc_map_at_1000_std": 0.050771, + "nauc_map_at_1000_diff1": 0.436545, + "nauc_recall_at_1_max": 0.337841, + "nauc_recall_at_1_std": -0.167151, + "nauc_recall_at_1_diff1": 0.567115, + "nauc_recall_at_3_max": 0.345368, + "nauc_recall_at_3_std": -0.140121, + "nauc_recall_at_3_diff1": 0.437783, + "nauc_recall_at_5_max": 0.332001, + "nauc_recall_at_5_std": -0.102003, + "nauc_recall_at_5_diff1": 0.397834, + "nauc_recall_at_10_max": 0.33292, + "nauc_recall_at_10_std": -0.078023, + "nauc_recall_at_10_diff1": 0.375216, + "nauc_recall_at_20_max": 0.386897, + "nauc_recall_at_20_std": 0.077639, + "nauc_recall_at_20_diff1": 0.381806, + "nauc_recall_at_100_max": 0.399231, + "nauc_recall_at_100_std": 0.370151, + "nauc_recall_at_100_diff1": 0.388326, + "nauc_recall_at_1000_max": 0.572966, + "nauc_recall_at_1000_std": 0.609953, + "nauc_recall_at_1000_diff1": -0.378056, + "nauc_precision_at_1_max": 0.476243, + "nauc_precision_at_1_std": 0.0733, + "nauc_precision_at_1_diff1": 0.444054, + "nauc_precision_at_3_max": 0.275909, + "nauc_precision_at_3_std": 0.196284, + "nauc_precision_at_3_diff1": -0.00628, + "nauc_precision_at_5_max": 0.195049, + "nauc_precision_at_5_std": 0.240608, + "nauc_precision_at_5_diff1": -0.081349, + "nauc_precision_at_10_max": 0.071539, + "nauc_precision_at_10_std": 0.270849, + "nauc_precision_at_10_diff1": -0.137789, + "nauc_precision_at_20_max": 0.033694, + "nauc_precision_at_20_std": 0.30661, + "nauc_precision_at_20_diff1": -0.165911, + "nauc_precision_at_100_max": -0.025678, + "nauc_precision_at_100_std": 0.310949, + "nauc_precision_at_100_diff1": -0.207522, + "nauc_precision_at_1000_max": -0.079547, + "nauc_precision_at_1000_std": 0.226692, + "nauc_precision_at_1000_diff1": -0.228219, + "nauc_mrr_at_1_max": 0.476243, + "nauc_mrr_at_1_std": 0.0733, + "nauc_mrr_at_1_diff1": 0.444054, + "nauc_mrr_at_3_max": 0.480265, + "nauc_mrr_at_3_std": 0.106116, + "nauc_mrr_at_3_diff1": 0.425814, + "nauc_mrr_at_5_max": 0.464966, + "nauc_mrr_at_5_std": 0.122717, + "nauc_mrr_at_5_diff1": 0.421744, + "nauc_mrr_at_10_max": 0.475949, + "nauc_mrr_at_10_std": 0.121001, + "nauc_mrr_at_10_diff1": 0.419819, + "nauc_mrr_at_20_max": 0.475889, + "nauc_mrr_at_20_std": 0.12285, + "nauc_mrr_at_20_diff1": 0.421273, + "nauc_mrr_at_100_max": 0.475742, + "nauc_mrr_at_100_std": 0.121316, + "nauc_mrr_at_100_diff1": 0.423483, + "nauc_mrr_at_1000_max": 0.475737, + "nauc_mrr_at_1000_std": 0.120879, + "nauc_mrr_at_1000_diff1": 0.423342, + "main_score": 0.64555, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 77.99413585662842, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-large-instruct/c9e87c786ffac96aeaeb42863276930883923ecb/model_meta.json b/results/intfloat__multilingual-e5-large-instruct/c9e87c786ffac96aeaeb42863276930883923ecb/model_meta.json new file mode 100644 index 000000000..38ea9c595 --- /dev/null +++ b/results/intfloat__multilingual-e5-large-instruct/c9e87c786ffac96aeaeb42863276930883923ecb/model_meta.json @@ -0,0 +1 @@ +{"name": "intfloat/multilingual-e5-large-instruct", "revision": "c9e87c786ffac96aeaeb42863276930883923ecb", "release_date": null, "languages": [], "n_parameters": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["Sentence Transformers"], "reference": null, "similarity_fn_name": "cosine", "use_instructions": null, "training_datasets": null, "adapted_from": null, "superseded_by": null, "modalities": ["text"], "loader": null} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/BuiltBenchClusteringP2P.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/BuiltBenchClusteringP2P.json new file mode 100644 index 000000000..41dfa98e0 --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/BuiltBenchClusteringP2P.json @@ -0,0 +1,42 @@ +{ + "dataset_revision": "919bb71053e9de62a68998161ce4f0cee8f786fb", + "task_name": "BuiltBenchClusteringP2P", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.481637, + "v_measure_std": 0.159361, + "v_measures": [ + 0.395816, + 0.385059, + 0.512107, + 0.429943, + 0.388138, + 0.480707, + 0.414581, + 0.820821, + 0.760083, + 0.237452, + 0.424245, + 0.346875, + 0.541195, + 0.365557, + 0.393135, + 0.468788, + 0.851506, + 0.325593, + 0.489086, + 0.602052 + ], + "main_score": 0.481637, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 14.45100736618042, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/BuiltBenchClusteringS2S.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/BuiltBenchClusteringS2S.json new file mode 100644 index 000000000..63b218062 --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/BuiltBenchClusteringS2S.json @@ -0,0 +1,40 @@ +{ + "dataset_revision": "1aaeb2ece89ea0a8c64e215c95c4cfaf7e891149", + "task_name": "BuiltBenchClusteringS2S", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.42976, + "v_measure_std": 0.120238, + "v_measures": [ + 0.325639, + 0.390028, + 0.539251, + 0.455128, + 0.427742, + 0.454103, + 0.451846, + 0.65115, + 0.38873, + 0.51801, + 0.478791, + 0.679437, + 0.502979, + 0.353408, + 0.200356, + 0.2763, + 0.323759, + 0.319028 + ], + "main_score": 0.42976, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.2568366527557373, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/BuiltBenchReranking.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/BuiltBenchReranking.json new file mode 100644 index 000000000..108262a83 --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/BuiltBenchReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "fd33b0b3454deb256be06a57e8147b32ba078ff9", + "task_name": "BuiltBenchReranking", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "map": 0.623398, + "mrr": 0.759193, + "nAUC_map_max": 0.326302, + "nAUC_map_std": 0.522297, + "nAUC_map_diff1": 0.021747, + "nAUC_mrr_max": 0.232314, + "nAUC_mrr_std": 0.317815, + "nAUC_mrr_diff1": 0.077531, + "main_score": 0.623398, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 6.771450519561768, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/BuiltBenchRetrieval.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/BuiltBenchRetrieval.json new file mode 100644 index 000000000..07efb9d37 --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/BuiltBenchRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ae611238a58dae85f3130563fe9f9e995444a8d6", + "task_name": "BuiltBenchRetrieval", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.51497, + "ndcg_at_3": 0.52112, + "ndcg_at_5": 0.53144, + "ndcg_at_10": 0.55031, + "ndcg_at_20": 0.57501, + "ndcg_at_100": 0.63325, + "ndcg_at_1000": 0.66355, + "map_at_1": 0.2399, + "map_at_3": 0.35393, + "map_at_5": 0.39422, + "map_at_10": 0.43413, + "map_at_20": 0.46569, + "map_at_100": 0.49879, + "map_at_1000": 0.50526, + "recall_at_1": 0.2399, + "recall_at_3": 0.40197, + "recall_at_5": 0.47469, + "recall_at_10": 0.56986, + "recall_at_20": 0.66455, + "recall_at_100": 0.85964, + "recall_at_1000": 0.97328, + "precision_at_1": 0.51497, + "precision_at_3": 0.36826, + "precision_at_5": 0.30539, + "precision_at_10": 0.22246, + "precision_at_20": 0.15853, + "precision_at_100": 0.05617, + "precision_at_1000": 0.0079, + "mrr_at_1": 0.51497, + "mrr_at_3": 0.586327, + "mrr_at_5": 0.599501, + "mrr_at_10": 0.607165, + "mrr_at_20": 0.610686, + "mrr_at_100": 0.613498, + "mrr_at_1000": 0.613702, + "nauc_ndcg_at_1_max": 0.629529, + "nauc_ndcg_at_1_std": 0.222627, + "nauc_ndcg_at_1_diff1": 0.479792, + "nauc_ndcg_at_3_max": 0.592382, + "nauc_ndcg_at_3_std": 0.206581, + "nauc_ndcg_at_3_diff1": 0.41724, + "nauc_ndcg_at_5_max": 0.582947, + "nauc_ndcg_at_5_std": 0.199325, + "nauc_ndcg_at_5_diff1": 0.407876, + "nauc_ndcg_at_10_max": 0.583206, + "nauc_ndcg_at_10_std": 0.205383, + "nauc_ndcg_at_10_diff1": 0.419204, + "nauc_ndcg_at_20_max": 0.597616, + "nauc_ndcg_at_20_std": 0.215161, + "nauc_ndcg_at_20_diff1": 0.429576, + "nauc_ndcg_at_100_max": 0.594511, + "nauc_ndcg_at_100_std": 0.253929, + "nauc_ndcg_at_100_diff1": 0.435818, + "nauc_ndcg_at_1000_max": 0.599459, + "nauc_ndcg_at_1000_std": 0.243828, + "nauc_ndcg_at_1000_diff1": 0.425857, + "nauc_map_at_1_max": 0.472463, + "nauc_map_at_1_std": 0.029191, + "nauc_map_at_1_diff1": 0.605275, + "nauc_map_at_3_max": 0.53785, + "nauc_map_at_3_std": 0.088388, + "nauc_map_at_3_diff1": 0.531558, + "nauc_map_at_5_max": 0.572828, + "nauc_map_at_5_std": 0.121642, + "nauc_map_at_5_diff1": 0.493026, + "nauc_map_at_10_max": 0.577143, + "nauc_map_at_10_std": 0.143082, + "nauc_map_at_10_diff1": 0.472395, + "nauc_map_at_20_max": 0.581415, + "nauc_map_at_20_std": 0.166236, + "nauc_map_at_20_diff1": 0.458382, + "nauc_map_at_100_max": 0.585066, + "nauc_map_at_100_std": 0.199552, + "nauc_map_at_100_diff1": 0.446576, + "nauc_map_at_1000_max": 0.586283, + "nauc_map_at_1000_std": 0.199792, + "nauc_map_at_1000_diff1": 0.444483, + "nauc_recall_at_1_max": 0.472463, + "nauc_recall_at_1_std": 0.029191, + "nauc_recall_at_1_diff1": 0.605275, + "nauc_recall_at_3_max": 0.473371, + "nauc_recall_at_3_std": 0.082708, + "nauc_recall_at_3_diff1": 0.469471, + "nauc_recall_at_5_max": 0.501644, + "nauc_recall_at_5_std": 0.120356, + "nauc_recall_at_5_diff1": 0.398847, + "nauc_recall_at_10_max": 0.47071, + "nauc_recall_at_10_std": 0.132678, + "nauc_recall_at_10_diff1": 0.359021, + "nauc_recall_at_20_max": 0.490973, + "nauc_recall_at_20_std": 0.162109, + "nauc_recall_at_20_diff1": 0.350181, + "nauc_recall_at_100_max": 0.513826, + "nauc_recall_at_100_std": 0.409712, + "nauc_recall_at_100_diff1": 0.38807, + "nauc_recall_at_1000_max": 0.610017, + "nauc_recall_at_1000_std": 0.774433, + "nauc_recall_at_1000_diff1": 0.344525, + "nauc_precision_at_1_max": 0.629529, + "nauc_precision_at_1_std": 0.222627, + "nauc_precision_at_1_diff1": 0.479792, + "nauc_precision_at_3_max": 0.43596, + "nauc_precision_at_3_std": 0.248416, + "nauc_precision_at_3_diff1": 0.082215, + "nauc_precision_at_5_max": 0.315923, + "nauc_precision_at_5_std": 0.239586, + "nauc_precision_at_5_diff1": -0.04147, + "nauc_precision_at_10_max": 0.165861, + "nauc_precision_at_10_std": 0.217093, + "nauc_precision_at_10_diff1": -0.123399, + "nauc_precision_at_20_max": 0.077088, + "nauc_precision_at_20_std": 0.187639, + "nauc_precision_at_20_diff1": -0.153577, + "nauc_precision_at_100_max": -0.049006, + "nauc_precision_at_100_std": 0.143801, + "nauc_precision_at_100_diff1": -0.18913, + "nauc_precision_at_1000_max": -0.108667, + "nauc_precision_at_1000_std": 0.039249, + "nauc_precision_at_1000_diff1": -0.206177, + "nauc_mrr_at_1_max": 0.629529, + "nauc_mrr_at_1_std": 0.222627, + "nauc_mrr_at_1_diff1": 0.479792, + "nauc_mrr_at_3_max": 0.616795, + "nauc_mrr_at_3_std": 0.253067, + "nauc_mrr_at_3_diff1": 0.449521, + "nauc_mrr_at_5_max": 0.614686, + "nauc_mrr_at_5_std": 0.257695, + "nauc_mrr_at_5_diff1": 0.443075, + "nauc_mrr_at_10_max": 0.612383, + "nauc_mrr_at_10_std": 0.251046, + "nauc_mrr_at_10_diff1": 0.44563, + "nauc_mrr_at_20_max": 0.612783, + "nauc_mrr_at_20_std": 0.248315, + "nauc_mrr_at_20_diff1": 0.446085, + "nauc_mrr_at_100_max": 0.613168, + "nauc_mrr_at_100_std": 0.248851, + "nauc_mrr_at_100_diff1": 0.446959, + "nauc_mrr_at_1000_max": 0.613278, + "nauc_mrr_at_1000_std": 0.248949, + "nauc_mrr_at_1000_diff1": 0.44692, + "main_score": 0.55031, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 7.072921514511108, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/model_meta.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/model_meta.json index 4a651b25c..1f4efb311 100644 --- a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/model_meta.json +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/model_meta.json @@ -1 +1 @@ -{"name": "intfloat/multilingual-e5-small", "revision": "fd1525a9fd15316a2d503bf26ab031a61d056e98", "release_date": "2024-02-08", "languages": ["afr_Latn", "amh_Latn", "ara_Latn", "asm_Latn", "aze_Latn", "bel_Latn", "bul_Latn", "ben_Latn", "ben_Beng", "bre_Latn", "bos_Latn", "cat_Latn", "ces_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "ell_Latn", "eng_Latn", "epo_Latn", "spa_Latn", "est_Latn", "eus_Latn", "fas_Latn", "fin_Latn", "fra_Latn", "fry_Latn", "gle_Latn", "gla_Latn", "glg_Latn", "guj_Latn", "hau_Latn", "heb_Latn", "hin_Latn", "hin_Deva", "hrv_Latn", "hun_Latn", "hye_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jpn_Latn", "jav_Latn", "kat_Latn", "kaz_Latn", "khm_Latn", "kan_Latn", "kor_Latn", "kur_Latn", "kir_Latn", "lat_Latn", "lao_Latn", "lit_Latn", "lav_Latn", "mlg_Latn", "mkd_Latn", "mal_Latn", "mon_Latn", "mar_Latn", "msa_Latn", "mya_Latn", "nep_Latn", "nld_Latn", "nob_Latn", "orm_Latn", "ori_Latn", "pan_Latn", "pol_Latn", "pus_Latn", "por_Latn", "ron_Latn", "rus_Latn", "san_Latn", "snd_Latn", "sin_Latn", "slk_Latn", "slv_Latn", "som_Latn", "sqi_Latn", "srp_Latn", "sun_Latn", "swe_Latn", "swa_Latn", "tam_Latn", "tam_Taml", "tel_Latn", "tel_Telu", "tha_Latn", "tgl_Latn", "tur_Latn", "uig_Latn", "ukr_Latn", "urd_Latn", "urd_Arab", "uzb_Latn", "vie_Latn", "xho_Latn", "yid_Latn", "zho_Hant", "zho_Hans"], "n_parameters": 118000000, "max_tokens": 512.0, "embed_dim": 384, "license": "mit", "open_weights": true, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/intfloat/multilingual-e5-small", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": {"MSMARCO": ["train"], "MSMARCOHardNegatives": ["train"], "NanoMSMARCORetrieval": ["train"], "MSMARCO-PL": ["train"], "NQ": ["train"], "NQHardNegatives": ["train"], "NanoNQRetrieval": ["train"], "NQ-PL": ["train"]}, "adapted_from": null, "superseded_by": null, "loader": "sentence_transformers_loader"} \ No newline at end of file +{"name": "intfloat/multilingual-e5-small", "revision": "fd1525a9fd15316a2d503bf26ab031a61d056e98", "release_date": "2024-02-08", "languages": ["afr_Latn", "amh_Latn", "ara_Latn", "asm_Latn", "aze_Latn", "bel_Latn", "bul_Latn", "ben_Latn", "ben_Beng", "bre_Latn", "bos_Latn", "cat_Latn", "ces_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "ell_Latn", "eng_Latn", "epo_Latn", "spa_Latn", "est_Latn", "eus_Latn", "fas_Latn", "fin_Latn", "fra_Latn", "fry_Latn", "gle_Latn", "gla_Latn", "glg_Latn", "guj_Latn", "hau_Latn", "heb_Latn", "hin_Latn", "hin_Deva", "hrv_Latn", "hun_Latn", "hye_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jpn_Latn", "jav_Latn", "kat_Latn", "kaz_Latn", "khm_Latn", "kan_Latn", "kor_Latn", "kur_Latn", "kir_Latn", "lat_Latn", "lao_Latn", "lit_Latn", "lav_Latn", "mlg_Latn", "mkd_Latn", "mal_Latn", "mon_Latn", "mar_Latn", "msa_Latn", "mya_Latn", "nep_Latn", "nld_Latn", "nob_Latn", "orm_Latn", "ori_Latn", "pan_Latn", "pol_Latn", "pus_Latn", "por_Latn", "ron_Latn", "rus_Latn", "san_Latn", "snd_Latn", "sin_Latn", "slk_Latn", "slv_Latn", "som_Latn", "sqi_Latn", "srp_Latn", "sun_Latn", "swe_Latn", "swa_Latn", "tam_Latn", "tam_Taml", "tel_Latn", "tel_Telu", "tha_Latn", "tgl_Latn", "tur_Latn", "uig_Latn", "ukr_Latn", "urd_Latn", "urd_Arab", "uzb_Latn", "vie_Latn", "xho_Latn", "yid_Latn", "zho_Hant", "zho_Hans"], "n_parameters": 118000000, "max_tokens": 512.0, "embed_dim": 384, "license": "mit", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/intfloat/multilingual-e5-small", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": {"MSMARCO": ["train"], "MSMARCOHardNegatives": ["train"], "NanoMSMARCORetrieval": ["train"], "MSMARCO-PL": ["train"], "mMARCO-NL": ["train"], "NQ": ["train"], "NQHardNegatives": ["train"], "NanoNQRetrieval": ["train"], "NQ-PL": ["train"], "NQ-NL": ["train"], "FEVER": ["train"], "FEVERHardNegatives": ["train"], "FEVER-NL": ["train"], "HotpotQA": ["train"], "HotpotQAHardNegatives": ["train"], "HotpotQA-PL": ["train"], "HotpotQA-NL": ["train"]}, "adapted_from": null, "superseded_by": null, "modalities": ["text"], "loader": "sentence_transformers_loader"} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/model_meta.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/model_meta.json index f3870aaf7..fb189bdb3 100644 --- a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/model_meta.json +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/model_meta.json @@ -1 +1 @@ -{"name": "mixedbread-ai/mxbai-embed-large-v1", "revision": "990580e27d329c7408b3741ecff85876e128e203", "release_date": "2024-03-07", "languages": ["eng_Latn"], "n_parameters": 335000000, "max_tokens": 512.0, "embed_dim": 1024, "license": "apache-2.0", "open_weights": true, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": null, "adapted_from": null, "superseded_by": null, "loader": "sentence_transformers_loader"} \ No newline at end of file +{"name": "mixedbread-ai/mxbai-embed-large-v1", "revision": "990580e27d329c7408b3741ecff85876e128e203", "release_date": "2024-03-07", "languages": ["eng_Latn"], "n_parameters": 335000000, "max_tokens": 512.0, "embed_dim": 1024, "license": "apache-2.0", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": null, "adapted_from": null, "superseded_by": null, "modalities": ["text"], "loader": "sentence_transformers_loader"} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/e7857440379da569f68f19e8403b69cd7be26e50/BuiltBenchClusteringP2P.json b/results/mixedbread-ai__mxbai-embed-large-v1/e7857440379da569f68f19e8403b69cd7be26e50/BuiltBenchClusteringP2P.json new file mode 100644 index 000000000..3c33a2f42 --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/e7857440379da569f68f19e8403b69cd7be26e50/BuiltBenchClusteringP2P.json @@ -0,0 +1,42 @@ +{ + "dataset_revision": "919bb71053e9de62a68998161ce4f0cee8f786fb", + "task_name": "BuiltBenchClusteringP2P", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.524472, + "v_measure_std": 0.123782, + "v_measures": [ + 0.403356, + 0.538866, + 0.564195, + 0.47365, + 0.368253, + 0.435216, + 0.597275, + 0.696816, + 0.739146, + 0.785778, + 0.625133, + 0.346875, + 0.517898, + 0.397557, + 0.379471, + 0.476666, + 0.575204, + 0.442652, + 0.484179, + 0.641256 + ], + "main_score": 0.524472, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 102.2237069606781, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/e7857440379da569f68f19e8403b69cd7be26e50/BuiltBenchClusteringS2S.json b/results/mixedbread-ai__mxbai-embed-large-v1/e7857440379da569f68f19e8403b69cd7be26e50/BuiltBenchClusteringS2S.json new file mode 100644 index 000000000..101e9026c --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/e7857440379da569f68f19e8403b69cd7be26e50/BuiltBenchClusteringS2S.json @@ -0,0 +1,40 @@ +{ + "dataset_revision": "1aaeb2ece89ea0a8c64e215c95c4cfaf7e891149", + "task_name": "BuiltBenchClusteringS2S", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.474893, + "v_measure_std": 0.129551, + "v_measures": [ + 0.324383, + 0.498541, + 0.37077, + 0.600596, + 0.42443, + 0.549042, + 0.360473, + 0.533933, + 0.464743, + 0.697135, + 0.585207, + 0.741065, + 0.404783, + 0.469589, + 0.247491, + 0.356135, + 0.350548, + 0.569202 + ], + "main_score": 0.474893, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 14.169285535812378, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/e7857440379da569f68f19e8403b69cd7be26e50/BuiltBenchReranking.json b/results/mixedbread-ai__mxbai-embed-large-v1/e7857440379da569f68f19e8403b69cd7be26e50/BuiltBenchReranking.json new file mode 100644 index 000000000..539d5a71d --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/e7857440379da569f68f19e8403b69cd7be26e50/BuiltBenchReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "fd33b0b3454deb256be06a57e8147b32ba078ff9", + "task_name": "BuiltBenchReranking", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "map": 0.696596, + "mrr": 0.87416, + "nAUC_map_max": 0.394611, + "nAUC_map_std": 0.485534, + "nAUC_map_diff1": 0.076013, + "nAUC_mrr_max": 0.290293, + "nAUC_mrr_std": 0.360305, + "nAUC_mrr_diff1": 0.090272, + "main_score": 0.696596, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 58.73047661781311, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/e7857440379da569f68f19e8403b69cd7be26e50/BuiltBenchRetrieval.json b/results/mixedbread-ai__mxbai-embed-large-v1/e7857440379da569f68f19e8403b69cd7be26e50/BuiltBenchRetrieval.json new file mode 100644 index 000000000..f84a8fe79 --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/e7857440379da569f68f19e8403b69cd7be26e50/BuiltBenchRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ae611238a58dae85f3130563fe9f9e995444a8d6", + "task_name": "BuiltBenchRetrieval", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.62575, + "ndcg_at_3": 0.62351, + "ndcg_at_5": 0.63677, + "ndcg_at_10": 0.66601, + "ndcg_at_20": 0.69029, + "ndcg_at_100": 0.73826, + "ndcg_at_1000": 0.75632, + "map_at_1": 0.29472, + "map_at_3": 0.4275, + "map_at_5": 0.48088, + "map_at_10": 0.53734, + "map_at_20": 0.57347, + "map_at_100": 0.61159, + "map_at_1000": 0.61752, + "recall_at_1": 0.29472, + "recall_at_3": 0.47584, + "recall_at_5": 0.5692, + "recall_at_10": 0.68699, + "recall_at_20": 0.77941, + "recall_at_100": 0.93718, + "recall_at_1000": 0.99903, + "precision_at_1": 0.62575, + "precision_at_3": 0.44411, + "precision_at_5": 0.36946, + "precision_at_10": 0.27874, + "precision_at_20": 0.19506, + "precision_at_100": 0.06545, + "precision_at_1000": 0.00822, + "mrr_at_1": 0.625749, + "mrr_at_3": 0.686627, + "mrr_at_5": 0.69995, + "mrr_at_10": 0.709804, + "mrr_at_20": 0.712143, + "mrr_at_100": 0.713668, + "mrr_at_1000": 0.713729, + "nauc_ndcg_at_1_max": 0.576702, + "nauc_ndcg_at_1_std": -0.094668, + "nauc_ndcg_at_1_diff1": 0.45686, + "nauc_ndcg_at_3_max": 0.543517, + "nauc_ndcg_at_3_std": -0.148536, + "nauc_ndcg_at_3_diff1": 0.42955, + "nauc_ndcg_at_5_max": 0.548391, + "nauc_ndcg_at_5_std": -0.150196, + "nauc_ndcg_at_5_diff1": 0.416523, + "nauc_ndcg_at_10_max": 0.574506, + "nauc_ndcg_at_10_std": -0.108343, + "nauc_ndcg_at_10_diff1": 0.422994, + "nauc_ndcg_at_20_max": 0.585495, + "nauc_ndcg_at_20_std": -0.080953, + "nauc_ndcg_at_20_diff1": 0.412151, + "nauc_ndcg_at_100_max": 0.578725, + "nauc_ndcg_at_100_std": -0.043202, + "nauc_ndcg_at_100_diff1": 0.420676, + "nauc_ndcg_at_1000_max": 0.568724, + "nauc_ndcg_at_1000_std": -0.069345, + "nauc_ndcg_at_1000_diff1": 0.417556, + "nauc_map_at_1_max": 0.439807, + "nauc_map_at_1_std": -0.199691, + "nauc_map_at_1_diff1": 0.57471, + "nauc_map_at_3_max": 0.493464, + "nauc_map_at_3_std": -0.215928, + "nauc_map_at_3_diff1": 0.523688, + "nauc_map_at_5_max": 0.527642, + "nauc_map_at_5_std": -0.194993, + "nauc_map_at_5_diff1": 0.491257, + "nauc_map_at_10_max": 0.554728, + "nauc_map_at_10_std": -0.162753, + "nauc_map_at_10_diff1": 0.472689, + "nauc_map_at_20_max": 0.569758, + "nauc_map_at_20_std": -0.137134, + "nauc_map_at_20_diff1": 0.455013, + "nauc_map_at_100_max": 0.581692, + "nauc_map_at_100_std": -0.097871, + "nauc_map_at_100_diff1": 0.440483, + "nauc_map_at_1000_max": 0.577782, + "nauc_map_at_1000_std": -0.098565, + "nauc_map_at_1000_diff1": 0.436376, + "nauc_recall_at_1_max": 0.439807, + "nauc_recall_at_1_std": -0.199691, + "nauc_recall_at_1_diff1": 0.57471, + "nauc_recall_at_3_max": 0.439412, + "nauc_recall_at_3_std": -0.223492, + "nauc_recall_at_3_diff1": 0.465373, + "nauc_recall_at_5_max": 0.44943, + "nauc_recall_at_5_std": -0.20507, + "nauc_recall_at_5_diff1": 0.412776, + "nauc_recall_at_10_max": 0.486393, + "nauc_recall_at_10_std": -0.152827, + "nauc_recall_at_10_diff1": 0.411557, + "nauc_recall_at_20_max": 0.518846, + "nauc_recall_at_20_std": -0.078562, + "nauc_recall_at_20_diff1": 0.358684, + "nauc_recall_at_100_max": 0.581758, + "nauc_recall_at_100_std": 0.287358, + "nauc_recall_at_100_diff1": 0.328029, + "nauc_recall_at_1000_max": 0.230787, + "nauc_recall_at_1000_std": 0.844638, + "nauc_recall_at_1000_diff1": 0.073144, + "nauc_precision_at_1_max": 0.576702, + "nauc_precision_at_1_std": -0.094668, + "nauc_precision_at_1_diff1": 0.45686, + "nauc_precision_at_3_max": 0.260329, + "nauc_precision_at_3_std": 0.013794, + "nauc_precision_at_3_diff1": -0.002215, + "nauc_precision_at_5_max": 0.178472, + "nauc_precision_at_5_std": 0.083961, + "nauc_precision_at_5_diff1": -0.121883, + "nauc_precision_at_10_max": 0.056991, + "nauc_precision_at_10_std": 0.168888, + "nauc_precision_at_10_diff1": -0.211967, + "nauc_precision_at_20_max": -0.02233, + "nauc_precision_at_20_std": 0.197012, + "nauc_precision_at_20_diff1": -0.253037, + "nauc_precision_at_100_max": -0.128769, + "nauc_precision_at_100_std": 0.203652, + "nauc_precision_at_100_diff1": -0.261418, + "nauc_precision_at_1000_max": -0.193074, + "nauc_precision_at_1000_std": 0.11628, + "nauc_precision_at_1000_diff1": -0.274281, + "nauc_mrr_at_1_max": 0.576702, + "nauc_mrr_at_1_std": -0.094668, + "nauc_mrr_at_1_diff1": 0.45686, + "nauc_mrr_at_3_max": 0.571105, + "nauc_mrr_at_3_std": -0.108716, + "nauc_mrr_at_3_diff1": 0.437331, + "nauc_mrr_at_5_max": 0.567134, + "nauc_mrr_at_5_std": -0.101158, + "nauc_mrr_at_5_diff1": 0.435083, + "nauc_mrr_at_10_max": 0.568323, + "nauc_mrr_at_10_std": -0.099441, + "nauc_mrr_at_10_diff1": 0.438551, + "nauc_mrr_at_20_max": 0.568656, + "nauc_mrr_at_20_std": -0.094926, + "nauc_mrr_at_20_diff1": 0.436663, + "nauc_mrr_at_100_max": 0.569126, + "nauc_mrr_at_100_std": -0.095528, + "nauc_mrr_at_100_diff1": 0.43789, + "nauc_mrr_at_1000_max": 0.569047, + "nauc_mrr_at_1000_std": -0.095769, + "nauc_mrr_at_1000_diff1": 0.437874, + "main_score": 0.66601, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 66.50235080718994, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/e7857440379da569f68f19e8403b69cd7be26e50/model_meta.json b/results/mixedbread-ai__mxbai-embed-large-v1/e7857440379da569f68f19e8403b69cd7be26e50/model_meta.json new file mode 100644 index 000000000..08a524498 --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/e7857440379da569f68f19e8403b69cd7be26e50/model_meta.json @@ -0,0 +1 @@ +{"name": "mixedbread-ai/mxbai-embed-large-v1", "revision": "e7857440379da569f68f19e8403b69cd7be26e50", "release_date": null, "languages": [], "n_parameters": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["Sentence Transformers"], "reference": null, "similarity_fn_name": "cosine", "use_instructions": null, "training_datasets": null, "adapted_from": null, "superseded_by": null, "modalities": ["text"], "loader": null} \ No newline at end of file diff --git a/results/sentence-transformers__all-MiniLM-L12-v2/364dd28d28dcd3359b537f3cf1f5348ba679da62/BuiltBenchClusteringP2P.json b/results/sentence-transformers__all-MiniLM-L12-v2/364dd28d28dcd3359b537f3cf1f5348ba679da62/BuiltBenchClusteringP2P.json new file mode 100644 index 000000000..652f554a0 --- /dev/null +++ b/results/sentence-transformers__all-MiniLM-L12-v2/364dd28d28dcd3359b537f3cf1f5348ba679da62/BuiltBenchClusteringP2P.json @@ -0,0 +1,42 @@ +{ + "dataset_revision": "919bb71053e9de62a68998161ce4f0cee8f786fb", + "task_name": "BuiltBenchClusteringP2P", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.465202, + "v_measure_std": 0.075582, + "v_measures": [ + 0.396326, + 0.420018, + 0.525416, + 0.405994, + 0.470459, + 0.469955, + 0.567013, + 0.461361, + 0.398172, + 0.547209, + 0.422024, + 0.608706, + 0.636513, + 0.438705, + 0.387035, + 0.445961, + 0.466299, + 0.367015, + 0.370801, + 0.499051 + ], + "main_score": 0.465202, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 10.39258861541748, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__all-MiniLM-L12-v2/364dd28d28dcd3359b537f3cf1f5348ba679da62/BuiltBenchClusteringS2S.json b/results/sentence-transformers__all-MiniLM-L12-v2/364dd28d28dcd3359b537f3cf1f5348ba679da62/BuiltBenchClusteringS2S.json new file mode 100644 index 000000000..b94982733 --- /dev/null +++ b/results/sentence-transformers__all-MiniLM-L12-v2/364dd28d28dcd3359b537f3cf1f5348ba679da62/BuiltBenchClusteringS2S.json @@ -0,0 +1,40 @@ +{ + "dataset_revision": "1aaeb2ece89ea0a8c64e215c95c4cfaf7e891149", + "task_name": "BuiltBenchClusteringS2S", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.419994, + "v_measure_std": 0.116448, + "v_measures": [ + 0.483244, + 0.508788, + 0.188413, + 0.328873, + 0.443664, + 0.522859, + 0.297361, + 0.568337, + 0.552256, + 0.514707, + 0.583364, + 0.381125, + 0.523143, + 0.393029, + 0.221255, + 0.345852, + 0.322179, + 0.381441 + ], + "main_score": 0.419994, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.1229214668273926, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__all-MiniLM-L12-v2/364dd28d28dcd3359b537f3cf1f5348ba679da62/BuiltBenchReranking.json b/results/sentence-transformers__all-MiniLM-L12-v2/364dd28d28dcd3359b537f3cf1f5348ba679da62/BuiltBenchReranking.json new file mode 100644 index 000000000..64ca3ff85 --- /dev/null +++ b/results/sentence-transformers__all-MiniLM-L12-v2/364dd28d28dcd3359b537f3cf1f5348ba679da62/BuiltBenchReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "fd33b0b3454deb256be06a57e8147b32ba078ff9", + "task_name": "BuiltBenchReranking", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "map": 0.639683, + "mrr": 0.784027, + "nAUC_map_max": 0.345001, + "nAUC_map_std": 0.592337, + "nAUC_map_diff1": 0.004484, + "nAUC_mrr_max": 0.378164, + "nAUC_mrr_std": 0.48592, + "nAUC_mrr_diff1": 0.139328, + "main_score": 0.639683, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 6.3445799350738525, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__all-MiniLM-L12-v2/364dd28d28dcd3359b537f3cf1f5348ba679da62/BuiltBenchRetrieval.json b/results/sentence-transformers__all-MiniLM-L12-v2/364dd28d28dcd3359b537f3cf1f5348ba679da62/BuiltBenchRetrieval.json new file mode 100644 index 000000000..f94fec8a6 --- /dev/null +++ b/results/sentence-transformers__all-MiniLM-L12-v2/364dd28d28dcd3359b537f3cf1f5348ba679da62/BuiltBenchRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ae611238a58dae85f3130563fe9f9e995444a8d6", + "task_name": "BuiltBenchRetrieval", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.54491, + "ndcg_at_3": 0.54742, + "ndcg_at_5": 0.55667, + "ndcg_at_10": 0.58811, + "ndcg_at_20": 0.61395, + "ndcg_at_100": 0.66653, + "ndcg_at_1000": 0.69593, + "map_at_1": 0.26292, + "map_at_3": 0.37587, + "map_at_5": 0.41792, + "map_at_10": 0.4662, + "map_at_20": 0.49919, + "map_at_100": 0.53433, + "map_at_1000": 0.54069, + "recall_at_1": 0.26292, + "recall_at_3": 0.42534, + "recall_at_5": 0.50097, + "recall_at_10": 0.6185, + "recall_at_20": 0.71463, + "recall_at_100": 0.87982, + "recall_at_1000": 0.99387, + "precision_at_1": 0.54491, + "precision_at_3": 0.38124, + "precision_at_5": 0.31257, + "precision_at_10": 0.23922, + "precision_at_20": 0.16961, + "precision_at_100": 0.05961, + "precision_at_1000": 0.00803, + "mrr_at_1": 0.54491, + "mrr_at_3": 0.621257, + "mrr_at_5": 0.631737, + "mrr_at_10": 0.641142, + "mrr_at_20": 0.644359, + "mrr_at_100": 0.646244, + "mrr_at_1000": 0.646465, + "nauc_ndcg_at_1_max": 0.645916, + "nauc_ndcg_at_1_std": 0.107596, + "nauc_ndcg_at_1_diff1": 0.576737, + "nauc_ndcg_at_3_max": 0.60804, + "nauc_ndcg_at_3_std": 0.08876, + "nauc_ndcg_at_3_diff1": 0.485178, + "nauc_ndcg_at_5_max": 0.614525, + "nauc_ndcg_at_5_std": 0.092442, + "nauc_ndcg_at_5_diff1": 0.489222, + "nauc_ndcg_at_10_max": 0.636316, + "nauc_ndcg_at_10_std": 0.107955, + "nauc_ndcg_at_10_diff1": 0.480683, + "nauc_ndcg_at_20_max": 0.647138, + "nauc_ndcg_at_20_std": 0.122186, + "nauc_ndcg_at_20_diff1": 0.486445, + "nauc_ndcg_at_100_max": 0.647551, + "nauc_ndcg_at_100_std": 0.167581, + "nauc_ndcg_at_100_diff1": 0.485508, + "nauc_ndcg_at_1000_max": 0.640834, + "nauc_ndcg_at_1000_std": 0.141537, + "nauc_ndcg_at_1000_diff1": 0.490407, + "nauc_map_at_1_max": 0.504358, + "nauc_map_at_1_std": -0.045533, + "nauc_map_at_1_diff1": 0.640538, + "nauc_map_at_3_max": 0.544548, + "nauc_map_at_3_std": -0.03073, + "nauc_map_at_3_diff1": 0.549912, + "nauc_map_at_5_max": 0.57908, + "nauc_map_at_5_std": -1.2e-05, + "nauc_map_at_5_diff1": 0.534099, + "nauc_map_at_10_max": 0.603501, + "nauc_map_at_10_std": 0.026869, + "nauc_map_at_10_diff1": 0.517198, + "nauc_map_at_20_max": 0.619261, + "nauc_map_at_20_std": 0.053226, + "nauc_map_at_20_diff1": 0.503793, + "nauc_map_at_100_max": 0.631174, + "nauc_map_at_100_std": 0.098405, + "nauc_map_at_100_diff1": 0.491876, + "nauc_map_at_1000_max": 0.632163, + "nauc_map_at_1000_std": 0.100318, + "nauc_map_at_1000_diff1": 0.49238, + "nauc_recall_at_1_max": 0.504358, + "nauc_recall_at_1_std": -0.045533, + "nauc_recall_at_1_diff1": 0.640538, + "nauc_recall_at_3_max": 0.488589, + "nauc_recall_at_3_std": -0.028041, + "nauc_recall_at_3_diff1": 0.491985, + "nauc_recall_at_5_max": 0.520845, + "nauc_recall_at_5_std": 0.02005, + "nauc_recall_at_5_diff1": 0.479315, + "nauc_recall_at_10_max": 0.542736, + "nauc_recall_at_10_std": 0.066436, + "nauc_recall_at_10_diff1": 0.422472, + "nauc_recall_at_20_max": 0.560687, + "nauc_recall_at_20_std": 0.11657, + "nauc_recall_at_20_diff1": 0.397574, + "nauc_recall_at_100_max": 0.619655, + "nauc_recall_at_100_std": 0.408307, + "nauc_recall_at_100_diff1": 0.396287, + "nauc_recall_at_1000_max": 0.388777, + "nauc_recall_at_1000_std": 0.720763, + "nauc_recall_at_1000_diff1": -0.004264, + "nauc_precision_at_1_max": 0.645916, + "nauc_precision_at_1_std": 0.107596, + "nauc_precision_at_1_diff1": 0.576737, + "nauc_precision_at_3_max": 0.385331, + "nauc_precision_at_3_std": 0.155134, + "nauc_precision_at_3_diff1": 0.089646, + "nauc_precision_at_5_max": 0.295978, + "nauc_precision_at_5_std": 0.172261, + "nauc_precision_at_5_diff1": -0.014012, + "nauc_precision_at_10_max": 0.149954, + "nauc_precision_at_10_std": 0.190283, + "nauc_precision_at_10_diff1": -0.140279, + "nauc_precision_at_20_max": 0.063663, + "nauc_precision_at_20_std": 0.203904, + "nauc_precision_at_20_diff1": -0.180041, + "nauc_precision_at_100_max": -0.035089, + "nauc_precision_at_100_std": 0.226995, + "nauc_precision_at_100_diff1": -0.182847, + "nauc_precision_at_1000_max": -0.112072, + "nauc_precision_at_1000_std": 0.108308, + "nauc_precision_at_1000_diff1": -0.198457, + "nauc_mrr_at_1_max": 0.645916, + "nauc_mrr_at_1_std": 0.107596, + "nauc_mrr_at_1_diff1": 0.576737, + "nauc_mrr_at_3_max": 0.64761, + "nauc_mrr_at_3_std": 0.129929, + "nauc_mrr_at_3_diff1": 0.538637, + "nauc_mrr_at_5_max": 0.645862, + "nauc_mrr_at_5_std": 0.131034, + "nauc_mrr_at_5_diff1": 0.544598, + "nauc_mrr_at_10_max": 0.644958, + "nauc_mrr_at_10_std": 0.134765, + "nauc_mrr_at_10_diff1": 0.535675, + "nauc_mrr_at_20_max": 0.644737, + "nauc_mrr_at_20_std": 0.134543, + "nauc_mrr_at_20_diff1": 0.53782, + "nauc_mrr_at_100_max": 0.645594, + "nauc_mrr_at_100_std": 0.134796, + "nauc_mrr_at_100_diff1": 0.539399, + "nauc_mrr_at_1000_max": 0.645473, + "nauc_mrr_at_1000_std": 0.134497, + "nauc_mrr_at_1000_diff1": 0.539376, + "main_score": 0.58811, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 6.83296799659729, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__all-MiniLM-L12-v2/364dd28d28dcd3359b537f3cf1f5348ba679da62/model_meta.json b/results/sentence-transformers__all-MiniLM-L12-v2/364dd28d28dcd3359b537f3cf1f5348ba679da62/model_meta.json index 043e7a27a..4559a12d0 100644 --- a/results/sentence-transformers__all-MiniLM-L12-v2/364dd28d28dcd3359b537f3cf1f5348ba679da62/model_meta.json +++ b/results/sentence-transformers__all-MiniLM-L12-v2/364dd28d28dcd3359b537f3cf1f5348ba679da62/model_meta.json @@ -1 +1 @@ -{"name": "sentence-transformers/all-MiniLM-L12-v2", "revision": "364dd28d28dcd3359b537f3cf1f5348ba679da62", "release_date": "2021-08-30", "languages": ["eng-Latn"], "n_parameters": 33400000, "max_tokens": 256.0, "embed_dim": 384, "license": "apache-2.0", "open_weights": true, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/sentence-transformers/all-MiniLM-L12-v2", "similarity_fn_name": "cosine", "use_instructions": false, "training_datasets": {"MSMARCO": ["train"], "MSMARCOHardNegatives": ["train"], "NanoMSMARCORetrieval": ["train"], "MSMARCO-PL": ["train"], "NQ": ["train"], "NQHardNegatives": ["train"], "NanoNQRetrieval": ["train"], "NQ-PL": ["train"]}, "adapted_from": null, "superseded_by": null, "loader": null} \ No newline at end of file +{"name": "sentence-transformers/all-MiniLM-L12-v2", "revision": "364dd28d28dcd3359b537f3cf1f5348ba679da62", "release_date": "2021-08-30", "languages": ["eng-Latn"], "n_parameters": 33400000, "max_tokens": 256.0, "embed_dim": 384, "license": "apache-2.0", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/sentence-transformers/all-MiniLM-L12-v2", "similarity_fn_name": "cosine", "use_instructions": false, "training_datasets": {"MSMARCO": ["train"], "MSMARCOHardNegatives": ["train"], "NanoMSMARCORetrieval": ["train"], "MSMARCO-PL": ["train"], "mMARCO-NL": ["train"], "NQ": ["train"], "NQHardNegatives": ["train"], "NanoNQRetrieval": ["train"], "NQ-PL": ["train"], "NQ-NL": ["train"]}, "adapted_from": null, "superseded_by": null, "modalities": ["text"], "loader": null} \ No newline at end of file diff --git a/results/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/bf3bf13ab40c3157080a7ab344c831b9ad18b5eb/BuiltBenchClusteringP2P.json b/results/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/bf3bf13ab40c3157080a7ab344c831b9ad18b5eb/BuiltBenchClusteringP2P.json new file mode 100644 index 000000000..4454ecf3c --- /dev/null +++ b/results/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/bf3bf13ab40c3157080a7ab344c831b9ad18b5eb/BuiltBenchClusteringP2P.json @@ -0,0 +1,42 @@ +{ + "dataset_revision": "919bb71053e9de62a68998161ce4f0cee8f786fb", + "task_name": "BuiltBenchClusteringP2P", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.457033, + "v_measure_std": 0.182393, + "v_measures": [ + 0.315783, + 0.492199, + 0.488347, + 0.462557, + 0.521076, + 0.388175, + 0.699785, + 0.62834, + 0.886459, + 0.005402, + 0.48816, + 0.682356, + 0.441454, + 0.334666, + 0.34753, + 0.323048, + 0.249124, + 0.359226, + 0.485359, + 0.541619 + ], + "main_score": 0.457033, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 9.86427116394043, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/bf3bf13ab40c3157080a7ab344c831b9ad18b5eb/BuiltBenchClusteringS2S.json b/results/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/bf3bf13ab40c3157080a7ab344c831b9ad18b5eb/BuiltBenchClusteringS2S.json new file mode 100644 index 000000000..95c730d85 --- /dev/null +++ b/results/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/bf3bf13ab40c3157080a7ab344c831b9ad18b5eb/BuiltBenchClusteringS2S.json @@ -0,0 +1,40 @@ +{ + "dataset_revision": "1aaeb2ece89ea0a8c64e215c95c4cfaf7e891149", + "task_name": "BuiltBenchClusteringS2S", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.376034, + "v_measure_std": 0.085628, + "v_measures": [ + 0.300732, + 0.331486, + 0.373288, + 0.310973, + 0.289519, + 0.383828, + 0.46632, + 0.467054, + 0.471431, + 0.455272, + 0.537733, + 0.346126, + 0.43439, + 0.368209, + 0.197339, + 0.243762, + 0.392267, + 0.398887 + ], + "main_score": 0.376034, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.268704891204834, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/bf3bf13ab40c3157080a7ab344c831b9ad18b5eb/BuiltBenchReranking.json b/results/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/bf3bf13ab40c3157080a7ab344c831b9ad18b5eb/BuiltBenchReranking.json new file mode 100644 index 000000000..11105cc73 --- /dev/null +++ b/results/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/bf3bf13ab40c3157080a7ab344c831b9ad18b5eb/BuiltBenchReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "fd33b0b3454deb256be06a57e8147b32ba078ff9", + "task_name": "BuiltBenchReranking", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "map": 0.591499, + "mrr": 0.753547, + "nAUC_map_max": 0.427153, + "nAUC_map_std": 0.667345, + "nAUC_map_diff1": 0.10948, + "nAUC_mrr_max": 0.387433, + "nAUC_mrr_std": 0.546869, + "nAUC_mrr_diff1": 0.200512, + "main_score": 0.591499, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 7.381944179534912, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/bf3bf13ab40c3157080a7ab344c831b9ad18b5eb/BuiltBenchRetrieval.json b/results/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/bf3bf13ab40c3157080a7ab344c831b9ad18b5eb/BuiltBenchRetrieval.json new file mode 100644 index 000000000..bd6abec91 --- /dev/null +++ b/results/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/bf3bf13ab40c3157080a7ab344c831b9ad18b5eb/BuiltBenchRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ae611238a58dae85f3130563fe9f9e995444a8d6", + "task_name": "BuiltBenchRetrieval", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.45808, + "ndcg_at_3": 0.46043, + "ndcg_at_5": 0.4694, + "ndcg_at_10": 0.49897, + "ndcg_at_20": 0.52038, + "ndcg_at_100": 0.5786, + "ndcg_at_1000": 0.61939, + "map_at_1": 0.20926, + "map_at_3": 0.30637, + "map_at_5": 0.34122, + "map_at_10": 0.382, + "map_at_20": 0.40988, + "map_at_100": 0.44161, + "map_at_1000": 0.44834, + "recall_at_1": 0.20926, + "recall_at_3": 0.35334, + "recall_at_5": 0.41898, + "recall_at_10": 0.53508, + "recall_at_20": 0.61854, + "recall_at_100": 0.8222, + "recall_at_1000": 0.98398, + "precision_at_1": 0.45808, + "precision_at_3": 0.32834, + "precision_at_5": 0.27246, + "precision_at_10": 0.20749, + "precision_at_20": 0.14641, + "precision_at_100": 0.0529, + "precision_at_1000": 0.00797, + "mrr_at_1": 0.458084, + "mrr_at_3": 0.532435, + "mrr_at_5": 0.543064, + "mrr_at_10": 0.555106, + "mrr_at_20": 0.556941, + "mrr_at_100": 0.56016, + "mrr_at_1000": 0.560546, + "nauc_ndcg_at_1_max": 0.520709, + "nauc_ndcg_at_1_std": 0.038274, + "nauc_ndcg_at_1_diff1": 0.427829, + "nauc_ndcg_at_3_max": 0.506456, + "nauc_ndcg_at_3_std": 0.001909, + "nauc_ndcg_at_3_diff1": 0.411515, + "nauc_ndcg_at_5_max": 0.50233, + "nauc_ndcg_at_5_std": 0.001786, + "nauc_ndcg_at_5_diff1": 0.409546, + "nauc_ndcg_at_10_max": 0.521462, + "nauc_ndcg_at_10_std": 0.031411, + "nauc_ndcg_at_10_diff1": 0.401043, + "nauc_ndcg_at_20_max": 0.523823, + "nauc_ndcg_at_20_std": 0.054644, + "nauc_ndcg_at_20_diff1": 0.392504, + "nauc_ndcg_at_100_max": 0.517788, + "nauc_ndcg_at_100_std": 0.091473, + "nauc_ndcg_at_100_diff1": 0.379062, + "nauc_ndcg_at_1000_max": 0.516207, + "nauc_ndcg_at_1000_std": 0.055922, + "nauc_ndcg_at_1000_diff1": 0.375381, + "nauc_map_at_1_max": 0.398523, + "nauc_map_at_1_std": -0.106204, + "nauc_map_at_1_diff1": 0.545034, + "nauc_map_at_3_max": 0.431609, + "nauc_map_at_3_std": -0.103801, + "nauc_map_at_3_diff1": 0.468316, + "nauc_map_at_5_max": 0.454972, + "nauc_map_at_5_std": -0.087368, + "nauc_map_at_5_diff1": 0.451592, + "nauc_map_at_10_max": 0.478668, + "nauc_map_at_10_std": -0.051651, + "nauc_map_at_10_diff1": 0.435919, + "nauc_map_at_20_max": 0.494545, + "nauc_map_at_20_std": -0.015079, + "nauc_map_at_20_diff1": 0.424267, + "nauc_map_at_100_max": 0.510879, + "nauc_map_at_100_std": 0.029953, + "nauc_map_at_100_diff1": 0.409794, + "nauc_map_at_1000_max": 0.515882, + "nauc_map_at_1000_std": 0.028924, + "nauc_map_at_1000_diff1": 0.408681, + "nauc_recall_at_1_max": 0.398523, + "nauc_recall_at_1_std": -0.106204, + "nauc_recall_at_1_diff1": 0.545034, + "nauc_recall_at_3_max": 0.380169, + "nauc_recall_at_3_std": -0.104038, + "nauc_recall_at_3_diff1": 0.417298, + "nauc_recall_at_5_max": 0.39641, + "nauc_recall_at_5_std": -0.071807, + "nauc_recall_at_5_diff1": 0.385179, + "nauc_recall_at_10_max": 0.408975, + "nauc_recall_at_10_std": -0.005154, + "nauc_recall_at_10_diff1": 0.354826, + "nauc_recall_at_20_max": 0.424172, + "nauc_recall_at_20_std": 0.067732, + "nauc_recall_at_20_diff1": 0.327879, + "nauc_recall_at_100_max": 0.438111, + "nauc_recall_at_100_std": 0.31585, + "nauc_recall_at_100_diff1": 0.278374, + "nauc_recall_at_1000_max": 0.38795, + "nauc_recall_at_1000_std": 0.514047, + "nauc_recall_at_1000_diff1": -0.021729, + "nauc_precision_at_1_max": 0.520709, + "nauc_precision_at_1_std": 0.038274, + "nauc_precision_at_1_diff1": 0.427829, + "nauc_precision_at_3_max": 0.400483, + "nauc_precision_at_3_std": 0.07404, + "nauc_precision_at_3_diff1": 0.15633, + "nauc_precision_at_5_max": 0.341023, + "nauc_precision_at_5_std": 0.112327, + "nauc_precision_at_5_diff1": 0.063788, + "nauc_precision_at_10_max": 0.271572, + "nauc_precision_at_10_std": 0.200805, + "nauc_precision_at_10_diff1": -0.030053, + "nauc_precision_at_20_max": 0.206321, + "nauc_precision_at_20_std": 0.253125, + "nauc_precision_at_20_diff1": -0.081034, + "nauc_precision_at_100_max": 0.138419, + "nauc_precision_at_100_std": 0.249559, + "nauc_precision_at_100_diff1": -0.1341, + "nauc_precision_at_1000_max": 0.043562, + "nauc_precision_at_1000_std": 0.06604, + "nauc_precision_at_1000_diff1": -0.142558, + "nauc_mrr_at_1_max": 0.520709, + "nauc_mrr_at_1_std": 0.038274, + "nauc_mrr_at_1_diff1": 0.427829, + "nauc_mrr_at_3_max": 0.503186, + "nauc_mrr_at_3_std": 0.031707, + "nauc_mrr_at_3_diff1": 0.407614, + "nauc_mrr_at_5_max": 0.504159, + "nauc_mrr_at_5_std": 0.039277, + "nauc_mrr_at_5_diff1": 0.398826, + "nauc_mrr_at_10_max": 0.506626, + "nauc_mrr_at_10_std": 0.043922, + "nauc_mrr_at_10_diff1": 0.394976, + "nauc_mrr_at_20_max": 0.50735, + "nauc_mrr_at_20_std": 0.045781, + "nauc_mrr_at_20_diff1": 0.394745, + "nauc_mrr_at_100_max": 0.507205, + "nauc_mrr_at_100_std": 0.045596, + "nauc_mrr_at_100_diff1": 0.395837, + "nauc_mrr_at_1000_max": 0.507271, + "nauc_mrr_at_1000_std": 0.045336, + "nauc_mrr_at_1000_diff1": 0.395918, + "main_score": 0.49897, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 7.859208345413208, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/bf3bf13ab40c3157080a7ab344c831b9ad18b5eb/model_meta.json b/results/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/bf3bf13ab40c3157080a7ab344c831b9ad18b5eb/model_meta.json index da83080f6..5bbd26ac8 100644 --- a/results/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/bf3bf13ab40c3157080a7ab344c831b9ad18b5eb/model_meta.json +++ b/results/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/bf3bf13ab40c3157080a7ab344c831b9ad18b5eb/model_meta.json @@ -1 +1 @@ -{"name": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", "revision": "bf3bf13ab40c3157080a7ab344c831b9ad18b5eb", "release_date": "2019-11-01", "languages": ["ara_Arab", "bul_Cyrl", "cat_Latn", "ces_Latn", "dan_Latn", "deu_Latn", "ell_Grek", "eng_Latn", "spa_Latn", "est_Latn", "fas_Arab", "fin_Latn", "fra_Latn", "fra_Latn", "glg_Latn", "guj_Gujr", "heb_Hebr", "hin_Deva", "hrv_Latn", "hun_Latn", "hye_Armn", "ind_Latn", "ita_Latn", "jpn_Jpan", "kat_Geor", "kor_Hang", "kur_Arab", "lit_Latn", "lav_Latn", "mkd_Cyrl", "mon_Cyrl", "mar_Deva", "msa_Latn", "mya_Mymr", "nob_Latn", "nld_Latn", "pol_Latn", "por_Latn", "por_Latn", "ron_Latn", "rus_Cyrl", "slk_Latn", "slv_Latn", "sqi_Latn", "srp_Cyrl", "swe_Latn", "tha_Thai", "tur_Latn", "ukr_Cyrl", "urd_Arab", "vie_Latn", "zho_Hans", "zho_Hant"], "n_parameters": 118000000, "max_tokens": 512.0, "embed_dim": 768, "license": "apache-2.0", "open_weights": true, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", "similarity_fn_name": "cosine", "use_instructions": false, "training_datasets": {"MSMARCO": ["train"], "MSMARCOHardNegatives": ["train"], "NanoMSMARCORetrieval": ["train"], "MSMARCO-PL": ["train"], "NQ": ["train"], "NQHardNegatives": ["train"], "NanoNQRetrieval": ["train"], "NQ-PL": ["train"]}, "adapted_from": null, "superseded_by": null, "loader": null} \ No newline at end of file +{"name": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", "revision": "bf3bf13ab40c3157080a7ab344c831b9ad18b5eb", "release_date": "2019-11-01", "languages": ["ara_Arab", "bul_Cyrl", "cat_Latn", "ces_Latn", "dan_Latn", "deu_Latn", "ell_Grek", "eng_Latn", "spa_Latn", "est_Latn", "fas_Arab", "fin_Latn", "fra_Latn", "fra_Latn", "glg_Latn", "guj_Gujr", "heb_Hebr", "hin_Deva", "hrv_Latn", "hun_Latn", "hye_Armn", "ind_Latn", "ita_Latn", "jpn_Jpan", "kat_Geor", "kor_Hang", "kur_Arab", "lit_Latn", "lav_Latn", "mkd_Cyrl", "mon_Cyrl", "mar_Deva", "msa_Latn", "mya_Mymr", "nob_Latn", "nld_Latn", "pol_Latn", "por_Latn", "por_Latn", "ron_Latn", "rus_Cyrl", "slk_Latn", "slv_Latn", "sqi_Latn", "srp_Cyrl", "swe_Latn", "tha_Thai", "tur_Latn", "ukr_Cyrl", "urd_Arab", "vie_Latn", "zho_Hans", "zho_Hant"], "n_parameters": 118000000, "max_tokens": 512.0, "embed_dim": 768, "license": "apache-2.0", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", "similarity_fn_name": "cosine", "use_instructions": false, "training_datasets": {"MSMARCO": ["train"], "MSMARCOHardNegatives": ["train"], "NanoMSMARCORetrieval": ["train"], "MSMARCO-PL": ["train"], "mMARCO-NL": ["train"], "NQ": ["train"], "NQHardNegatives": ["train"], "NanoNQRetrieval": ["train"], "NQ-PL": ["train"], "NQ-NL": ["train"]}, "adapted_from": null, "superseded_by": null, "modalities": ["text"], "loader": null} \ No newline at end of file diff --git a/results/thenlper__gte-large/4bef63f39fcc5e2d6b0aae83089f307af4970164/BuiltBenchClusteringP2P.json b/results/thenlper__gte-large/4bef63f39fcc5e2d6b0aae83089f307af4970164/BuiltBenchClusteringP2P.json new file mode 100644 index 000000000..9d60e98c4 --- /dev/null +++ b/results/thenlper__gte-large/4bef63f39fcc5e2d6b0aae83089f307af4970164/BuiltBenchClusteringP2P.json @@ -0,0 +1,42 @@ +{ + "dataset_revision": "919bb71053e9de62a68998161ce4f0cee8f786fb", + "task_name": "BuiltBenchClusteringP2P", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.552368, + "v_measure_std": 0.152209, + "v_measures": [ + 0.387989, + 0.610574, + 0.375192, + 0.396693, + 0.426001, + 0.485597, + 0.511027, + 0.701069, + 0.784321, + 0.761026, + 0.617684, + 0.727065, + 0.501644, + 0.376522, + 0.359955, + 0.47665, + 0.867352, + 0.543496, + 0.457256, + 0.680251 + ], + "main_score": 0.552368, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 93.08124113082886, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/thenlper__gte-large/4bef63f39fcc5e2d6b0aae83089f307af4970164/BuiltBenchClusteringS2S.json b/results/thenlper__gte-large/4bef63f39fcc5e2d6b0aae83089f307af4970164/BuiltBenchClusteringS2S.json new file mode 100644 index 000000000..4ee0d2674 --- /dev/null +++ b/results/thenlper__gte-large/4bef63f39fcc5e2d6b0aae83089f307af4970164/BuiltBenchClusteringS2S.json @@ -0,0 +1,40 @@ +{ + "dataset_revision": "1aaeb2ece89ea0a8c64e215c95c4cfaf7e891149", + "task_name": "BuiltBenchClusteringS2S", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.48545, + "v_measure_std": 0.132422, + "v_measures": [ + 0.357168, + 0.475529, + 0.458679, + 0.520568, + 0.425878, + 0.504853, + 0.25882, + 0.677642, + 0.631691, + 0.664905, + 0.583861, + 0.741065, + 0.456305, + 0.367581, + 0.252162, + 0.401648, + 0.475066, + 0.484676 + ], + "main_score": 0.48545, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 16.030216455459595, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/thenlper__gte-large/4bef63f39fcc5e2d6b0aae83089f307af4970164/BuiltBenchReranking.json b/results/thenlper__gte-large/4bef63f39fcc5e2d6b0aae83089f307af4970164/BuiltBenchReranking.json new file mode 100644 index 000000000..980cf2d05 --- /dev/null +++ b/results/thenlper__gte-large/4bef63f39fcc5e2d6b0aae83089f307af4970164/BuiltBenchReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "fd33b0b3454deb256be06a57e8147b32ba078ff9", + "task_name": "BuiltBenchReranking", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "map": 0.692512, + "mrr": 0.855454, + "nAUC_map_max": 0.406467, + "nAUC_map_std": 0.535486, + "nAUC_map_diff1": 0.081108, + "nAUC_mrr_max": 0.306746, + "nAUC_mrr_std": 0.335101, + "nAUC_mrr_diff1": 0.229935, + "main_score": 0.692512, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 61.02307176589966, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/thenlper__gte-large/4bef63f39fcc5e2d6b0aae83089f307af4970164/BuiltBenchRetrieval.json b/results/thenlper__gte-large/4bef63f39fcc5e2d6b0aae83089f307af4970164/BuiltBenchRetrieval.json new file mode 100644 index 000000000..e43ee68d1 --- /dev/null +++ b/results/thenlper__gte-large/4bef63f39fcc5e2d6b0aae83089f307af4970164/BuiltBenchRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ae611238a58dae85f3130563fe9f9e995444a8d6", + "task_name": "BuiltBenchRetrieval", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.61377, + "ndcg_at_3": 0.6227, + "ndcg_at_5": 0.63913, + "ndcg_at_10": 0.6608, + "ndcg_at_20": 0.6882, + "ndcg_at_100": 0.73319, + "ndcg_at_1000": 0.75306, + "map_at_1": 0.29127, + "map_at_3": 0.42668, + "map_at_5": 0.48272, + "map_at_10": 0.53548, + "map_at_20": 0.57484, + "map_at_100": 0.61137, + "map_at_1000": 0.61733, + "recall_at_1": 0.29127, + "recall_at_3": 0.47586, + "recall_at_5": 0.57266, + "recall_at_10": 0.67444, + "recall_at_20": 0.77992, + "recall_at_100": 0.92588, + "recall_at_1000": 0.99766, + "precision_at_1": 0.61377, + "precision_at_3": 0.4481, + "precision_at_5": 0.37605, + "precision_at_10": 0.27874, + "precision_at_20": 0.19581, + "precision_at_100": 0.06542, + "precision_at_1000": 0.00822, + "mrr_at_1": 0.613772, + "mrr_at_3": 0.679641, + "mrr_at_5": 0.693563, + "mrr_at_10": 0.701287, + "mrr_at_20": 0.70431, + "mrr_at_100": 0.705863, + "mrr_at_1000": 0.706025, + "nauc_ndcg_at_1_max": 0.528465, + "nauc_ndcg_at_1_std": -0.051979, + "nauc_ndcg_at_1_diff1": 0.454041, + "nauc_ndcg_at_3_max": 0.492252, + "nauc_ndcg_at_3_std": -0.059878, + "nauc_ndcg_at_3_diff1": 0.397151, + "nauc_ndcg_at_5_max": 0.508144, + "nauc_ndcg_at_5_std": -0.066427, + "nauc_ndcg_at_5_diff1": 0.399141, + "nauc_ndcg_at_10_max": 0.522106, + "nauc_ndcg_at_10_std": -0.036453, + "nauc_ndcg_at_10_diff1": 0.393387, + "nauc_ndcg_at_20_max": 0.545837, + "nauc_ndcg_at_20_std": -0.004858, + "nauc_ndcg_at_20_diff1": 0.417711, + "nauc_ndcg_at_100_max": 0.530515, + "nauc_ndcg_at_100_std": 0.02797, + "nauc_ndcg_at_100_diff1": 0.406339, + "nauc_ndcg_at_1000_max": 0.526063, + "nauc_ndcg_at_1000_std": 0.002337, + "nauc_ndcg_at_1000_diff1": 0.404848, + "nauc_map_at_1_max": 0.428604, + "nauc_map_at_1_std": -0.164492, + "nauc_map_at_1_diff1": 0.592502, + "nauc_map_at_3_max": 0.453787, + "nauc_map_at_3_std": -0.148417, + "nauc_map_at_3_diff1": 0.516134, + "nauc_map_at_5_max": 0.488606, + "nauc_map_at_5_std": -0.132986, + "nauc_map_at_5_diff1": 0.492892, + "nauc_map_at_10_max": 0.507393, + "nauc_map_at_10_std": -0.095651, + "nauc_map_at_10_diff1": 0.454792, + "nauc_map_at_20_max": 0.524031, + "nauc_map_at_20_std": -0.061421, + "nauc_map_at_20_diff1": 0.437416, + "nauc_map_at_100_max": 0.525863, + "nauc_map_at_100_std": -0.016718, + "nauc_map_at_100_diff1": 0.414592, + "nauc_map_at_1000_max": 0.524075, + "nauc_map_at_1000_std": -0.017135, + "nauc_map_at_1000_diff1": 0.41109, + "nauc_recall_at_1_max": 0.428604, + "nauc_recall_at_1_std": -0.164492, + "nauc_recall_at_1_diff1": 0.592502, + "nauc_recall_at_3_max": 0.41341, + "nauc_recall_at_3_std": -0.128386, + "nauc_recall_at_3_diff1": 0.466558, + "nauc_recall_at_5_max": 0.428246, + "nauc_recall_at_5_std": -0.147218, + "nauc_recall_at_5_diff1": 0.439546, + "nauc_recall_at_10_max": 0.45194, + "nauc_recall_at_10_std": -0.111673, + "nauc_recall_at_10_diff1": 0.39193, + "nauc_recall_at_20_max": 0.515363, + "nauc_recall_at_20_std": -0.020473, + "nauc_recall_at_20_diff1": 0.43457, + "nauc_recall_at_100_max": 0.523701, + "nauc_recall_at_100_std": 0.312531, + "nauc_recall_at_100_diff1": 0.328546, + "nauc_recall_at_1000_max": 0.522512, + "nauc_recall_at_1000_std": 0.879334, + "nauc_recall_at_1000_diff1": 0.422373, + "nauc_precision_at_1_max": 0.528465, + "nauc_precision_at_1_std": -0.051979, + "nauc_precision_at_1_diff1": 0.454041, + "nauc_precision_at_3_max": 0.211332, + "nauc_precision_at_3_std": 0.064255, + "nauc_precision_at_3_diff1": -0.032243, + "nauc_precision_at_5_max": 0.147767, + "nauc_precision_at_5_std": 0.134615, + "nauc_precision_at_5_diff1": -0.14755, + "nauc_precision_at_10_max": 0.043998, + "nauc_precision_at_10_std": 0.186908, + "nauc_precision_at_10_diff1": -0.237397, + "nauc_precision_at_20_max": -0.020424, + "nauc_precision_at_20_std": 0.213857, + "nauc_precision_at_20_diff1": -0.265475, + "nauc_precision_at_100_max": -0.112104, + "nauc_precision_at_100_std": 0.224909, + "nauc_precision_at_100_diff1": -0.271076, + "nauc_precision_at_1000_max": -0.148487, + "nauc_precision_at_1000_std": 0.133649, + "nauc_precision_at_1000_diff1": -0.262931, + "nauc_mrr_at_1_max": 0.528465, + "nauc_mrr_at_1_std": -0.051979, + "nauc_mrr_at_1_diff1": 0.454041, + "nauc_mrr_at_3_max": 0.512708, + "nauc_mrr_at_3_std": -0.045014, + "nauc_mrr_at_3_diff1": 0.431511, + "nauc_mrr_at_5_max": 0.514749, + "nauc_mrr_at_5_std": -0.044201, + "nauc_mrr_at_5_diff1": 0.429183, + "nauc_mrr_at_10_max": 0.518004, + "nauc_mrr_at_10_std": -0.04843, + "nauc_mrr_at_10_diff1": 0.432071, + "nauc_mrr_at_20_max": 0.518567, + "nauc_mrr_at_20_std": -0.045304, + "nauc_mrr_at_20_diff1": 0.433899, + "nauc_mrr_at_100_max": 0.518525, + "nauc_mrr_at_100_std": -0.044054, + "nauc_mrr_at_100_diff1": 0.433069, + "nauc_mrr_at_1000_max": 0.518296, + "nauc_mrr_at_1000_std": -0.044415, + "nauc_mrr_at_1000_diff1": 0.432997, + "main_score": 0.6608, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 68.53197383880615, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/thenlper__gte-large/4bef63f39fcc5e2d6b0aae83089f307af4970164/model_meta.json b/results/thenlper__gte-large/4bef63f39fcc5e2d6b0aae83089f307af4970164/model_meta.json index 7bad15165..9604051d1 100644 --- a/results/thenlper__gte-large/4bef63f39fcc5e2d6b0aae83089f307af4970164/model_meta.json +++ b/results/thenlper__gte-large/4bef63f39fcc5e2d6b0aae83089f307af4970164/model_meta.json @@ -1 +1 @@ -{"name": "thenlper/gte-large", "revision": "4bef63f39fcc5e2d6b0aae83089f307af4970164", "release_date": "2023-07-27", "languages": ["eng_Latn"], "n_parameters": 335142400, "max_tokens": 512.0, "embed_dim": 1024, "license": "mit", "open_weights": true, "public_training_code": null, "framework": ["PyTorch"], "reference": "https://huggingface.co/thenlper/gte-large", "similarity_fn_name": "cosine", "use_instructions": null, "training_datasets": null, "adapted_from": null, "superseded_by": null, "loader": null} \ No newline at end of file +{"name": "thenlper/gte-large", "revision": "4bef63f39fcc5e2d6b0aae83089f307af4970164", "release_date": "2023-07-27", "languages": ["eng_Latn"], "n_parameters": 335142400, "max_tokens": 512.0, "embed_dim": 1024, "license": "mit", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["PyTorch"], "reference": "https://huggingface.co/thenlper/gte-large", "similarity_fn_name": "cosine", "use_instructions": null, "training_datasets": null, "adapted_from": null, "superseded_by": null, "modalities": ["text"], "loader": null} \ No newline at end of file diff --git a/results/thenlper__gte-small/17e1f347d17fe144873b1201da91788898c639cd/BuiltBenchClusteringP2P.json b/results/thenlper__gte-small/17e1f347d17fe144873b1201da91788898c639cd/BuiltBenchClusteringP2P.json new file mode 100644 index 000000000..8b289bfbf --- /dev/null +++ b/results/thenlper__gte-small/17e1f347d17fe144873b1201da91788898c639cd/BuiltBenchClusteringP2P.json @@ -0,0 +1,42 @@ +{ + "dataset_revision": "919bb71053e9de62a68998161ce4f0cee8f786fb", + "task_name": "BuiltBenchClusteringP2P", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.555505, + "v_measure_std": 0.097721, + "v_measures": [ + 0.503072, + 0.659374, + 0.624128, + 0.605594, + 0.460889, + 0.466455, + 0.515093, + 0.609314, + 0.751364, + 0.569381, + 0.491286, + 0.616208, + 0.58848, + 0.481193, + 0.388886, + 0.497358, + 0.758498, + 0.488176, + 0.427936, + 0.607416 + ], + "main_score": 0.555505, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 9.003475666046143, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/thenlper__gte-small/17e1f347d17fe144873b1201da91788898c639cd/BuiltBenchClusteringS2S.json b/results/thenlper__gte-small/17e1f347d17fe144873b1201da91788898c639cd/BuiltBenchClusteringS2S.json new file mode 100644 index 000000000..dfa979ea6 --- /dev/null +++ b/results/thenlper__gte-small/17e1f347d17fe144873b1201da91788898c639cd/BuiltBenchClusteringS2S.json @@ -0,0 +1,40 @@ +{ + "dataset_revision": "1aaeb2ece89ea0a8c64e215c95c4cfaf7e891149", + "task_name": "BuiltBenchClusteringS2S", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "v_measure": 0.443124, + "v_measure_std": 0.101976, + "v_measures": [ + 0.56075, + 0.587169, + 0.333391, + 0.538352, + 0.370651, + 0.506393, + 0.355711, + 0.594486, + 0.447722, + 0.453034, + 0.5598, + 0.373921, + 0.489272, + 0.438925, + 0.206924, + 0.351933, + 0.36013, + 0.447668 + ], + "main_score": 0.443124, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.2421770095825195, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/thenlper__gte-small/17e1f347d17fe144873b1201da91788898c639cd/BuiltBenchReranking.json b/results/thenlper__gte-small/17e1f347d17fe144873b1201da91788898c639cd/BuiltBenchReranking.json new file mode 100644 index 000000000..0701672bb --- /dev/null +++ b/results/thenlper__gte-small/17e1f347d17fe144873b1201da91788898c639cd/BuiltBenchReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "fd33b0b3454deb256be06a57e8147b32ba078ff9", + "task_name": "BuiltBenchReranking", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "map": 0.652301, + "mrr": 0.785597, + "nAUC_map_max": 0.408079, + "nAUC_map_std": 0.604568, + "nAUC_map_diff1": 0.120964, + "nAUC_mrr_max": 0.455689, + "nAUC_mrr_std": 0.511318, + "nAUC_mrr_diff1": 0.259004, + "main_score": 0.652301, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 6.371310710906982, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/thenlper__gte-small/17e1f347d17fe144873b1201da91788898c639cd/BuiltBenchRetrieval.json b/results/thenlper__gte-small/17e1f347d17fe144873b1201da91788898c639cd/BuiltBenchRetrieval.json new file mode 100644 index 000000000..0a7de14f2 --- /dev/null +++ b/results/thenlper__gte-small/17e1f347d17fe144873b1201da91788898c639cd/BuiltBenchRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ae611238a58dae85f3130563fe9f9e995444a8d6", + "task_name": "BuiltBenchRetrieval", + "mteb_version": "1.31.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.56587, + "ndcg_at_3": 0.57177, + "ndcg_at_5": 0.57808, + "ndcg_at_10": 0.60545, + "ndcg_at_20": 0.62598, + "ndcg_at_100": 0.68211, + "ndcg_at_1000": 0.70722, + "map_at_1": 0.26289, + "map_at_3": 0.38866, + "map_at_5": 0.43182, + "map_at_10": 0.48208, + "map_at_20": 0.51264, + "map_at_100": 0.54943, + "map_at_1000": 0.5558, + "recall_at_1": 0.26289, + "recall_at_3": 0.43779, + "recall_at_5": 0.51914, + "recall_at_10": 0.63071, + "recall_at_20": 0.715, + "recall_at_100": 0.89601, + "recall_at_1000": 0.98953, + "precision_at_1": 0.56587, + "precision_at_3": 0.40918, + "precision_at_5": 0.33174, + "precision_at_10": 0.25, + "precision_at_20": 0.1729, + "precision_at_100": 0.06105, + "precision_at_1000": 0.00808, + "mrr_at_1": 0.565868, + "mrr_at_3": 0.639721, + "mrr_at_5": 0.652745, + "mrr_at_10": 0.660413, + "mrr_at_20": 0.663849, + "mrr_at_100": 0.665578, + "mrr_at_1000": 0.665759, + "nauc_ndcg_at_1_max": 0.565249, + "nauc_ndcg_at_1_std": 0.112176, + "nauc_ndcg_at_1_diff1": 0.49784, + "nauc_ndcg_at_3_max": 0.549294, + "nauc_ndcg_at_3_std": 0.108892, + "nauc_ndcg_at_3_diff1": 0.436107, + "nauc_ndcg_at_5_max": 0.5312, + "nauc_ndcg_at_5_std": 0.081125, + "nauc_ndcg_at_5_diff1": 0.424392, + "nauc_ndcg_at_10_max": 0.543143, + "nauc_ndcg_at_10_std": 0.092194, + "nauc_ndcg_at_10_diff1": 0.440848, + "nauc_ndcg_at_20_max": 0.562151, + "nauc_ndcg_at_20_std": 0.115383, + "nauc_ndcg_at_20_diff1": 0.455734, + "nauc_ndcg_at_100_max": 0.577576, + "nauc_ndcg_at_100_std": 0.168612, + "nauc_ndcg_at_100_diff1": 0.456036, + "nauc_ndcg_at_1000_max": 0.564061, + "nauc_ndcg_at_1000_std": 0.138276, + "nauc_ndcg_at_1000_diff1": 0.448824, + "nauc_map_at_1_max": 0.446174, + "nauc_map_at_1_std": -0.054777, + "nauc_map_at_1_diff1": 0.675274, + "nauc_map_at_3_max": 0.495911, + "nauc_map_at_3_std": -0.020478, + "nauc_map_at_3_diff1": 0.541513, + "nauc_map_at_5_max": 0.515578, + "nauc_map_at_5_std": -0.01112, + "nauc_map_at_5_diff1": 0.508261, + "nauc_map_at_10_max": 0.537426, + "nauc_map_at_10_std": 0.019955, + "nauc_map_at_10_diff1": 0.486083, + "nauc_map_at_20_max": 0.549599, + "nauc_map_at_20_std": 0.049785, + "nauc_map_at_20_diff1": 0.472189, + "nauc_map_at_100_max": 0.563447, + "nauc_map_at_100_std": 0.094063, + "nauc_map_at_100_diff1": 0.464744, + "nauc_map_at_1000_max": 0.563014, + "nauc_map_at_1000_std": 0.096464, + "nauc_map_at_1000_diff1": 0.461882, + "nauc_recall_at_1_max": 0.446174, + "nauc_recall_at_1_std": -0.054777, + "nauc_recall_at_1_diff1": 0.675274, + "nauc_recall_at_3_max": 0.44451, + "nauc_recall_at_3_std": -0.012434, + "nauc_recall_at_3_diff1": 0.46747, + "nauc_recall_at_5_max": 0.416443, + "nauc_recall_at_5_std": -0.01789, + "nauc_recall_at_5_diff1": 0.396213, + "nauc_recall_at_10_max": 0.441401, + "nauc_recall_at_10_std": 0.005832, + "nauc_recall_at_10_diff1": 0.398194, + "nauc_recall_at_20_max": 0.487608, + "nauc_recall_at_20_std": 0.090919, + "nauc_recall_at_20_diff1": 0.393717, + "nauc_recall_at_100_max": 0.624854, + "nauc_recall_at_100_std": 0.521372, + "nauc_recall_at_100_diff1": 0.446539, + "nauc_recall_at_1000_max": 0.509004, + "nauc_recall_at_1000_std": 0.781676, + "nauc_recall_at_1000_diff1": 0.182853, + "nauc_precision_at_1_max": 0.565249, + "nauc_precision_at_1_std": 0.112176, + "nauc_precision_at_1_diff1": 0.49784, + "nauc_precision_at_3_max": 0.329456, + "nauc_precision_at_3_std": 0.177332, + "nauc_precision_at_3_diff1": 0.005268, + "nauc_precision_at_5_max": 0.233952, + "nauc_precision_at_5_std": 0.171237, + "nauc_precision_at_5_diff1": -0.108149, + "nauc_precision_at_10_max": 0.115357, + "nauc_precision_at_10_std": 0.188162, + "nauc_precision_at_10_diff1": -0.179949, + "nauc_precision_at_20_max": 0.039231, + "nauc_precision_at_20_std": 0.200757, + "nauc_precision_at_20_diff1": -0.204552, + "nauc_precision_at_100_max": -0.043204, + "nauc_precision_at_100_std": 0.216176, + "nauc_precision_at_100_diff1": -0.200003, + "nauc_precision_at_1000_max": -0.109421, + "nauc_precision_at_1000_std": 0.119884, + "nauc_precision_at_1000_diff1": -0.223215, + "nauc_mrr_at_1_max": 0.565249, + "nauc_mrr_at_1_std": 0.112176, + "nauc_mrr_at_1_diff1": 0.49784, + "nauc_mrr_at_3_max": 0.563881, + "nauc_mrr_at_3_std": 0.132401, + "nauc_mrr_at_3_diff1": 0.4692, + "nauc_mrr_at_5_max": 0.555254, + "nauc_mrr_at_5_std": 0.140633, + "nauc_mrr_at_5_diff1": 0.462054, + "nauc_mrr_at_10_max": 0.55795, + "nauc_mrr_at_10_std": 0.134361, + "nauc_mrr_at_10_diff1": 0.467719, + "nauc_mrr_at_20_max": 0.560349, + "nauc_mrr_at_20_std": 0.13567, + "nauc_mrr_at_20_diff1": 0.469042, + "nauc_mrr_at_100_max": 0.561102, + "nauc_mrr_at_100_std": 0.13745, + "nauc_mrr_at_100_diff1": 0.470436, + "nauc_mrr_at_1000_max": 0.560914, + "nauc_mrr_at_1000_std": 0.136967, + "nauc_mrr_at_1000_diff1": 0.470257, + "main_score": 0.60545, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 6.867646217346191, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/thenlper__gte-small/17e1f347d17fe144873b1201da91788898c639cd/model_meta.json b/results/thenlper__gte-small/17e1f347d17fe144873b1201da91788898c639cd/model_meta.json index 9770ad9ab..5bb33f43c 100644 --- a/results/thenlper__gte-small/17e1f347d17fe144873b1201da91788898c639cd/model_meta.json +++ b/results/thenlper__gte-small/17e1f347d17fe144873b1201da91788898c639cd/model_meta.json @@ -1 +1 @@ -{"name": "thenlper/gte-small", "revision": "17e1f347d17fe144873b1201da91788898c639cd", "release_date": "2023-07-27", "languages": ["eng_Latn"], "n_parameters": 33360512, "max_tokens": 512.0, "embed_dim": 384, "license": "mit", "open_weights": true, "public_training_code": null, "framework": ["PyTorch"], "reference": "https://huggingface.co/thenlper/gte-small", "similarity_fn_name": "cosine", "use_instructions": null, "training_datasets": null, "adapted_from": null, "superseded_by": null, "loader": null} \ No newline at end of file +{"name": "thenlper/gte-small", "revision": "17e1f347d17fe144873b1201da91788898c639cd", "release_date": "2023-07-27", "languages": ["eng_Latn"], "n_parameters": 33360512, "max_tokens": 512.0, "embed_dim": 384, "license": "mit", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["PyTorch"], "reference": "https://huggingface.co/thenlper/gte-small", "similarity_fn_name": "cosine", "use_instructions": null, "training_datasets": null, "adapted_from": null, "superseded_by": null, "modalities": ["text"], "loader": null} \ No newline at end of file