diff --git a/mteb/descriptive_stats/Retrieval/WebFAQRetrieval.json b/mteb/descriptive_stats/Retrieval/WebFAQRetrieval.json new file mode 100644 index 0000000000..0b5bfc4b13 --- /dev/null +++ b/mteb/descriptive_stats/Retrieval/WebFAQRetrieval.json @@ -0,0 +1,904 @@ +{ + "test": { + "number_of_characters": 35239389, + "num_samples": 11434472, + "num_queries": 271168, + "num_documents": 11163304, + "min_document_length": 7, + "average_document_length": 1.1567167748903013, + "max_document_length": 4317, + "unique_documents": 11163304, + "min_query_length": 2, + "average_query_length": 82.33496577767288, + "max_query_length": 2, + "unique_queries": 271168, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 271168, + "hf_subset_descriptive_stats": { + "ara": { + "number_of_characters": 746242, + "num_samples": 152664, + "num_queries": 10000, + "num_documents": 142664, + "min_document_length": 10, + "average_document_length": 3.2307659956260864, + "max_document_length": 1503, + "unique_documents": 142664, + "min_query_length": 2, + "average_query_length": 28.5328, + "max_query_length": 2, + "unique_queries": 10000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10000 + }, + "aze": { + "number_of_characters": 33357, + "num_samples": 5356, + "num_queries": 487, + "num_documents": 4869, + "min_document_length": 10, + "average_document_length": 4.850893407270487, + "max_document_length": 149, + "unique_documents": 4869, + "min_query_length": 2, + "average_query_length": 19.9958932238193, + "max_query_length": 2, + "unique_queries": 487, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 487 + }, + "ben": { + "number_of_characters": 92226, + "num_samples": 15747, + "num_queries": 1432, + "num_documents": 14315, + "min_document_length": 10, + "average_document_length": 4.442612644079637, + "max_document_length": 322, + "unique_documents": 14315, + "min_query_length": 2, + "average_query_length": 19.993016759776538, + "max_query_length": 2, + "unique_queries": 1432, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1432 + }, + "bul": { + "number_of_characters": 248572, + "num_samples": 38208, + "num_queries": 3474, + "num_documents": 34734, + "min_document_length": 10, + "average_document_length": 5.156446133471526, + "max_document_length": 396, + "unique_documents": 34734, + "min_query_length": 2, + "average_query_length": 19.996545768566495, + "max_query_length": 2, + "unique_queries": 3474, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 3474 + }, + "cat": { + "number_of_characters": 93488, + "num_samples": 13964, + "num_queries": 1270, + "num_documents": 12694, + "min_document_length": 10, + "average_document_length": 5.364739246888294, + "max_document_length": 184, + "unique_documents": 12694, + "min_query_length": 2, + "average_query_length": 19.99055118110236, + "max_query_length": 2, + "unique_queries": 1270, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1270 + }, + "ces": { + "number_of_characters": 481288, + "num_samples": 79539, + "num_queries": 7231, + "num_documents": 72308, + "min_document_length": 9, + "average_document_length": 4.656082314543342, + "max_document_length": 994, + "unique_documents": 72308, + "min_query_length": 2, + "average_query_length": 19.999446826165123, + "max_query_length": 2, + "unique_queries": 7231, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 7231 + }, + "dan": { + "number_of_characters": 760053, + "num_samples": 147686, + "num_queries": 10000, + "num_documents": 137686, + "min_document_length": 10, + "average_document_length": 3.5201908690789185, + "max_document_length": 1420, + "unique_documents": 137686, + "min_query_length": 2, + "average_query_length": 27.5372, + "max_query_length": 2, + "unique_queries": 10000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10000 + }, + "deu": { + "number_of_characters": 2318297, + "num_samples": 901201, + "num_queries": 10000, + "num_documents": 891201, + "min_document_length": 7, + "average_document_length": 0.6013177723095015, + "max_document_length": 1191, + "unique_documents": 891201, + "min_query_length": 2, + "average_query_length": 178.2402, + "max_query_length": 2, + "unique_queries": 10000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10000 + }, + "ell": { + "number_of_characters": 295282, + "num_samples": 42368, + "num_queries": 3852, + "num_documents": 38516, + "min_document_length": 12, + "average_document_length": 5.666476269602243, + "max_document_length": 229, + "unique_documents": 38516, + "min_query_length": 2, + "average_query_length": 19.99792315680166, + "max_query_length": 2, + "unique_queries": 3852, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 3852 + }, + "eng": { + "number_of_characters": 11040979, + "num_samples": 5288725, + "num_queries": 10000, + "num_documents": 5278725, + "min_document_length": 9, + "average_document_length": 0.09159958133829665, + "max_document_length": 1177, + "unique_documents": 5278725, + "min_query_length": 2, + "average_query_length": 1055.745, + "max_query_length": 2, + "unique_queries": 10000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10000 + }, + "est": { + "number_of_characters": 90781, + "num_samples": 14181, + "num_queries": 1290, + "num_documents": 12891, + "min_document_length": 10, + "average_document_length": 5.0421999844853, + "max_document_length": 177, + "unique_documents": 12891, + "min_query_length": 2, + "average_query_length": 19.986046511627908, + "max_query_length": 2, + "unique_queries": 1290, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1290 + }, + "fas": { + "number_of_characters": 914501, + "num_samples": 236940, + "num_queries": 10000, + "num_documents": 226940, + "min_document_length": 10, + "average_document_length": 2.029703886489821, + "max_document_length": 3115, + "unique_documents": 226940, + "min_query_length": 2, + "average_query_length": 45.388, + "max_query_length": 2, + "unique_queries": 10000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10000 + }, + "fin": { + "number_of_characters": 500237, + "num_samples": 80902, + "num_queries": 7355, + "num_documents": 73547, + "min_document_length": 9, + "average_document_length": 4.801596258175045, + "max_document_length": 294, + "unique_documents": 73547, + "min_query_length": 2, + "average_query_length": 19.999184228416045, + "max_query_length": 2, + "unique_queries": 7355, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 7355 + }, + "fra": { + "number_of_characters": 1727184, + "num_samples": 579505, + "num_queries": 10000, + "num_documents": 569505, + "min_document_length": 8, + "average_document_length": 1.0327810993757738, + "max_document_length": 489, + "unique_documents": 569505, + "min_query_length": 2, + "average_query_length": 113.901, + "max_query_length": 2, + "unique_queries": 10000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10000 + }, + "heb": { + "number_of_characters": 232589, + "num_samples": 42847, + "num_queries": 3896, + "num_documents": 38951, + "min_document_length": 10, + "average_document_length": 3.9713229442119586, + "max_document_length": 278, + "unique_documents": 38951, + "min_query_length": 2, + "average_query_length": 19.995379876796715, + "max_query_length": 2, + "unique_queries": 3896, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 3896 + }, + "hin": { + "number_of_characters": 685269, + "num_samples": 110031, + "num_queries": 10000, + "num_documents": 100031, + "min_document_length": 10, + "average_document_length": 4.8505663244394235, + "max_document_length": 879, + "unique_documents": 100031, + "min_query_length": 2, + "average_query_length": 20.0062, + "max_query_length": 2, + "unique_queries": 10000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10000 + }, + "hrv": { + "number_of_characters": 38324, + "num_samples": 6100, + "num_queries": 555, + "num_documents": 5545, + "min_document_length": 11, + "average_document_length": 4.9114517583408475, + "max_document_length": 132, + "unique_documents": 5545, + "min_query_length": 2, + "average_query_length": 19.98198198198198, + "max_query_length": 2, + "unique_queries": 555, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 555 + }, + "hun": { + "number_of_characters": 319732, + "num_samples": 49824, + "num_queries": 4530, + "num_documents": 45294, + "min_document_length": 10, + "average_document_length": 5.059036516977966, + "max_document_length": 575, + "unique_documents": 45294, + "min_query_length": 2, + "average_query_length": 19.997350993377484, + "max_query_length": 2, + "unique_queries": 4530, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 4530 + }, + "ind": { + "number_of_characters": 768010, + "num_samples": 121315, + "num_queries": 10000, + "num_documents": 111315, + "min_document_length": 8, + "average_document_length": 4.899429546781656, + "max_document_length": 850, + "unique_documents": 111315, + "min_query_length": 2, + "average_query_length": 22.263, + "max_query_length": 2, + "unique_queries": 10000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10000 + }, + "isl": { + "number_of_characters": 37490, + "num_samples": 5256, + "num_queries": 478, + "num_documents": 4778, + "min_document_length": 12, + "average_document_length": 5.846379238174968, + "max_document_length": 149, + "unique_documents": 4778, + "min_query_length": 2, + "average_query_length": 19.99163179916318, + "max_query_length": 2, + "unique_queries": 478, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 478 + }, + "ita": { + "number_of_characters": 1094898, + "num_samples": 267803, + "num_queries": 10000, + "num_documents": 257803, + "min_document_length": 10, + "average_document_length": 2.247033587661897, + "max_document_length": 905, + "unique_documents": 257803, + "min_query_length": 2, + "average_query_length": 51.5606, + "max_query_length": 2, + "unique_queries": 10000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10000 + }, + "jpn": { + "number_of_characters": 882722, + "num_samples": 319157, + "num_queries": 10000, + "num_documents": 309157, + "min_document_length": 10, + "average_document_length": 0.8552547734646151, + "max_document_length": 645, + "unique_documents": 309157, + "min_query_length": 2, + "average_query_length": 61.8314, + "max_query_length": 2, + "unique_queries": 10000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10000 + }, + "kat": { + "number_of_characters": 16624, + "num_samples": 2646, + "num_queries": 241, + "num_documents": 2405, + "min_document_length": 13, + "average_document_length": 4.912266112266113, + "max_document_length": 125, + "unique_documents": 2405, + "min_query_length": 2, + "average_query_length": 19.95850622406639, + "max_query_length": 2, + "unique_queries": 241, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 241 + }, + "kaz": { + "number_of_characters": 21230, + "num_samples": 3295, + "num_queries": 300, + "num_documents": 2995, + "min_document_length": 16, + "average_document_length": 5.088480801335559, + "max_document_length": 260, + "unique_documents": 2995, + "min_query_length": 2, + "average_query_length": 19.966666666666665, + "max_query_length": 2, + "unique_queries": 300, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 300 + }, + "kor": { + "number_of_characters": 481069, + "num_samples": 112000, + "num_queries": 10000, + "num_documents": 102000, + "min_document_length": 10, + "average_document_length": 2.7163627450980394, + "max_document_length": 264, + "unique_documents": 102000, + "min_query_length": 2, + "average_query_length": 20.4, + "max_query_length": 2, + "unique_queries": 10000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10000 + }, + "lav": { + "number_of_characters": 89318, + "num_samples": 14429, + "num_queries": 1312, + "num_documents": 13117, + "min_document_length": 10, + "average_document_length": 4.809331401997408, + "max_document_length": 184, + "unique_documents": 13117, + "min_query_length": 2, + "average_query_length": 19.995426829268293, + "max_query_length": 2, + "unique_queries": 1312, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1312 + }, + "lit": { + "number_of_characters": 126055, + "num_samples": 20200, + "num_queries": 1837, + "num_documents": 18363, + "min_document_length": 9, + "average_document_length": 4.864619070957905, + "max_document_length": 420, + "unique_documents": 18363, + "min_query_length": 2, + "average_query_length": 19.992378878606424, + "max_query_length": 2, + "unique_queries": 1837, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1837 + }, + "mar": { + "number_of_characters": 46162, + "num_samples": 8145, + "num_queries": 741, + "num_documents": 7404, + "min_document_length": 14, + "average_document_length": 4.234737979470556, + "max_document_length": 184, + "unique_documents": 7404, + "min_query_length": 2, + "average_query_length": 19.983805668016196, + "max_query_length": 2, + "unique_queries": 741, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 741 + }, + "msa": { + "number_of_characters": 50848, + "num_samples": 7072, + "num_queries": 643, + "num_documents": 6429, + "min_document_length": 11, + "average_document_length": 5.909161611448126, + "max_document_length": 189, + "unique_documents": 6429, + "min_query_length": 2, + "average_query_length": 19.996889580093313, + "max_query_length": 2, + "unique_queries": 643, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 643 + }, + "nld": { + "number_of_characters": 1238345, + "num_samples": 380662, + "num_queries": 10000, + "num_documents": 370662, + "min_document_length": 9, + "average_document_length": 1.3409008746512996, + "max_document_length": 1603, + "unique_documents": 370662, + "min_query_length": 2, + "average_query_length": 74.1324, + "max_query_length": 2, + "unique_queries": 10000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10000 + }, + "nor": { + "number_of_characters": 433342, + "num_samples": 69556, + "num_queries": 6324, + "num_documents": 63232, + "min_document_length": 7, + "average_document_length": 4.853207236842105, + "max_document_length": 227, + "unique_documents": 63232, + "min_query_length": 2, + "average_query_length": 19.997469955724224, + "max_query_length": 2, + "unique_queries": 6324, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 6324 + }, + "pol": { + "number_of_characters": 902187, + "num_samples": 202515, + "num_queries": 10000, + "num_documents": 192515, + "min_document_length": 10, + "average_document_length": 2.686320546450926, + "max_document_length": 1038, + "unique_documents": 192515, + "min_query_length": 2, + "average_query_length": 38.503, + "max_query_length": 2, + "unique_queries": 10000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10000 + }, + "por": { + "number_of_characters": 918403, + "num_samples": 219353, + "num_queries": 10000, + "num_documents": 209353, + "min_document_length": 10, + "average_document_length": 2.3868633360878517, + "max_document_length": 1154, + "unique_documents": 209353, + "min_query_length": 2, + "average_query_length": 41.8706, + "max_query_length": 2, + "unique_queries": 10000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10000 + }, + "ron": { + "number_of_characters": 440199, + "num_samples": 65882, + "num_queries": 5990, + "num_documents": 59892, + "min_document_length": 10, + "average_document_length": 5.349879783610499, + "max_document_length": 1176, + "unique_documents": 59892, + "min_query_length": 2, + "average_query_length": 19.997328881469116, + "max_query_length": 2, + "unique_queries": 5990, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 5990 + }, + "rus": { + "number_of_characters": 1328024, + "num_samples": 397504, + "num_queries": 10000, + "num_documents": 387504, + "min_document_length": 10, + "average_document_length": 1.4271233329204345, + "max_document_length": 1937, + "unique_documents": 387504, + "min_query_length": 2, + "average_query_length": 77.5008, + "max_query_length": 2, + "unique_queries": 10000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10000 + }, + "slk": { + "number_of_characters": 229346, + "num_samples": 34675, + "num_queries": 3153, + "num_documents": 31522, + "min_document_length": 10, + "average_document_length": 5.2757439248778635, + "max_document_length": 4317, + "unique_documents": 31522, + "min_query_length": 2, + "average_query_length": 19.994925467808436, + "max_query_length": 2, + "unique_queries": 3153, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 3153 + }, + "slv": { + "number_of_characters": 114951, + "num_samples": 17785, + "num_queries": 1617, + "num_documents": 16168, + "min_document_length": 11, + "average_document_length": 5.109784760019792, + "max_document_length": 426, + "unique_documents": 16168, + "min_query_length": 2, + "average_query_length": 19.997526283240568, + "max_query_length": 2, + "unique_queries": 1617, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 1617 + }, + "spa": { + "number_of_characters": 1758740, + "num_samples": 614661, + "num_queries": 10000, + "num_documents": 604661, + "min_document_length": 10, + "average_document_length": 0.9086380633115084, + "max_document_length": 559, + "unique_documents": 604661, + "min_query_length": 2, + "average_query_length": 120.9322, + "max_query_length": 2, + "unique_queries": 10000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10000 + }, + "sqi": { + "number_of_characters": 15784, + "num_samples": 2285, + "num_queries": 208, + "num_documents": 2077, + "min_document_length": 11, + "average_document_length": 5.599422243620607, + "max_document_length": 141, + "unique_documents": 2077, + "min_query_length": 2, + "average_query_length": 19.971153846153847, + "max_query_length": 2, + "unique_queries": 208, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 208 + }, + "srp": { + "number_of_characters": 40489, + "num_samples": 6407, + "num_queries": 583, + "num_documents": 5824, + "min_document_length": 8, + "average_document_length": 4.95209478021978, + "max_document_length": 128, + "unique_documents": 5824, + "min_query_length": 2, + "average_query_length": 19.97941680960549, + "max_query_length": 2, + "unique_queries": 583, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 583 + }, + "swe": { + "number_of_characters": 779495, + "num_samples": 168738, + "num_queries": 10000, + "num_documents": 158738, + "min_document_length": 10, + "average_document_length": 2.9105759175496733, + "max_document_length": 257, + "unique_documents": 158738, + "min_query_length": 2, + "average_query_length": 31.7476, + "max_query_length": 2, + "unique_queries": 10000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10000 + }, + "tgl": { + "number_of_characters": 27050, + "num_samples": 4067, + "num_queries": 370, + "num_documents": 3697, + "min_document_length": 13, + "average_document_length": 5.316743305382743, + "max_document_length": 198, + "unique_documents": 3697, + "min_query_length": 2, + "average_query_length": 19.983783783783785, + "max_query_length": 2, + "unique_queries": 370, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 370 + }, + "tha": { + "number_of_characters": 311898, + "num_samples": 52172, + "num_queries": 4743, + "num_documents": 47429, + "min_document_length": 10, + "average_document_length": 4.5761032279828795, + "max_document_length": 472, + "unique_documents": 47429, + "min_query_length": 2, + "average_query_length": 19.999578325954037, + "max_query_length": 2, + "unique_queries": 4743, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 4743 + }, + "tur": { + "number_of_characters": 706899, + "num_samples": 154846, + "num_queries": 10000, + "num_documents": 144846, + "min_document_length": 7, + "average_document_length": 2.8803487842260056, + "max_document_length": 703, + "unique_documents": 144846, + "min_query_length": 2, + "average_query_length": 28.9692, + "max_query_length": 2, + "unique_queries": 10000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10000 + }, + "ukr": { + "number_of_characters": 478281, + "num_samples": 75352, + "num_queries": 6851, + "num_documents": 68501, + "min_document_length": 10, + "average_document_length": 4.9821024510591085, + "max_document_length": 1015, + "unique_documents": 68501, + "min_query_length": 2, + "average_query_length": 19.997372646329, + "max_query_length": 2, + "unique_queries": 6851, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 6851 + }, + "urd": { + "number_of_characters": 18987, + "num_samples": 3053, + "num_queries": 278, + "num_documents": 2775, + "min_document_length": 19, + "average_document_length": 4.842162162162162, + "max_document_length": 108, + "unique_documents": 2775, + "min_query_length": 2, + "average_query_length": 19.964028776978417, + "max_query_length": 2, + "unique_queries": 278, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 278 + }, + "uzb": { + "number_of_characters": 8789, + "num_samples": 1390, + "num_queries": 127, + "num_documents": 1263, + "min_document_length": 16, + "average_document_length": 4.958828186856691, + "max_document_length": 110, + "unique_documents": 1263, + "min_query_length": 2, + "average_query_length": 19.88976377952756, + "max_query_length": 2, + "unique_queries": 127, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 127 + }, + "vie": { + "number_of_characters": 787166, + "num_samples": 133972, + "num_queries": 10000, + "num_documents": 123972, + "min_document_length": 9, + "average_document_length": 4.349546671829123, + "max_document_length": 1061, + "unique_documents": 123972, + "min_query_length": 2, + "average_query_length": 24.7944, + "max_query_length": 2, + "unique_queries": 10000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10000 + }, + "zho": { + "number_of_characters": 468187, + "num_samples": 142491, + "num_queries": 10000, + "num_documents": 132491, + "min_document_length": 7, + "average_document_length": 1.5337268191801707, + "max_document_length": 671, + "unique_documents": 132491, + "min_query_length": 2, + "average_query_length": 26.4982, + "max_query_length": 2, + "unique_queries": 10000, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 10000 + } + } + } +} \ No newline at end of file diff --git a/mteb/tasks/Retrieval/__init__.py b/mteb/tasks/Retrieval/__init__.py index 96cd664696..a13fa94bfc 100644 --- a/mteb/tasks/Retrieval/__init__.py +++ b/mteb/tasks/Retrieval/__init__.py @@ -135,6 +135,7 @@ from .multilingual.NeuCLIR2023Retrieval import * from .multilingual.PublicHealthQARetrieval import * from .multilingual.StatcanDialogueDatasetRetrieval import * +from .multilingual.WebFAQRetrieval import * from .multilingual.WikipediaRetrievalMultilingual import * from .multilingual.XMarketRetrieval import * from .multilingual.XPQARetrieval import * diff --git a/mteb/tasks/Retrieval/multilingual/WebFAQRetrieval.py b/mteb/tasks/Retrieval/multilingual/WebFAQRetrieval.py new file mode 100644 index 0000000000..703103a782 --- /dev/null +++ b/mteb/tasks/Retrieval/multilingual/WebFAQRetrieval.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import datasets + +from mteb.abstasks.MultilingualTask import MultilingualTask +from mteb.abstasks.TaskMetadata import TaskMetadata + +from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval + +_EVAL_SPLIT = "test" + +_LANGUAGES = { + "ara": ["ara-Arab"], + "aze": ["aze-Latn"], + "ben": ["ben-Beng"], + "bul": ["bul-Cyrl"], + "cat": ["cat-Latn"], + "ces": ["ces-Latn"], + "dan": ["dan-Latn"], + "deu": ["deu-Latn"], + "ell": ["ell-Grek"], + "eng": ["eng-Latn"], + "est": ["est-Latn"], + "fas": ["fas-Arab"], + "fin": ["fin-Latn"], + "fra": ["fra-Latn"], + "heb": ["heb-Hebr"], + "hin": ["hin-Deva"], + "hrv": ["hrv-Latn"], + "hun": ["hun-Latn"], + "ind": ["ind-Latn"], + "isl": ["isl-Latn"], + "ita": ["ita-Latn"], + "jpn": ["jpn-Jpan"], + "kat": ["kat-Geor"], + "kaz": ["kaz-Cyrl"], + "kor": ["kor-Kore"], + "lav": ["lav-Latn"], + "lit": ["lit-Latn"], + "mar": ["mar-Deva"], + "msa": ["msa-Latn"], + "nld": ["nld-Latn"], + "nor": ["nor-Latn"], + "pol": ["pol-Latn"], + "por": ["por-Latn"], + "ron": ["ron-Latn"], + "rus": ["rus-Cyrl"], + "slk": ["slk-Latn"], + "slv": ["slv-Latn"], + "spa": ["spa-Latn"], + "sqi": ["sqi-Latn"], + "srp": ["srp-Cyrl"], + "swe": ["swe-Latn"], + "tgl": ["tgl-Latn"], + "tha": ["tha-Thai"], + "tur": ["tur-Latn"], + "ukr": ["ukr-Cyrl"], + "urd": ["urd-Arab"], + "uzb": ["uzb-Latn"], + "vie": ["vie-Latn"], + "zho": ["zho-Hans"], +} + + +def _load_webfaq_data( + path: str, langs: list, splits: str, cache_dir: str = None, revision: str = None +): + corpus = {lang: {split: None for split in splits} for lang in langs} + queries = {lang: {split: None for split in splits} for lang in langs} + relevant_docs = {lang: {split: None for split in splits} for lang in langs} + + split = _EVAL_SPLIT + + for lang in langs: + # Load corpus data (Can be several millions for languages) + corpus_identifier = f"{lang}-corpus" + corpus_data = datasets.load_dataset( + path, + corpus_identifier, + cache_dir=cache_dir, + revision=revision, + ) + corpus[lang][split] = {} + for row in corpus_data["corpus"]: + corpus_id = row["_id"] + title = row["title"] + text = row["text"] + corpus[lang][split][corpus_id] = {"title": title, "text": text} + + # Load queries data + queries_identifier = f"{lang}-queries" + queries_data = datasets.load_dataset( + path, + queries_identifier, + cache_dir=cache_dir, + revision=revision, + ) + queries[lang][split] = {} + for row in queries_data[split]: + query_id = row["_id"] + text = row["text"] + queries[lang][split][query_id] = text + + # Load relevant documents data + qrels_identifier = f"{lang}-qrels" + qrels_data = datasets.load_dataset( + path, + qrels_identifier, + cache_dir=cache_dir, + revision=revision, + ) + relevant_docs[lang][split] = {} + for row in qrels_data[split]: + query_id = row["query-id"] + corpus_id = row["corpus-id"] + score = row["score"] + if query_id not in relevant_docs[lang][split]: + relevant_docs[lang][split][query_id] = {} + relevant_docs[lang][split][query_id][corpus_id] = int(score) + + corpus = datasets.DatasetDict(corpus) + queries = datasets.DatasetDict(queries) + relevant_docs = datasets.DatasetDict(relevant_docs) + + return corpus, queries, relevant_docs + + +class WebFAQRetrieval(MultilingualTask, AbsTaskRetrieval): + metadata = TaskMetadata( + name="WebFAQRetrieval", + description="WebFAQ is a broad-coverage corpus of natural question-answer pairs in 75 languages, gathered from FAQ pages on the web.", + reference="https://huggingface.co/PaDaS-Lab", + dataset={ + "path": "PaDaS-Lab/webfaq-retrieval", + "revision": "c3262adb1c32ac0c3ea8de6393a44366edaa62e1", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=[_EVAL_SPLIT], + eval_langs=_LANGUAGES, + main_score="ndcg_at_10", + date=("2022-09-01", "2024-10-01"), + domains=["Web", "Written"], + task_subtypes=["Question answering"], + license="cc-by-4.0", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation="""@misc{dinzinger2025webfaq, + title={WebFAQ: A Multilingual Collection of Natural Q&A Datasets for Dense Retrieval}, + author={Michael Dinzinger and Laura Caspari and Kanishka Ghosh Dastidar and Jelena Mitrović and Michael Granitzer}, + year={2025}, + eprint={2502.20936}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2502.20936}, +}""", + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus, self.queries, self.relevant_docs = _load_webfaq_data( + path=self.metadata_dict["dataset"]["path"], + langs=self.hf_subsets, + splits=self.metadata_dict["eval_splits"], + cache_dir=kwargs.get("cache_dir", None), + revision=self.metadata_dict["dataset"]["revision"], + ) + + self.data_loaded = True