Main merge for maeb -> 1.38.52

.github/workflows/leaderboard_build.yml

@@ -22,7 +22,7 @@ jobs:
 
       - name: Install dependencies (incl. leaderboard extra)
         run: |
-          pip install ".[dev,leaderboard]"
+          pip install ".[leaderboard]" --group dev
 
       - name: Run leaderboard build test
         run: |

Makefile

@@ -1,12 +1,12 @@
 install:
 	@echo "--- 🚀 Installing project dependencies ---"
-	pip install -e ".[dev,image]"
+	pip install -e ".[image]" --group dev
 	pre-commit install
 
 install-for-tests:
 	@echo "--- 🚀 Installing project dependencies for test ---"
 	@echo "This ensures that the project is not installed in editable mode"
-	pip install ".[dev,image]"
+	pip install ".[image]" --group dev
 
 lint:
 	@echo "--- 🧹 Running linters ---"
@@ -17,7 +17,7 @@ lint-check:
 	@echo "--- 🧹 Check is project is linted ---"
 	# Required for CI to work, otherwise it will just pass
 	ruff format . --check						    # running ruff formatting
-	ruff check **/*.py 						        # running ruff linting
+	ruff check .    						        # running ruff linting
 
 test:
 	@echo "--- 🧪 Running tests ---"
@@ -43,7 +43,7 @@ build-docs:
 
 model-load-test:
 	@echo "--- 🚀 Running model load test ---"
-	pip install ".[dev, pylate,gritlm,xformers,model2vec]"
+	pip install ".[pylate,gritlm,xformers,model2vec]" --group dev
 	python scripts/extract_model_names.py $(BASE_BRANCH) --return_one_model_name_per_file
 	python tests/test_models/model_loading.py --model_name_file scripts/model_names.txt
 

docs/benchmarks.md

@@ -30,6 +30,7 @@ The following table gives you an overview of the benchmarks in MTEB.
 | [MTEB(Indic, v1)](https://arxiv.org/abs/2502.13595) | Indic | 23 | BitextMining: 4, Clustering: 1, Classification: 13, PairClassification: 1, Retrieval: 2, Reranking: 1, STS: 1 | [Constructed, Encyclopaedic, Fiction, Government, Legal, News, Non-fiction, Religious, Reviews, Social, Spoken, Web, Written] | asm,awa,ben,bgc,bho,bod,boy,brx,doi,eng,gbm,gom,guj,hin,hne,kan,kas,mai,mal,mar,mni,mup,mwr,nep,npi,ory,pan,pus,raj,san,sat,snd,tam,tel,urd |
 | MTEB(Law, v1) | Legal | 8 | Retrieval: 8 | [Legal, Written] | deu,eng,zho |
 | MTEB(Medical, v1) | Medical | 12 | Retrieval: 9, Clustering: 2, Reranking: 1 | [Academic, Government, Medical, Non-fiction, Web, Written] | ara,cmn,eng,fra,kor,pol,rus,spa,vie,zho |
+| [MTEB(Multilingual, v1)](https://arxiv.org/abs/2502.13595) | Multilingual | 132 | BitextMining: 13, Classification: 43, Clustering: 17, Retrieval: 18, InstructionRetrieval: 3, MultilabelClassification: 5, PairClassification: 11, Reranking: 6, STS: 16 | [Academic, Blog, Constructed, Encyclopaedic, Entertainment, Fiction, Financial, Government, Legal, Medical, News, Non-fiction, Programming, Religious, Reviews, Social, Spoken, Subtitles, Web, Written] | aai,aak,aau,aaz,abs,abt,abx,aby,ace,acf,acm,acq,acr,acu,adz,aeb,aer,aey,afr,agd,agg,agm,agn,agr,agt,agu,aia,aii,ajp,aka,ake,alp,alq,als,aly,ame,amf,amh,amk,amm,amn,amo,amp,amr,amu,amx,ang,anh,anv,aoi,aoj,aom,aon,apb,apc,ape,apn,apr,apu,apw,apz,ara,arb,are,arl,arn,arp,arq,ars,ary,arz,asm,aso,ast,ata,atb,atd,atg,att,auc,aui,auy,avt,awa,awb,awk,awx,ayr,azb,aze,azg,azj,azz,bak,bam,ban,bao,bba,bbb,bbc,bbr,bch,bco,bdd,bea,bef,bel,bem,ben,beo,ber,beu,bew,bgc,bgs,bgt,bhg,bhl,bho,bhp,big,bjk,bjn,bjp,bjr,bjv,bjz,bkd,bki,bkq,bkx,blw,blz,bmh,bmk,bmr,bmu,bnp,boa,bod,boj,bon,bos,box,boy,bpr,bps,bqc,bqp,bre,brx,bsj,bsn,bsp,bss,bug,buk,bul,bus,bvd,bvr,bxh,byr,byx,bzd,bzh,bzj,caa,cab,cac,caf,cak,cao,cap,car,cat,cav,cax,cbc,cbi,cbk,cbr,cbs,cbt,cbu,cbv,cco,ceb,cek,ces,cgc,cha,chd,chf,chk,chq,chv,chz,cjk,cjo,cjv,ckb,cle,clu,cme,cmn,cmo,cni,cnl,cnt,cof,con,cop,cor,cot,cpa,cpb,cpc,cpu,cpy,crh,crn,crx,csb,cso,csy,cta,cth,ctp,ctu,cub,cuc,cui,cuk,cut,cux,cwe,cya,cym,daa,dad,dah,dan,ded,deu,dgc,dgr,dgz,dhg,dif,dik,div,dji,djk,djr,dob,doi,dop,dov,dsb,dtp,dwr,dww,dwy,dyu,dzo,ebk,eko,ell,emi,emp,eng,enq,epo,eri,ese,esk,est,etr,eus,ewe,faa,fai,fao,far,fas,ffm,fij,fil,fin,fon,for,fra,fry,fuc,fue,fuf,fuh,fur,fuv,gah,gai,gam,gaw,gaz,gbm,gdn,gdr,geb,gfk,ghs,gla,gle,glg,glk,glv,gmv,gng,gnn,gnw,gof,gom,grc,grn,gsw,gub,guh,gui,guj,gul,gum,gun,guo,gup,gux,gvc,gvf,gvn,gvs,gwi,gym,gyr,hat,hau,haw,hbo,hch,heb,heg,hin,hix,hla,hlt,hmn,hmo,hne,hns,hop,hot,hrv,hsb,hto,hub,hui,hun,hus,huu,huv,hvn,hye,ian,ibo,ido,ign,ikk,ikw,ile,ilo,imo,ina,inb,ind,ino,iou,ipi,isl,isn,ita,iws,ixl,jac,jae,jao,jav,jic,jid,jiv,jni,jpn,jvn,kab,kac,kam,kan,kaq,kas,kat,kaz,kbc,kbh,kbm,kbp,kbq,kdc,kde,kdl,kea,kek,ken,kew,kgf,kgk,kgp,khk,khm,khs,khz,kik,kin,kir,kiw,kiz,kje,kjs,kkc,kkl,klt,klv,kmb,kmg,kmh,kmk,kmo,kmr,kms,kmu,knc,kne,knf,knj,knv,kon,kor,kos,kpf,kpg,kpj,kpr,kpw,kpx,kqa,kqc,kqf,kql,kqw,krc,ksd,ksj,ksr,ktm,kto,kud,kue,kup,kur,kvg,kvn,kwd,kwf,kwi,kwj,kyc,kyf,kyg,kyq,kyz,kze,kzj,lac,lao,lat,lav,lbb,lbk,lcm,leu,lex,lfn,lgl,lid,lif,lij,lim,lin,lit,llg,lmo,ltg,ltz,lua,lug,luo,lus,lvs,lww,maa,mad,mag,mai,maj,mak,mal,mam,maq,mar,mau,mav,max,maz,mbb,mbc,mbh,mbj,mbl,mbs,mbt,mca,mcb,mcd,mcf,mco,mcp,mcq,mcr,mdy,med,mee,mek,meq,met,meu,mey,mgc,mgh,mgw,mhl,mhr,mib,mic,mie,mig,mih,mil,min,mio,mir,mit,miz,mjc,mkd,mkj,mkl,mkn,mks,mle,mlg,mlh,mlp,mlt,mmo,mmx,mna,mni,mon,mop,mos,mox,mph,mpj,mpm,mpp,mps,mpt,mpx,mqb,mqj,mri,msa,msb,msc,msk,msm,msy,mti,mto,mui,mup,mux,muy,mva,mvn,mwc,mwe,mwf,mwp,mwr,mxb,mxp,mxq,mxt,mya,myk,myu,myw,myy,mzz,nab,naf,nak,nas,nbq,nca,nch,ncj,ncl,ncu,nde,ndg,ndj,nds,nep,nfa,ngp,ngu,nhe,nhg,nhi,nho,nhr,nhu,nhw,nhy,nif,nii,nij,nin,nko,nld,nlg,nna,nno,nnq,noa,nob,nop,nor,not,nou,nov,npi,npl,nqo,nsn,nso,nss,ntj,ntp,ntu,nus,nuy,nvm,nwi,nya,nys,nyu,obo,oci,okv,omw,ong,ons,ood,opm,orm,orv,ory,ote,otm,otn,otq,ots,pab,pad,pag,pah,pam,pan,pao,pap,pbt,pcm,pes,pib,pio,pir,piu,pjt,pls,plt,plu,pma,pms,poe,poh,poi,pol,pon,por,poy,ppo,prf,pri,prs,ptp,ptu,pus,pwg,qub,quc,quf,quh,qul,qup,quy,qvc,qve,qvh,qvm,qvn,qvs,qvw,qvz,qwh,qxh,qxn,qxo,rai,raj,reg,rej,rgu,rkb,rmc,rmy,rom,ron,roo,rop,row,rro,ruf,rug,run,rus,rwo,sab,sag,sah,san,sat,sbe,sbk,sbs,scn,sco,seh,sey,sgb,sgz,shi,shj,shn,shp,sim,sin,sja,slk,sll,slv,smk,smo,sna,snc,snd,snn,snp,snx,sny,som,soq,sot,soy,spa,spl,spm,spp,sps,spy,sqi,srd,sri,srm,srn,srp,srq,ssd,ssg,ssw,ssx,stp,sua,sue,sun,sus,suz,svk,swa,swe,swg,swh,swp,sxb,szl,tac,tah,taj,tam,taq,tat,tav,taw,tbc,tbf,tbg,tbo,tbz,tca,tcs,tcz,tdt,tee,tel,ter,tet,tew,tfr,tgk,tgl,tgo,tgp,tha,tif,tim,tir,tiw,tiy,tke,tku,tlf,tmd,tna,tnc,tnk,tnn,tnp,toc,tod,tof,toj,ton,too,top,tos,tpa,tpi,tpt,tpz,trc,tsn,tso,tsw,ttc,tte,tuc,tue,tuf,tuk,tum,tuo,tur,tvk,twi,txq,txu,tyv,tzj,tzl,tzm,tzo,ubr,ubu,udu,uig,ukr,uli,ulk,umb,upv,ura,urb,urd,uri,urt,urw,usa,usp,uvh,uvl,uzb,uzn,vec,ven,vid,vie,viv,vmy,waj,wal,wap,war,wat,wbi,wbp,wed,wer,wim,wiu,wiv,wln,wmt,wmw,wnc,wnu,wol,wos,wrk,wro,wrs,wsk,wuu,wuv,xav,xbi,xed,xho,xla,xnn,xon,xsi,xtd,xtm,yaa,yad,yal,yap,yaq,yby,ycn,ydd,yid,yka,yle,yml,yon,yor,yrb,yre,yss,yue,yuj,yut,yuw,yva,zaa,zab,zac,zad,zai,zaj,zam,zao,zap,zar,zas,zat,zav,zaw,zca,zga,zho,zia,ziw,zlm,zos,zpc,zpl,zpm,zpo,zpq,zpu,zpv,zpz,zsm,zsr,ztq,zty,zul,zyp |
 | [MTEB(Multilingual, v2)](https://arxiv.org/abs/2502.13595) | Multilingual | 131 | BitextMining: 13, Classification: 43, Clustering: 16, Retrieval: 18, InstructionRetrieval: 3, MultilabelClassification: 5, PairClassification: 11, Reranking: 6, STS: 16 | [Academic, Blog, Constructed, Encyclopaedic, Entertainment, Fiction, Financial, Government, Legal, Medical, News, Non-fiction, Programming, Religious, Reviews, Social, Spoken, Subtitles, Web, Written] | aai,aak,aau,aaz,abs,abt,abx,aby,ace,acf,acm,acq,acr,acu,adz,aeb,aer,aey,afr,agd,agg,agm,agn,agr,agt,agu,aia,aii,ajp,aka,ake,alp,alq,als,aly,ame,amf,amh,amk,amm,amn,amo,amp,amr,amu,amx,ang,anh,anv,aoi,aoj,aom,aon,apb,apc,ape,apn,apr,apu,apw,apz,ara,arb,are,arl,arn,arp,arq,ars,ary,arz,asm,aso,ast,ata,atb,atd,atg,att,auc,aui,auy,avt,awa,awb,awk,awx,ayr,azb,aze,azg,azj,azz,bak,bam,ban,bao,bba,bbb,bbc,bbr,bch,bco,bdd,bea,bef,bel,bem,ben,beo,ber,beu,bew,bgc,bgs,bgt,bhg,bhl,bho,bhp,big,bjk,bjn,bjp,bjr,bjv,bjz,bkd,bki,bkq,bkx,blw,blz,bmh,bmk,bmr,bmu,bnp,boa,bod,boj,bon,bos,box,boy,bpr,bps,bqc,bqp,bre,brx,bsj,bsn,bsp,bss,bug,buk,bul,bus,bvd,bvr,bxh,byr,byx,bzd,bzh,bzj,caa,cab,cac,caf,cak,cao,cap,car,cat,cav,cax,cbc,cbi,cbk,cbr,cbs,cbt,cbu,cbv,cco,ceb,cek,ces,cgc,cha,chd,chf,chk,chq,chv,chz,cjk,cjo,cjv,ckb,cle,clu,cme,cmn,cmo,cni,cnl,cnt,cof,con,cop,cor,cot,cpa,cpb,cpc,cpu,cpy,crh,crn,crx,csb,cso,csy,cta,cth,ctp,ctu,cub,cuc,cui,cuk,cut,cux,cwe,cya,cym,daa,dad,dah,dan,ded,deu,dgc,dgr,dgz,dhg,dif,dik,div,dji,djk,djr,dob,doi,dop,dov,dsb,dtp,dwr,dww,dwy,dyu,dzo,ebk,eko,ell,emi,emp,eng,enq,epo,eri,ese,esk,est,etr,eus,ewe,faa,fai,fao,far,fas,ffm,fij,fil,fin,fon,for,fra,fry,fuc,fue,fuf,fuh,fur,fuv,gah,gai,gam,gaw,gaz,gbm,gdn,gdr,geb,gfk,ghs,gla,gle,glg,glk,glv,gmv,gng,gnn,gnw,gof,gom,grc,grn,gsw,gub,guh,gui,guj,gul,gum,gun,guo,gup,gux,gvc,gvf,gvn,gvs,gwi,gym,gyr,hat,hau,haw,hbo,hch,heb,heg,hin,hix,hla,hlt,hmn,hmo,hne,hns,hop,hot,hrv,hsb,hto,hub,hui,hun,hus,huu,huv,hvn,hye,ian,ibo,ido,ign,ikk,ikw,ile,ilo,imo,ina,inb,ind,ino,iou,ipi,isl,isn,ita,iws,ixl,jac,jae,jao,jav,jic,jid,jiv,jni,jpn,jvn,kab,kac,kam,kan,kaq,kas,kat,kaz,kbc,kbh,kbm,kbp,kbq,kdc,kde,kdl,kea,kek,ken,kew,kgf,kgk,kgp,khk,khm,khs,khz,kik,kin,kir,kiw,kiz,kje,kjs,kkc,kkl,klt,klv,kmb,kmg,kmh,kmk,kmo,kmr,kms,kmu,knc,kne,knf,knj,knv,kon,kor,kos,kpf,kpg,kpj,kpr,kpw,kpx,kqa,kqc,kqf,kql,kqw,krc,ksd,ksj,ksr,ktm,kto,kud,kue,kup,kur,kvg,kvn,kwd,kwf,kwi,kwj,kyc,kyf,kyg,kyq,kyz,kze,kzj,lac,lao,lat,lav,lbb,lbk,lcm,leu,lex,lfn,lgl,lid,lif,lij,lim,lin,lit,llg,lmo,ltg,ltz,lua,lug,luo,lus,lvs,lww,maa,mad,mag,mai,maj,mak,mal,mam,maq,mar,mau,mav,max,maz,mbb,mbc,mbh,mbj,mbl,mbs,mbt,mca,mcb,mcd,mcf,mco,mcp,mcq,mcr,mdy,med,mee,mek,meq,met,meu,mey,mgc,mgh,mgw,mhl,mhr,mib,mic,mie,mig,mih,mil,min,mio,mir,mit,miz,mjc,mkd,mkj,mkl,mkn,mks,mle,mlg,mlh,mlp,mlt,mmo,mmx,mna,mni,mon,mop,mos,mox,mph,mpj,mpm,mpp,mps,mpt,mpx,mqb,mqj,mri,msa,msb,msc,msk,msm,msy,mti,mto,mui,mup,mux,muy,mva,mvn,mwc,mwe,mwf,mwp,mwr,mxb,mxp,mxq,mxt,mya,myk,myu,myw,myy,mzz,nab,naf,nak,nas,nbq,nca,nch,ncj,ncl,ncu,nde,ndg,ndj,nds,nep,nfa,ngp,ngu,nhe,nhg,nhi,nho,nhr,nhu,nhw,nhy,nif,nii,nij,nin,nko,nld,nlg,nna,nno,nnq,noa,nob,nop,nor,not,nou,nov,npi,npl,nqo,nsn,nso,nss,ntj,ntp,ntu,nus,nuy,nvm,nwi,nya,nys,nyu,obo,oci,okv,omw,ong,ons,ood,opm,orm,orv,ory,ote,otm,otn,otq,ots,pab,pad,pag,pah,pam,pan,pao,pap,pbt,pcm,pes,pib,pio,pir,piu,pjt,pls,plt,plu,pma,pms,poe,poh,poi,pol,pon,por,poy,ppo,prf,pri,prs,ptp,ptu,pus,pwg,qub,quc,quf,quh,qul,qup,quy,qvc,qve,qvh,qvm,qvn,qvs,qvw,qvz,qwh,qxh,qxn,qxo,rai,raj,reg,rej,rgu,rkb,rmc,rmy,rom,ron,roo,rop,row,rro,ruf,rug,run,rus,rwo,sab,sag,sah,san,sat,sbe,sbk,sbs,scn,sco,seh,sey,sgb,sgz,shi,shj,shn,shp,sim,sin,sja,slk,sll,slv,smk,smo,sna,snc,snd,snn,snp,snx,sny,som,soq,sot,soy,spa,spl,spm,spp,sps,spy,sqi,srd,sri,srm,srn,srp,srq,ssd,ssg,ssw,ssx,stp,sua,sue,sun,sus,suz,svk,swa,swe,swg,swh,swp,sxb,szl,tac,tah,taj,tam,taq,tat,tav,taw,tbc,tbf,tbg,tbo,tbz,tca,tcs,tcz,tdt,tee,tel,ter,tet,tew,tfr,tgk,tgl,tgo,tgp,tha,tif,tim,tir,tiw,tiy,tke,tku,tlf,tmd,tna,tnc,tnk,tnn,tnp,toc,tod,tof,toj,ton,too,top,tos,tpa,tpi,tpt,tpz,trc,tsn,tso,tsw,ttc,tte,tuc,tue,tuf,tuk,tum,tuo,tur,tvk,twi,txq,txu,tyv,tzj,tzl,tzm,tzo,ubr,ubu,udu,uig,ukr,uli,ulk,umb,upv,ura,urb,urd,uri,urt,urw,usa,usp,uvh,uvl,uzb,uzn,vec,ven,vid,vie,viv,vmy,waj,wal,wap,war,wat,wbi,wbp,wed,wer,wim,wiu,wiv,wln,wmt,wmw,wnc,wnu,wol,wos,wrk,wro,wrs,wsk,wuu,wuv,xav,xbi,xed,xho,xla,xnn,xon,xsi,xtd,xtm,yaa,yad,yal,yap,yaq,yby,ycn,ydd,yid,yka,yle,yml,yon,yor,yrb,yre,yss,yue,yuj,yut,yuw,yva,zaa,zab,zac,zad,zai,zaj,zam,zao,zap,zar,zas,zat,zav,zaw,zca,zga,zho,zia,ziw,zlm,zos,zpc,zpl,zpm,zpo,zpq,zpu,zpv,zpz,zsm,zsr,ztq,zty,zul,zyp |
 | [MTEB(Scandinavian, v1)](https://kennethenevoldsen.github.io/scandinavian-embedding-benchmark/) | Scandinavian | 28 | BitextMining: 2, Classification: 13, Retrieval: 7, Clustering: 6 | [Blog, Encyclopaedic, Fiction, Government, Legal, News, Non-fiction, Reviews, Social, Spoken, Web, Written] | dan,fao,isl,nno,nob,swe |
 | [MTEB(cmn, v1)](https://github.com/FlagOpen/FlagEmbedding/tree/master/research/C_MTEB) | Chinese | 32 | Retrieval: 8, Reranking: 4, PairClassification: 2, Clustering: 4, STS: 7, Classification: 7 | [Academic, Entertainment, Financial, Government, Medical, Non-fiction, Written] | cmn |

mteb/abstasks/TaskMetadata.py

@@ -235,16 +235,15 @@
 METRIC_VALUE = Union[int, float, dict[str, Any]]
 
 
-class PromptDict(TypedDict, total=False):
-    """A dictionary containing the prompt used for the task.
-
-    Args:
-        query: The prompt used for the queries in the task.
-        passage: The prompt used for the passages in the task.
-    """
+PromptDict = TypedDict(
+    "PromptDict", {prompt_type.value: str for prompt_type in PromptType}, total=False
+)
+"""A dictionary containing the prompt used for the task.
 
-    query: str
-    passage: str
+Args:
+    query: The prompt used for the queries in the task.
+    document: The prompt used for the passages in the task.
+"""
 
 
 class DescriptiveStatistics(TypedDict):
@@ -253,9 +252,6 @@ class DescriptiveStatistics(TypedDict):
     pass
 
 
-METRIC_VALUE = Union[int, float, dict[str, Any]]
-
-
 logger = logging.getLogger(__name__)
 
 
@@ -289,6 +285,7 @@ class TaskMetadata(BaseModel):
         prompt: The prompt used for the task. Can be a string or a dictionary containing the query and passage prompts.
         bibtex_citation: The BibTeX citation for the dataset. Should be an empty string if no citation is available.
         adapted_from: Datasets adapted (translated, sampled from, etc.) from other datasets.
+        is_public: Whether the dataset is publicly available. If False (closed/private), a HuggingFace token is required to run the datasets.
     """
 
     dataset: dict[str, Any]
@@ -316,35 +313,39 @@ class TaskMetadata(BaseModel):
     sample_creation: SAMPLE_CREATION_METHOD | None = None
     bibtex_citation: str | None = None
     adapted_from: list[str] | None = None
+    is_public: bool = True
 
     def validate_metadata(self) -> None:
         self.dataset_path_is_specified(self.dataset)
         self.dataset_revision_is_specified(self.dataset)
         self.eval_langs_are_valid(self.eval_langs)
 
     @field_validator("dataset")
+    @classmethod
     def _check_dataset_path_is_specified(
         cls, dataset: dict[str, Any]
     ) -> dict[str, Any]:
         cls.dataset_path_is_specified(dataset)
         return dataset
 
     @field_validator("dataset")
+    @classmethod
     def _check_dataset_revision_is_specified(
         cls, dataset: dict[str, Any]
     ) -> dict[str, Any]:
         cls.dataset_revision_is_specified(dataset)
         return dataset
 
     @field_validator("prompt")
+    @classmethod
     def _check_prompt_is_valid(
         cls, prompt: str | PromptDict | None
     ) -> str | PromptDict | None:
         if isinstance(prompt, dict):
             for key in prompt:
                 if key not in [e.value for e in PromptType]:
                     raise ValueError(
-                        "The prompt dictionary should only contain the keys 'query' and 'passage'."
+                        "The prompt dictionary should only contain the keys 'query' and 'document'."
                     )
         return prompt
 
@@ -419,7 +420,7 @@ def is_filled(self) -> bool:
         return all(
             getattr(self, field_name) is not None
             for field_name in self.model_fields
-            if field_name not in ["prompt", "adapted_from"]
+            if field_name not in ["prompt", "adapted_from", "is_public"]
         )
 
     @property

mteb/benchmarks/benchmarks/__init__.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+from mteb.benchmarks.benchmark import Benchmark
+from mteb.benchmarks.benchmarks.benchmarks import (
+    BEIR,
+    BEIR_NL,
+    BRIGHT,
+    BRIGHT_LONG,
+    BUILT_MTEB,
+    C_MTEB,
+    CHEMTEB,
+    CODE_RAG,
+    ENCODECHKA,
+    FA_MTEB,
+    JINA_VDR,
+    LONG_EMBED,
+    MIEB_ENG,
+    MIEB_IMG,
+    MIEB_LITE,
+    MIEB_MULTILINGUAL,
+    MTEB_DEU,
+    MTEB_EN,
+    MTEB_ENG_CLASSIC,
+    MTEB_EU,
+    MTEB_FRA,
+    MTEB_INDIC,
+    MTEB_JPN,
+    MTEB_KOR,
+    MTEB_MAIN_RU,
+    MTEB_MINERS_BITEXT_MINING,
+    MTEB_POL,
+    MTEB_RETRIEVAL_LAW,
+    MTEB_RETRIEVAL_MEDICAL,
+    MTEB_RETRIEVAL_WITH_INSTRUCTIONS,
+    NANOBEIR,
+    R2MED,
+    RU_SCI_BENCH,
+    SEB,
+    VIDORE,
+    VIDORE_V2,
+    VISUAL_DOCUMENT_RETRIEVAL,
+    VN_MTEB,
+    CoIR,
+    MTEB_code,
+    MTEB_multilingual_v1,
+    MTEB_multilingual_v2,
+    RAR_b,
+)
+
+__all__ = [
+    "Benchmark",
+    "MTEB_EN",
+    "MTEB_ENG_CLASSIC",
+    "MTEB_MAIN_RU",
+    "RU_SCI_BENCH",
+    "MTEB_RETRIEVAL_WITH_INSTRUCTIONS",
+    "MTEB_RETRIEVAL_LAW",
+    "MTEB_RETRIEVAL_MEDICAL",
+    "MTEB_MINERS_BITEXT_MINING",
+    "SEB",
+    "CoIR",
+    "RAR_b",
+    "MTEB_FRA",
+    "MTEB_DEU",
+    "MTEB_KOR",
+    "MTEB_POL",
+    "MTEB_code",
+    "MTEB_multilingual_v1",
+    "MTEB_multilingual_v2",
+    "MTEB_JPN",
+    "MTEB_INDIC",
+    "MTEB_EU",
+    "LONG_EMBED",
+    "BRIGHT",
+    "BRIGHT_LONG",
+    "CODE_RAG",
+    "BEIR",
+    "NANOBEIR",
+    "C_MTEB",
+    "FA_MTEB",
+    "CHEMTEB",
+    "BEIR_NL",
+    "MIEB_ENG",
+    "MIEB_MULTILINGUAL",
+    "MIEB_LITE",
+    "MIEB_IMG",
+    "BUILT_MTEB",
+    "ENCODECHKA",
+    "VIDORE",
+    "VIDORE_V2",
+    "VISUAL_DOCUMENT_RETRIEVAL",
+    "R2MED",
+    "VN_MTEB",
+    "JINA_VDR",
+]

mteb/benchmarks/benchmarks.py → mteb/benchmarks/benchmarks/benchmarks.py

@@ -855,7 +855,7 @@
     ],
 )
 
-MTEB_multilingual = Benchmark(
+MTEB_multilingual_v1 = Benchmark(
     name="MTEB(Multilingual, v1)",
     display_name="Multilingual",
     icon="https://github.com/DennisSuitters/LibreICONS/raw/2d2172d15e3c6ca03c018629d60050e4b99e5c55/svg-color/libre-gui-globe.svg",
@@ -869,7 +869,7 @@
 )
 
 
-MTEB_multilingual = Benchmark(
+MTEB_multilingual_v2 = Benchmark(
     name="MTEB(Multilingual, v2)",
     display_name="Multilingual",
     icon="https://github.com/DennisSuitters/LibreICONS/raw/2d2172d15e3c6ca03c018629d60050e4b99e5c55/svg-color/libre-gui-globe.svg",