From 0bffbccac031305d914a92f5e3d65c2c94e52a55 Mon Sep 17 00:00:00 2001 From: fzoll Date: Wed, 10 Sep 2025 17:11:45 +0200 Subject: [PATCH 01/30] Adding JapaneseCode1Retrieval as the first non-public dataset --- mteb/tasks/Retrieval/__init__.py | 1 + .../Retrieval/code/JapaneseCode1Retrieval.py | 95 +++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 mteb/tasks/Retrieval/code/JapaneseCode1Retrieval.py diff --git a/mteb/tasks/Retrieval/__init__.py b/mteb/tasks/Retrieval/__init__.py index 670d4ea5b1..1b25b0b37b 100644 --- a/mteb/tasks/Retrieval/__init__.py +++ b/mteb/tasks/Retrieval/__init__.py @@ -15,6 +15,7 @@ from .code.DS1000Retrieval import * from .code.FreshStackRetrieval import * from .code.HumanEvalRetrieval import * +from .code.JapaneseCode1Retrieval import * from .code.MBPPRetrieval import * from .code.StackOverflowQARetrieval import * from .code.SyntheticText2SqlRetrieval import * diff --git a/mteb/tasks/Retrieval/code/JapaneseCode1Retrieval.py b/mteb/tasks/Retrieval/code/JapaneseCode1Retrieval.py new file mode 100644 index 0000000000..62cd6ee527 --- /dev/null +++ b/mteb/tasks/Retrieval/code/JapaneseCode1Retrieval.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import logging +from pathlib import Path + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + +logger = logging.getLogger(__name__) + + +class JapaneseCode1Retrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="JapaneseCode1Retrieval", + description="Japanese code retrieval dataset. Japanese natural language queries paired with Python code snippets for cross-lingual code retrieval evaluation.", + reference="https://huggingface.co/datasets/mteb-private/JapaneseCode1Retrieval", + dataset={ + "path": "local", + "revision": "1.0.0", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train", "test"], + eval_langs=["jpn-Jpan"], + main_score="ndcg_at_10", + date=("2024-01-01", "2024-01-01"), + domains=["Programming", "Written"], + task_subtypes=["Code retrieval"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation="", + is_public=False, + ) + + def load_data(self, **kwargs): + """Load data from local directory.""" + if self.data_loaded: + return + + # Path to the local dataset + dataset_path = Path( + "/Users/fodizoltan/Projects/toptal/voyageai/tmp/rteb/data/JapaneseCode1Retrieval" + ) + + self.corpus, self.queries, self.relevant_docs = {}, {}, {} + + for split in kwargs.get("eval_splits", self.metadata_dict["eval_splits"]): + logger.info(f"Loading {split} split...") + + # Load the dataset from parquet files + import pandas as pd + + split_path = dataset_path / f"{split}.parquet" + + if not split_path.exists(): + raise FileNotFoundError(f"Split file not found: {split_path}") + + df = pd.read_parquet(split_path) + + # Initialize split dictionaries + corpus_split = {} + queries_split = {} + qrels_split = {} + + # Process each row in the dataframe + for _, row in df.iterrows(): + qid = row["qid"] + query_text = row["query"] + positive_docs = row["positive"] if row["positive"] is not None else [] + + # Add query + queries_split[qid] = query_text + + # Add documents to corpus and create qrels + qrels_split[qid] = {} + + for doc in positive_docs: + doc_id = doc["docid"] + doc_text = doc["text"] + corpus_split[doc_id] = doc_text + qrels_split[qid][doc_id] = 1 + + # Store the processed data + self.corpus[split] = corpus_split + self.queries[split] = queries_split + self.relevant_docs[split] = qrels_split + + logger.info( + f"Loaded {len(queries_split)} queries and {len(corpus_split)} documents for {split} split" + ) + + self.data_loaded = True From da4fcad869c8455e68b143629507849919ce7316 Mon Sep 17 00:00:00 2001 From: fzoll Date: Wed, 10 Sep 2025 22:44:41 +0200 Subject: [PATCH 02/30] Transformed dataset --- .../Retrieval/code/JapaneseCode1Retrieval.py | 70 +------------------ 1 file changed, 3 insertions(+), 67 deletions(-) diff --git a/mteb/tasks/Retrieval/code/JapaneseCode1Retrieval.py b/mteb/tasks/Retrieval/code/JapaneseCode1Retrieval.py index 62cd6ee527..372dd2d57e 100644 --- a/mteb/tasks/Retrieval/code/JapaneseCode1Retrieval.py +++ b/mteb/tasks/Retrieval/code/JapaneseCode1Retrieval.py @@ -1,13 +1,8 @@ from __future__ import annotations -import logging -from pathlib import Path - from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval from mteb.abstasks.TaskMetadata import TaskMetadata -logger = logging.getLogger(__name__) - class JapaneseCode1Retrieval(AbsTaskRetrieval): metadata = TaskMetadata( @@ -15,13 +10,13 @@ class JapaneseCode1Retrieval(AbsTaskRetrieval): description="Japanese code retrieval dataset. Japanese natural language queries paired with Python code snippets for cross-lingual code retrieval evaluation.", reference="https://huggingface.co/datasets/mteb-private/JapaneseCode1Retrieval", dataset={ - "path": "local", - "revision": "1.0.0", + "path": "mteb-private/JapaneseCode1Retrieval", + "revision": "fc4cb6390055e65490dfc42526e1d6a379e8cd86", }, type="Retrieval", category="s2p", modalities=["text"], - eval_splits=["train", "test"], + eval_splits=["test"], eval_langs=["jpn-Jpan"], main_score="ndcg_at_10", date=("2024-01-01", "2024-01-01"), @@ -34,62 +29,3 @@ class JapaneseCode1Retrieval(AbsTaskRetrieval): bibtex_citation="", is_public=False, ) - - def load_data(self, **kwargs): - """Load data from local directory.""" - if self.data_loaded: - return - - # Path to the local dataset - dataset_path = Path( - "/Users/fodizoltan/Projects/toptal/voyageai/tmp/rteb/data/JapaneseCode1Retrieval" - ) - - self.corpus, self.queries, self.relevant_docs = {}, {}, {} - - for split in kwargs.get("eval_splits", self.metadata_dict["eval_splits"]): - logger.info(f"Loading {split} split...") - - # Load the dataset from parquet files - import pandas as pd - - split_path = dataset_path / f"{split}.parquet" - - if not split_path.exists(): - raise FileNotFoundError(f"Split file not found: {split_path}") - - df = pd.read_parquet(split_path) - - # Initialize split dictionaries - corpus_split = {} - queries_split = {} - qrels_split = {} - - # Process each row in the dataframe - for _, row in df.iterrows(): - qid = row["qid"] - query_text = row["query"] - positive_docs = row["positive"] if row["positive"] is not None else [] - - # Add query - queries_split[qid] = query_text - - # Add documents to corpus and create qrels - qrels_split[qid] = {} - - for doc in positive_docs: - doc_id = doc["docid"] - doc_text = doc["text"] - corpus_split[doc_id] = doc_text - qrels_split[qid][doc_id] = 1 - - # Store the processed data - self.corpus[split] = corpus_split - self.queries[split] = queries_split - self.relevant_docs[split] = qrels_split - - logger.info( - f"Loaded {len(queries_split)} queries and {len(corpus_split)} documents for {split} split" - ) - - self.data_loaded = True From e1966635814ef1a941181e0f7f51ca11af4a7190 Mon Sep 17 00:00:00 2001 From: fzoll Date: Wed, 10 Sep 2025 22:54:31 +0200 Subject: [PATCH 03/30] Adding as private dataset to tests --- tests/test_tasks/test_private_tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_tasks/test_private_tasks.py b/tests/test_tasks/test_private_tasks.py index 50139dbdfd..45348027c5 100644 --- a/tests/test_tasks/test_private_tasks.py +++ b/tests/test_tasks/test_private_tasks.py @@ -6,6 +6,7 @@ # List of accepted private tasks - update this list as needed ACCEPTED_PRIVATE_TASKS = [ + "JapaneseCode1Retrieval", # Add task names here that are allowed to be private # Example: "SomePrivateTask" ] From f0807c39b7c0125317a68312d5f2b04f95ca52c6 Mon Sep 17 00:00:00 2001 From: fzoll Date: Thu, 11 Sep 2025 00:42:35 +0200 Subject: [PATCH 04/30] Correct the private task test --- tests/test_tasks/test_private_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_tasks/test_private_tasks.py b/tests/test_tasks/test_private_tasks.py index 45348027c5..15c2ec4124 100644 --- a/tests/test_tasks/test_private_tasks.py +++ b/tests/test_tasks/test_private_tasks.py @@ -32,6 +32,6 @@ def test_private_tasks_fail_unless_accepted(): def test_accepted_private_task_exist(task_name: str): """Test that all tasks in ACCEPTED_PRIVATE_TASKS actually exist and are private.""" task = get_task(task_name) - assert task.metadata.is_public == ( + assert task.metadata.is_public is False, ( f"Task '{task_name}' is in ACCEPTED_PRIVATE_TASKS but is not private (is_public={task.metadata.is_public})" ) From ef047d3d7f2d52d01fc9f3d8996cf365cb201cc7 Mon Sep 17 00:00:00 2001 From: fzoll Date: Thu, 11 Sep 2025 12:41:34 +0200 Subject: [PATCH 05/30] Use the sample dataset as a reference --- mteb/tasks/Retrieval/code/JapaneseCode1Retrieval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mteb/tasks/Retrieval/code/JapaneseCode1Retrieval.py b/mteb/tasks/Retrieval/code/JapaneseCode1Retrieval.py index 372dd2d57e..1a5df870f2 100644 --- a/mteb/tasks/Retrieval/code/JapaneseCode1Retrieval.py +++ b/mteb/tasks/Retrieval/code/JapaneseCode1Retrieval.py @@ -8,7 +8,7 @@ class JapaneseCode1Retrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="JapaneseCode1Retrieval", description="Japanese code retrieval dataset. Japanese natural language queries paired with Python code snippets for cross-lingual code retrieval evaluation.", - reference="https://huggingface.co/datasets/mteb-private/JapaneseCode1Retrieval", + reference="https://huggingface.co/datasets/mteb-private/JapaneseCode1Retrieval-sample", dataset={ "path": "mteb-private/JapaneseCode1Retrieval", "revision": "fc4cb6390055e65490dfc42526e1d6a379e8cd86", From 2f48f66ad17cdcda9d9d9e8b308738e7a929c9a0 Mon Sep 17 00:00:00 2001 From: fzoll Date: Thu, 11 Sep 2025 17:44:22 +0200 Subject: [PATCH 06/30] Use the sample dataset as a reference --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1400c284e1..ce554fcb33 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,6 +45,8 @@ jobs: - name: Run tests if: runner.os != 'Windows' shell: bash + env: + HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} run: | make test @@ -54,6 +56,8 @@ jobs: # failures when running on windows when loading the datasets if: runner.os == 'Windows' shell: bash + env: + HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} run: | # run the test once and if it fails, run it again # if it fails again, the workflow will fail. From 84566689a2e490c3e91095f72147540937778eff Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Thu, 11 Sep 2025 18:59:42 +0300 Subject: [PATCH 07/30] fix ds loading --- .github/workflows/dataset_loading_pr.yml | 6 ++++++ .github/workflows/test.yml | 4 ---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index c9de1d2300..6eadfeb079 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -23,6 +23,12 @@ jobs: run: | make install-for-tests + - name: HF auth + env: + HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} + run: | + hf auth login --token $HF_TOKEN + - name: Run dataset loading tests run: | make dataset-load-test-pr BASE_BRANCH=${{ github.event.pull_request.base.ref }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ce554fcb33..1400c284e1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,8 +45,6 @@ jobs: - name: Run tests if: runner.os != 'Windows' shell: bash - env: - HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} run: | make test @@ -56,8 +54,6 @@ jobs: # failures when running on windows when loading the datasets if: runner.os == 'Windows' shell: bash - env: - HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} run: | # run the test once and if it fails, run it again # if it fails again, the workflow will fail. From 53476f6f4dde6892918fa146fe48a532cca31d7a Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Thu, 11 Sep 2025 19:12:57 +0300 Subject: [PATCH 08/30] allow on forks --- .github/workflows/dataset_loading.yml | 6 ++++++ .github/workflows/dataset_loading_pr.yml | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/.github/workflows/dataset_loading.yml b/.github/workflows/dataset_loading.yml index 3f8b4b040f..e488436b81 100644 --- a/.github/workflows/dataset_loading.yml +++ b/.github/workflows/dataset_loading.yml @@ -22,6 +22,12 @@ jobs: run: | make install-for-tests + - name: HF auth + env: + HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} + run: | + hf auth login --token $HF_TOKEN + - name: Run dataset loading tests run: | make dataset-load-test diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index 6eadfeb079..0ff1c209a1 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -5,6 +5,11 @@ on: paths: - "mteb/tasks/**.py" +permissions: + contents: read + pull-requests: write + + jobs: run-pr-datasets-loading-check: runs-on: ubuntu-latest From cb655a595ab1816865efa03396cbda39ee2250d1 Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Thu, 11 Sep 2025 19:32:04 +0300 Subject: [PATCH 09/30] upd aciton --- .github/workflows/dataset_loading_pr.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index 0ff1c209a1..0385555a9d 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -1,14 +1,10 @@ name: Datasets available on HuggingFace - PR on: - pull_request: + pull_request_target: paths: - "mteb/tasks/**.py" -permissions: - contents: read - pull-requests: write - jobs: run-pr-datasets-loading-check: From 08d7c873712e637655856107dd85536c46d89c5a Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Thu, 11 Sep 2025 19:37:11 +0300 Subject: [PATCH 10/30] remove paths --- .github/workflows/dataset_loading_pr.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index 0385555a9d..eeb3339305 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -2,8 +2,6 @@ name: Datasets available on HuggingFace - PR on: pull_request_target: - paths: - - "mteb/tasks/**.py" jobs: From 10eea6a972cd838a2beef2c4ed5a0c8a50d5e069 Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Thu, 11 Sep 2025 19:40:49 +0300 Subject: [PATCH 11/30] try to trigger ci --- .github/workflows/dataset_loading_pr.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index eeb3339305..88b49c7757 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -2,7 +2,9 @@ name: Datasets available on HuggingFace - PR on: pull_request_target: - + types: [opened, synchronize, edited] + paths: + - "mteb/tasks/**.py" jobs: run-pr-datasets-loading-check: From c6aded6c9c6cca23a7a536f2a033c0bf5221474f Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Thu, 11 Sep 2025 19:44:00 +0300 Subject: [PATCH 12/30] add ref --- .github/workflows/dataset_loading_pr.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index 88b49c7757..de1b8bbb9d 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -11,8 +11,11 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout repository + - name: Checkout code uses: actions/checkout@v4 + with: + # IMPORTANT: For pull_request_target, check out the PR branch explicitly + ref: ${{ github.event.pull_request.head.sha }} - name: Set up Python uses: actions/setup-python@v4 From 1242651c817ddae25410cfe34af16692d25a1c59 Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Thu, 11 Sep 2025 19:49:10 +0300 Subject: [PATCH 13/30] add permissions --- .github/workflows/dataset_loading_pr.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index de1b8bbb9d..4f4cdae766 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -6,6 +6,11 @@ on: paths: - "mteb/tasks/**.py" +permissions: + contents: read + pull-requests: read + + jobs: run-pr-datasets-loading-check: runs-on: ubuntu-latest From db30e8ab6f39019e24197dc513b2f192c2c7a6ab Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Thu, 11 Sep 2025 19:50:32 +0300 Subject: [PATCH 14/30] remove paths --- .github/workflows/dataset_loading_pr.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index 4f4cdae766..67ce8711bf 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -3,8 +3,6 @@ name: Datasets available on HuggingFace - PR on: pull_request_target: types: [opened, synchronize, edited] - paths: - - "mteb/tasks/**.py" permissions: contents: read From d1e4553dd2046bbdd63a73dd4a8be17ad759606b Mon Sep 17 00:00:00 2001 From: Solomatin Roman Date: Fri, 12 Sep 2025 10:49:14 +0300 Subject: [PATCH 15/30] add paths back --- .github/workflows/dataset_loading_pr.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index 67ce8711bf..4f4cdae766 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -3,6 +3,8 @@ name: Datasets available on HuggingFace - PR on: pull_request_target: types: [opened, synchronize, edited] + paths: + - "mteb/tasks/**.py" permissions: contents: read From 7a9167dd416634ed8493883a043f88725d842fb0 Mon Sep 17 00:00:00 2001 From: Solomatin Roman Date: Fri, 12 Sep 2025 12:19:52 +0300 Subject: [PATCH 16/30] get back to pull request --- .github/workflows/dataset_loading_pr.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index 4f4cdae766..dfff8aae33 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -1,8 +1,7 @@ name: Datasets available on HuggingFace - PR on: - pull_request_target: - types: [opened, synchronize, edited] + pull_request: paths: - "mteb/tasks/**.py" @@ -18,9 +17,6 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 - with: - # IMPORTANT: For pull_request_target, check out the PR branch explicitly - ref: ${{ github.event.pull_request.head.sha }} - name: Set up Python uses: actions/setup-python@v4 From 0582b6547de4f0174d708449dbf4e19cf7e6ba01 Mon Sep 17 00:00:00 2001 From: Solomatin Roman Date: Fri, 12 Sep 2025 12:37:39 +0300 Subject: [PATCH 17/30] rollback action --- .github/workflows/dataset_loading_pr.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index dfff8aae33..c9de1d2300 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -5,17 +5,12 @@ on: paths: - "mteb/tasks/**.py" -permissions: - contents: read - pull-requests: read - - jobs: run-pr-datasets-loading-check: runs-on: ubuntu-latest steps: - - name: Checkout code + - name: Checkout repository uses: actions/checkout@v4 - name: Set up Python @@ -28,12 +23,6 @@ jobs: run: | make install-for-tests - - name: HF auth - env: - HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} - run: | - hf auth login --token $HF_TOKEN - - name: Run dataset loading tests run: | make dataset-load-test-pr BASE_BRANCH=${{ github.event.pull_request.base.ref }} From b297b78d25e718fd2267f4c76a56bc0624a65d3e Mon Sep 17 00:00:00 2001 From: fzoll Date: Fri, 12 Sep 2025 19:08:08 +0200 Subject: [PATCH 18/30] Trying to resolve the token/secret problem --- .github/workflows/dataset_loading.yml | 2 +- .github/workflows/dataset_loading_pr.yml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dataset_loading.yml b/.github/workflows/dataset_loading.yml index e488436b81..cc910a0a4c 100644 --- a/.github/workflows/dataset_loading.yml +++ b/.github/workflows/dataset_loading.yml @@ -26,7 +26,7 @@ jobs: env: HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} run: | - hf auth login --token $HF_TOKEN + hf auth login --token "$HF_TOKEN" - name: Run dataset loading tests run: | diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index c9de1d2300..edef80bab9 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -23,6 +23,12 @@ jobs: run: | make install-for-tests + - name: HF auth + env: + HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} + run: | + hf auth login --token "$HF_TOKEN" + - name: Run dataset loading tests run: | make dataset-load-test-pr BASE_BRANCH=${{ github.event.pull_request.base.ref }} From 2224ed5cfdab3bb1112e7d8ad41f15c3d86c9d0c Mon Sep 17 00:00:00 2001 From: fzoll Date: Fri, 12 Sep 2025 19:18:36 +0200 Subject: [PATCH 19/30] Trying to resolve the token/secret problem --- .github/workflows/dataset_loading.yml | 6 +----- .github/workflows/dataset_loading_pr.yml | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/dataset_loading.yml b/.github/workflows/dataset_loading.yml index cc910a0a4c..03a7de55d6 100644 --- a/.github/workflows/dataset_loading.yml +++ b/.github/workflows/dataset_loading.yml @@ -22,12 +22,8 @@ jobs: run: | make install-for-tests - - name: HF auth + - name: Run dataset loading tests env: HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} - run: | - hf auth login --token "$HF_TOKEN" - - - name: Run dataset loading tests run: | make dataset-load-test diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index edef80bab9..675ed679c6 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -23,12 +23,8 @@ jobs: run: | make install-for-tests - - name: HF auth + - name: Run dataset loading tests env: HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} - run: | - hf auth login --token "$HF_TOKEN" - - - name: Run dataset loading tests run: | make dataset-load-test-pr BASE_BRANCH=${{ github.event.pull_request.base.ref }} From 3364b46d93c649f93196cb7eee4804642970a587 Mon Sep 17 00:00:00 2001 From: Roman Solomatin Date: Fri, 12 Sep 2025 22:20:38 +0300 Subject: [PATCH 20/30] Update dataset_loading_pr.yml --- .github/workflows/dataset_loading_pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index 675ed679c6..35d12050f0 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -1,7 +1,7 @@ name: Datasets available on HuggingFace - PR on: - pull_request: + pull_request_target: paths: - "mteb/tasks/**.py" From 669a24f4e98803dad95c08fe6e7ec6e4f59e9f2c Mon Sep 17 00:00:00 2001 From: Roman Solomatin Date: Fri, 12 Sep 2025 22:21:54 +0300 Subject: [PATCH 21/30] Update dataset_loading_pr.yml --- .github/workflows/dataset_loading_pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index 35d12050f0..675ed679c6 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -1,7 +1,7 @@ name: Datasets available on HuggingFace - PR on: - pull_request_target: + pull_request: paths: - "mteb/tasks/**.py" From d53e9a00e8195669b5a8d87acfe40f9220fdf824 Mon Sep 17 00:00:00 2001 From: fzoll Date: Sat, 13 Sep 2025 19:59:12 +0200 Subject: [PATCH 22/30] Try the latest datasets package (worked for me) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3398e29fd1..19e7479462 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ ] requires-python = ">=3.9,<3.13" dependencies = [ - "datasets>=2.19.0, <4.0.0", + "datasets==4.0.0", "numpy>=1.0.0,<3.0.0", "requests>=2.26.0", "scikit_learn>=1.0.2", From 2e1d0a07806fe47bc194663a3c7e2950b26302e4 Mon Sep 17 00:00:00 2001 From: fzoll Date: Sat, 13 Sep 2025 20:06:20 +0200 Subject: [PATCH 23/30] Try the latest datasets package (worked for me) --- .github/workflows/dataset_loading.yml | 6 ++++++ .github/workflows/dataset_loading_pr.yml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/.github/workflows/dataset_loading.yml b/.github/workflows/dataset_loading.yml index 03a7de55d6..eba57d13ae 100644 --- a/.github/workflows/dataset_loading.yml +++ b/.github/workflows/dataset_loading.yml @@ -22,6 +22,12 @@ jobs: run: | make install-for-tests + - name: HF auth + env: + HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} + run: | + hf auth login --token "$HF_TOKEN" + - name: Run dataset loading tests env: HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index 675ed679c6..c6cb06d36c 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -23,6 +23,12 @@ jobs: run: | make install-for-tests + - name: HF auth + env: + HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} + run: | + hf auth login --token "$HF_TOKEN" + - name: Run dataset loading tests env: HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} From ff541208a35d9250da14b2275372a5b45a82330b Mon Sep 17 00:00:00 2001 From: fzoll Date: Sat, 13 Sep 2025 20:10:31 +0200 Subject: [PATCH 24/30] Try the latest datasets package (worked for me) --- .github/workflows/dataset_loading.yml | 2 +- .github/workflows/dataset_loading_pr.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dataset_loading.yml b/.github/workflows/dataset_loading.yml index eba57d13ae..57456e9bda 100644 --- a/.github/workflows/dataset_loading.yml +++ b/.github/workflows/dataset_loading.yml @@ -26,7 +26,7 @@ jobs: env: HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} run: | - hf auth login --token "$HF_TOKEN" + hf auth login --add-to-git-credential --token "$HF_TOKEN" - name: Run dataset loading tests env: diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index c6cb06d36c..a139fd7076 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -27,7 +27,7 @@ jobs: env: HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} run: | - hf auth login --token "$HF_TOKEN" + hf auth login --add-to-git-credential --token "$HF_TOKEN" - name: Run dataset loading tests env: From 4ebc3b3e0bfe6f33087f5ba5a9cc440188b9c459 Mon Sep 17 00:00:00 2001 From: fzoll Date: Sat, 13 Sep 2025 20:24:25 +0200 Subject: [PATCH 25/30] (last?) try --- .github/workflows/dataset_loading_pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index a139fd7076..4c0e37e870 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -27,7 +27,7 @@ jobs: env: HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} run: | - hf auth login --add-to-git-credential --token "$HF_TOKEN" + hf auth login --add-to-git-credential --token "$HF_TOKEN" - name: Run dataset loading tests env: From efbf197ae7840f1abe119e341a6a323c4f159512 Mon Sep 17 00:00:00 2001 From: fzoll Date: Sat, 13 Sep 2025 20:26:49 +0200 Subject: [PATCH 26/30] (last?) try --- .github/workflows/dataset_loading_pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index 4c0e37e870..d2b4b5c064 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -1,7 +1,7 @@ name: Datasets available on HuggingFace - PR on: - pull_request: + pull_request_target: paths: - "mteb/tasks/**.py" From e55641095fc3fc966a2eab495787a49367f1daa7 Mon Sep 17 00:00:00 2001 From: fzoll Date: Sat, 13 Sep 2025 20:33:07 +0200 Subject: [PATCH 27/30] (last?) try --- .github/workflows/dataset_loading_pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index d2b4b5c064..4c0e37e870 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -1,7 +1,7 @@ name: Datasets available on HuggingFace - PR on: - pull_request_target: + pull_request: paths: - "mteb/tasks/**.py" From bea2d52e599865090f0a0d60cbcfa2ffee0bacb7 Mon Sep 17 00:00:00 2001 From: fzoll Date: Sat, 13 Sep 2025 20:45:25 +0200 Subject: [PATCH 28/30] Reverting the changes --- .github/workflows/dataset_loading.yml | 6 ------ .github/workflows/dataset_loading_pr.yml | 6 ------ pyproject.toml | 2 +- 3 files changed, 1 insertion(+), 13 deletions(-) diff --git a/.github/workflows/dataset_loading.yml b/.github/workflows/dataset_loading.yml index 57456e9bda..03a7de55d6 100644 --- a/.github/workflows/dataset_loading.yml +++ b/.github/workflows/dataset_loading.yml @@ -22,12 +22,6 @@ jobs: run: | make install-for-tests - - name: HF auth - env: - HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} - run: | - hf auth login --add-to-git-credential --token "$HF_TOKEN" - - name: Run dataset loading tests env: HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index 4c0e37e870..675ed679c6 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -23,12 +23,6 @@ jobs: run: | make install-for-tests - - name: HF auth - env: - HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} - run: | - hf auth login --add-to-git-credential --token "$HF_TOKEN" - - name: Run dataset loading tests env: HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} diff --git a/pyproject.toml b/pyproject.toml index 19e7479462..3398e29fd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ ] requires-python = ">=3.9,<3.13" dependencies = [ - "datasets==4.0.0", + "datasets>=2.19.0, <4.0.0", "numpy>=1.0.0,<3.0.0", "requests>=2.26.0", "scikit_learn>=1.0.2", From 87a831a2795a285cf3d046e6842c8fc7a81cb6a9 Mon Sep 17 00:00:00 2001 From: fzoll Date: Mon, 15 Sep 2025 12:37:16 +0200 Subject: [PATCH 29/30] Exclude the private datasets from tests --- scripts/extract_datasets.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/extract_datasets.py b/scripts/extract_datasets.py index c65a0ba153..9d97164542 100644 --- a/scripts/extract_datasets.py +++ b/scripts/extract_datasets.py @@ -5,7 +5,7 @@ import logging import os -from .extract_model_names import get_changed_files +from scripts.extract_model_names import get_changed_files logging.basicConfig(level=logging.INFO) @@ -70,6 +70,13 @@ def extract_datasets(files: list[str]) -> list[tuple[str, str]]: def extract_dataset_from_metadata(call_node: ast.Call) -> tuple[str, str] | None: """Extract dataset info from TaskMetadata call.""" + for keyword in call_node.keywords: + if ( + keyword.arg == "is_public" + and isinstance(keyword.value, ast.Constant) + and not keyword.value.value + ): + return None for keyword in call_node.keywords: if keyword.arg == "dataset" and isinstance(keyword.value, ast.Dict): return extract_dataset_from_dict(keyword.value) From ac6eab737f454a95e95825e64f89d6e34405603a Mon Sep 17 00:00:00 2001 From: Kenneth Enevoldsen Date: Tue, 16 Sep 2025 14:43:24 +0200 Subject: [PATCH 30/30] Apply suggestions from code review --- .github/workflows/dataset_loading.yml | 2 -- .github/workflows/dataset_loading_pr.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/.github/workflows/dataset_loading.yml b/.github/workflows/dataset_loading.yml index 03a7de55d6..3f8b4b040f 100644 --- a/.github/workflows/dataset_loading.yml +++ b/.github/workflows/dataset_loading.yml @@ -23,7 +23,5 @@ jobs: make install-for-tests - name: Run dataset loading tests - env: - HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} run: | make dataset-load-test diff --git a/.github/workflows/dataset_loading_pr.yml b/.github/workflows/dataset_loading_pr.yml index 675ed679c6..c9de1d2300 100644 --- a/.github/workflows/dataset_loading_pr.yml +++ b/.github/workflows/dataset_loading_pr.yml @@ -24,7 +24,5 @@ jobs: make install-for-tests - name: Run dataset loading tests - env: - HF_TOKEN: ${{ secrets.MTEB_PRIVATE }} run: | make dataset-load-test-pr BASE_BRANCH=${{ github.event.pull_request.base.ref }}