dataset: Adding JapaneseCode1Retrieval as the first non-public dataset

.github/workflows/dataset_loading.yml

@@ -22,6 +22,12 @@ jobs:
       run: |
           make install-for-tests
 
+    - name: HF auth
+      env:
+        HF_TOKEN: ${{ secrets.MTEB_PRIVATE }}
+      run: |
+        hf auth login --token $HF_TOKEN
+
     - name: Run dataset loading tests
       run: |
         make dataset-load-test

.github/workflows/dataset_loading_pr.yml

@@ -1,17 +1,26 @@
 name: Datasets available on HuggingFace - PR
 
 on:
-  pull_request:
+  pull_request_target:
+    types: [opened, synchronize, edited]
     paths:
       - "mteb/tasks/**.py"
 
+permissions:
+  contents: read
+  pull-requests: read
+
+
 jobs:
   run-pr-datasets-loading-check:
     runs-on: ubuntu-latest
 
     steps:
-    - name: Checkout repository
+    - name: Checkout code
       uses: actions/checkout@v4
+      with:
+        # IMPORTANT: For pull_request_target, check out the PR branch explicitly
+        ref: ${{ github.event.pull_request.head.sha }}
 
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -23,6 +32,12 @@ jobs:
       run: |
           make install-for-tests
 
+    - name: HF auth
+      env:
+        HF_TOKEN: ${{ secrets.MTEB_PRIVATE }}
+      run: |
+        hf auth login --token $HF_TOKEN
+
     - name: Run dataset loading tests
       run: |
         make dataset-load-test-pr BASE_BRANCH=${{ github.event.pull_request.base.ref }}

mteb/tasks/Retrieval/__init__.py

@@ -15,6 +15,7 @@
 from .code.DS1000Retrieval import *
 from .code.FreshStackRetrieval import *
 from .code.HumanEvalRetrieval import *
+from .code.JapaneseCode1Retrieval import *
 from .code.MBPPRetrieval import *
 from .code.StackOverflowQARetrieval import *
 from .code.SyntheticText2SqlRetrieval import *

mteb/tasks/Retrieval/code/JapaneseCode1Retrieval.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+
+class JapaneseCode1Retrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="JapaneseCode1Retrieval",
+        description="Japanese code retrieval dataset. Japanese natural language queries paired with Python code snippets for cross-lingual code retrieval evaluation.",
+        reference="https://huggingface.co/datasets/mteb-private/JapaneseCode1Retrieval-sample",
+        dataset={
+            "path": "mteb-private/JapaneseCode1Retrieval",
+            "revision": "fc4cb6390055e65490dfc42526e1d6a379e8cd86",
+        },
+        type="Retrieval",
+        category="s2p",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["jpn-Jpan"],
+        main_score="ndcg_at_10",
+        date=("2024-01-01", "2024-01-01"),
+        domains=["Programming", "Written"],
+        task_subtypes=["Code retrieval"],
+        license="not specified",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation="",
+        is_public=False,
+    )

tests/test_tasks/test_private_tasks.py

@@ -6,6 +6,7 @@
 
 # List of accepted private tasks - update this list as needed
 ACCEPTED_PRIVATE_TASKS = [
+    "JapaneseCode1Retrieval",
     # Add task names here that are allowed to be private
     # Example: "SomePrivateTask"
 ]
@@ -31,6 +32,6 @@ def test_private_tasks_fail_unless_accepted():
 def test_accepted_private_task_exist(task_name: str):
     """Test that all tasks in ACCEPTED_PRIVATE_TASKS actually exist and are private."""
     task = get_task(task_name)
-    assert task.metadata.is_public == (
+    assert task.metadata.is_public is False, (
         f"Task '{task_name}' is in ACCEPTED_PRIVATE_TASKS but is not private (is_public={task.metadata.is_public})"
     )