dataset: Adding JapaneseCode1Retrieval as the first non-public dataset

.github/workflows/dataset_loading.yml

@@ -23,5 +23,7 @@ jobs:
           make install-for-tests
 
     - name: Run dataset loading tests
+      env:
+        HF_TOKEN: ${{ secrets.MTEB_PRIVATE }}
       run: |
         make dataset-load-test

.github/workflows/dataset_loading_pr.yml

@@ -24,5 +24,7 @@ jobs:
           make install-for-tests
 
     - name: Run dataset loading tests
+      env:
+        HF_TOKEN: ${{ secrets.MTEB_PRIVATE }}
       run: |
         make dataset-load-test-pr BASE_BRANCH=${{ github.event.pull_request.base.ref }}

mteb/tasks/Retrieval/__init__.py

@@ -15,6 +15,7 @@
 from .code.DS1000Retrieval import *
 from .code.FreshStackRetrieval import *
 from .code.HumanEvalRetrieval import *
+from .code.JapaneseCode1Retrieval import *
 from .code.MBPPRetrieval import *
 from .code.StackOverflowQARetrieval import *
 from .code.SyntheticText2SqlRetrieval import *

mteb/tasks/Retrieval/code/JapaneseCode1Retrieval.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+
+class JapaneseCode1Retrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="JapaneseCode1Retrieval",
+        description="Japanese code retrieval dataset. Japanese natural language queries paired with Python code snippets for cross-lingual code retrieval evaluation.",
+        reference="https://huggingface.co/datasets/mteb-private/JapaneseCode1Retrieval-sample",
+        dataset={
+            "path": "mteb-private/JapaneseCode1Retrieval",
+            "revision": "fc4cb6390055e65490dfc42526e1d6a379e8cd86",
+        },
+        type="Retrieval",
+        category="s2p",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["jpn-Jpan"],
+        main_score="ndcg_at_10",
+        date=("2024-01-01", "2024-01-01"),
+        domains=["Programming", "Written"],
+        task_subtypes=["Code retrieval"],
+        license="not specified",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation="",
+        is_public=False,
+    )

tests/test_tasks/test_private_tasks.py

@@ -6,6 +6,7 @@
 
 # List of accepted private tasks - update this list as needed
 ACCEPTED_PRIVATE_TASKS = [
+    "JapaneseCode1Retrieval",
     # Add task names here that are allowed to be private
     # Example: "SomePrivateTask"
 ]
@@ -31,6 +32,6 @@ def test_private_tasks_fail_unless_accepted():
 def test_accepted_private_task_exist(task_name: str):
     """Test that all tasks in ACCEPTED_PRIVATE_TASKS actually exist and are private."""
     task = get_task(task_name)
-    assert task.metadata.is_public == (
+    assert task.metadata.is_public is False, (
         f"Task '{task_name}' is in ACCEPTED_PRIVATE_TASKS but is not private (is_public={task.metadata.is_public})"
     )