Add results for KaLM-Team/KaLM_Embedding-X-0605 in MMTEB

.github/workflows/model-results-comparison.yaml

@@ -0,0 +1,75 @@
+name: Model Results Comparison
+
+on:
+  pull_request_target:
+    types: [opened, synchronize, edited]
+    paths:
+      - 'results/**/*.json'
+  workflow_dispatch:
+    inputs:
+      reference_models:
+        description: 'Space-separated list of reference models for comparison'
+        required: true
+        type: string
+        default: 'intfloat/multilingual-e5-large google/gemini-embedding-001'
+      pull_request_number:
+        description: 'The pull request number to comment on (required if triggered manually)'
+        required: false
+        type: string
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  compare-results:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        # IMPORTANT: For pull_request_target, check out the PR branch explicitly
+        ref: ${{ github.event.pull_request.head.sha }}
+        fetch-depth: 0
+
+    - name: Fetch origin main
+      run: git fetch origin main
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+
+    - name: Install dependencies
+      run: |
+        pip install git+https://github.com/embeddings-benchmark/mteb.git tabulate
+
+    - name: Generate model comparison
+      env:
+        REFERENCE_MODELS: ${{ github.event.inputs.reference_models || 'intfloat/multilingual-e5-large google/gemini-embedding-001' }}
+      run: |
+        python scripts/create_pr_results_comment.py --reference-models $REFERENCE_MODELS --output model-comparison.md
+
+    - name: Upload comparison report
+      uses: actions/upload-artifact@v4
+      with:
+        name: model-comparison
+        path: model-comparison.md
+
+    - name: Determine PR Number
+      id: pr_info
+      run: |
+        if [ "${{ github.event_name }}" == "pull_request_target" ]; then
+          echo "pr_number=${{ github.event.number }}" >> $GITHUB_OUTPUT
+        elif [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ -n "${{ github.event.inputs.pull_request_number }}" ]; then
+          echo "pr_number=${{ github.event.inputs.pull_request_number }}" >> $GITHUB_OUTPUT
+        else
+          echo "pr_number=" >> $GITHUB_OUTPUT
+        fi
+
+    - name: Post PR comment
+      if: steps.pr_info.outputs.pr_number != ''
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: gh pr comment ${{ steps.pr_info.outputs.pr_number }} --body-file model-comparison.md --create-if-none --edit-last

results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/R2MEDBioinformaticsRetrieval.json

@@ -0,0 +1,158 @@
+{
+  "dataset_revision": "6021fce366892cbfd7837fa85a4128ea93315e18",
+  "task_name": "R2MEDBioinformaticsRetrieval",
+  "mteb_version": "1.38.20",
+  "scores": {
+    "test": [
+      {
+        "ndcg_at_1": 0.33766,
+        "ndcg_at_3": 0.3315,
+        "ndcg_at_5": 0.38632,
+        "ndcg_at_10": 0.42491,
+        "ndcg_at_20": 0.45599,
+        "ndcg_at_100": 0.50643,
+        "ndcg_at_1000": 0.53475,
+        "map_at_1": 0.15813,
+        "map_at_3": 0.24478,
+        "map_at_5": 0.29691,
+        "map_at_10": 0.32723,
+        "map_at_20": 0.34036,
+        "map_at_100": 0.35319,
+        "map_at_1000": 0.35551,
+        "recall_at_1": 0.15813,
+        "recall_at_3": 0.31487,
+        "recall_at_5": 0.45441,
+        "recall_at_10": 0.56146,
+        "recall_at_20": 0.64712,
+        "recall_at_100": 0.8287,
+        "recall_at_1000": 0.99382,
+        "precision_at_1": 0.33766,
+        "precision_at_3": 0.22078,
+        "precision_at_5": 0.21039,
+        "precision_at_10": 0.14156,
+        "precision_at_20": 0.08636,
+        "precision_at_100": 0.02429,
+        "precision_at_1000": 0.00292,
+        "mrr_at_1": 0.337662,
+        "mrr_at_3": 0.422078,
+        "mrr_at_5": 0.448701,
+        "mrr_at_10": 0.459699,
+        "mrr_at_20": 0.464088,
+        "mrr_at_100": 0.467727,
+        "mrr_at_1000": 0.468137,
+        "nauc_ndcg_at_1_max": 0.107933,
+        "nauc_ndcg_at_1_std": -0.036251,
+        "nauc_ndcg_at_1_diff1": 0.20485,
+        "nauc_ndcg_at_3_max": 0.003896,
+        "nauc_ndcg_at_3_std": -0.159925,
+        "nauc_ndcg_at_3_diff1": 0.158787,
+        "nauc_ndcg_at_5_max": -0.024326,
+        "nauc_ndcg_at_5_std": -0.259057,
+        "nauc_ndcg_at_5_diff1": 0.087966,
+        "nauc_ndcg_at_10_max": 0.046541,
+        "nauc_ndcg_at_10_std": -0.216917,
+        "nauc_ndcg_at_10_diff1": 0.102002,
+        "nauc_ndcg_at_20_max": 0.08808,
+        "nauc_ndcg_at_20_std": -0.193759,
+        "nauc_ndcg_at_20_diff1": 0.123764,
+        "nauc_ndcg_at_100_max": 0.101127,
+        "nauc_ndcg_at_100_std": -0.140113,
+        "nauc_ndcg_at_100_diff1": 0.094295,
+        "nauc_ndcg_at_1000_max": 0.105675,
+        "nauc_ndcg_at_1000_std": -0.119119,
+        "nauc_ndcg_at_1000_diff1": 0.122725,
+        "nauc_map_at_1_max": -0.128017,
+        "nauc_map_at_1_std": -0.228048,
+        "nauc_map_at_1_diff1": 0.206813,
+        "nauc_map_at_3_max": -0.077104,
+        "nauc_map_at_3_std": -0.271212,
+        "nauc_map_at_3_diff1": 0.163482,
+        "nauc_map_at_5_max": -0.051938,
+        "nauc_map_at_5_std": -0.301827,
+        "nauc_map_at_5_diff1": 0.10682,
+        "nauc_map_at_10_max": 0.009518,
+        "nauc_map_at_10_std": -0.253407,
+        "nauc_map_at_10_diff1": 0.119989,
+        "nauc_map_at_20_max": 0.027802,
+        "nauc_map_at_20_std": -0.240282,
+        "nauc_map_at_20_diff1": 0.128026,
+        "nauc_map_at_100_max": 0.036107,
+        "nauc_map_at_100_std": -0.222729,
+        "nauc_map_at_100_diff1": 0.120636,
+        "nauc_map_at_1000_max": 0.035463,
+        "nauc_map_at_1000_std": -0.220357,
+        "nauc_map_at_1000_diff1": 0.122269,
+        "nauc_recall_at_1_max": -0.128017,
+        "nauc_recall_at_1_std": -0.228048,
+        "nauc_recall_at_1_diff1": 0.206813,
+        "nauc_recall_at_3_max": -0.092021,
+        "nauc_recall_at_3_std": -0.277492,
+        "nauc_recall_at_3_diff1": 0.137652,
+        "nauc_recall_at_5_max": -0.114084,
+        "nauc_recall_at_5_std": -0.409315,
+        "nauc_recall_at_5_diff1": -0.005854,
+        "nauc_recall_at_10_max": -0.005245,
+        "nauc_recall_at_10_std": -0.310963,
+        "nauc_recall_at_10_diff1": 0.003225,
+        "nauc_recall_at_20_max": 0.092056,
+        "nauc_recall_at_20_std": -0.296093,
+        "nauc_recall_at_20_diff1": 0.075963,
+        "nauc_recall_at_100_max": 0.087527,
+        "nauc_recall_at_100_std": -0.212826,
+        "nauc_recall_at_100_diff1": -0.104752,
+        "nauc_recall_at_1000_max": -0.431216,
+        "nauc_recall_at_1000_std": -1.730542,
+        "nauc_recall_at_1000_diff1": -0.013341,
+        "nauc_precision_at_1_max": 0.107933,
+        "nauc_precision_at_1_std": -0.036251,
+        "nauc_precision_at_1_diff1": 0.20485,
+        "nauc_precision_at_3_max": 0.169329,
+        "nauc_precision_at_3_std": -0.048836,
+        "nauc_precision_at_3_diff1": 0.120776,
+        "nauc_precision_at_5_max": 0.159786,
+        "nauc_precision_at_5_std": -0.050926,
+        "nauc_precision_at_5_diff1": -0.036858,
+        "nauc_precision_at_10_max": 0.274123,
+        "nauc_precision_at_10_std": 0.126855,
+        "nauc_precision_at_10_diff1": -0.026095,
+        "nauc_precision_at_20_max": 0.332965,
+        "nauc_precision_at_20_std": 0.222284,
+        "nauc_precision_at_20_diff1": -0.010906,
+        "nauc_precision_at_100_max": 0.306047,
+        "nauc_precision_at_100_std": 0.407784,
+        "nauc_precision_at_100_diff1": -0.111434,
+        "nauc_precision_at_1000_max": 0.310909,
+        "nauc_precision_at_1000_std": 0.502172,
+        "nauc_precision_at_1000_diff1": -0.022547,
+        "nauc_mrr_at_1_max": 0.107933,
+        "nauc_mrr_at_1_std": -0.036251,
+        "nauc_mrr_at_1_diff1": 0.20485,
+        "nauc_mrr_at_3_max": 0.133737,
+        "nauc_mrr_at_3_std": -0.020626,
+        "nauc_mrr_at_3_diff1": 0.198239,
+        "nauc_mrr_at_5_max": 0.118186,
+        "nauc_mrr_at_5_std": -0.052252,
+        "nauc_mrr_at_5_diff1": 0.164958,
+        "nauc_mrr_at_10_max": 0.123855,
+        "nauc_mrr_at_10_std": -0.036904,
+        "nauc_mrr_at_10_diff1": 0.161794,
+        "nauc_mrr_at_20_max": 0.132658,
+        "nauc_mrr_at_20_std": -0.031363,
+        "nauc_mrr_at_20_diff1": 0.165566,
+        "nauc_mrr_at_100_max": 0.129236,
+        "nauc_mrr_at_100_std": -0.033646,
+        "nauc_mrr_at_100_diff1": 0.166151,
+        "nauc_mrr_at_1000_max": 0.12938,
+        "nauc_mrr_at_1000_std": -0.03328,
+        "nauc_mrr_at_1000_diff1": 0.16674,
+        "main_score": 0.42491,
+        "hf_subset": "default",
+        "languages": [
+          "eng-Latn"
+        ]
+      }
+    ]
+  },
+  "evaluation_time": 76.0640599639842,
+  "kg_co2_emissions": null
+}

results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/R2MEDBiologyRetrieval.json

@@ -0,0 +1,158 @@
+{
+  "dataset_revision": "8b9fec2db9eda4b5742d03732213fbaee8169556",
+  "task_name": "R2MEDBiologyRetrieval",
+  "mteb_version": "1.38.20",
+  "scores": {
+    "test": [
+      {
+        "ndcg_at_1": 0.15534,
+        "ndcg_at_3": 0.15733,
+        "ndcg_at_5": 0.15068,
+        "ndcg_at_10": 0.17158,
+        "ndcg_at_20": 0.20041,
+        "ndcg_at_100": 0.26314,
+        "ndcg_at_1000": 0.32761,
+        "map_at_1": 0.0376,
+        "map_at_3": 0.08401,
+        "map_at_5": 0.09239,
+        "map_at_10": 0.11209,
+        "map_at_20": 0.12309,
+        "map_at_100": 0.13613,
+        "map_at_1000": 0.13986,
+        "recall_at_1": 0.0376,
+        "recall_at_3": 0.12183,
+        "recall_at_5": 0.1426,
+        "recall_at_10": 0.20792,
+        "recall_at_20": 0.28802,
+        "recall_at_100": 0.52119,
+        "recall_at_1000": 0.88396,
+        "precision_at_1": 0.15534,
+        "precision_at_3": 0.14239,
+        "precision_at_5": 0.1068,
+        "precision_at_10": 0.08155,
+        "precision_at_20": 0.05534,
+        "precision_at_100": 0.02,
+        "precision_at_1000": 0.00324,
+        "mrr_at_1": 0.135922,
+        "mrr_at_3": 0.203883,
+        "mrr_at_5": 0.212621,
+        "mrr_at_10": 0.224149,
+        "mrr_at_20": 0.230638,
+        "mrr_at_100": 0.23588,
+        "mrr_at_1000": 0.236849,
+        "nauc_ndcg_at_1_max": 0.043162,
+        "nauc_ndcg_at_1_std": -0.197054,
+        "nauc_ndcg_at_1_diff1": 0.033,
+        "nauc_ndcg_at_3_max": 0.036832,
+        "nauc_ndcg_at_3_std": -0.217969,
+        "nauc_ndcg_at_3_diff1": 0.042707,
+        "nauc_ndcg_at_5_max": 0.083252,
+        "nauc_ndcg_at_5_std": -0.209761,
+        "nauc_ndcg_at_5_diff1": 0.076015,
+        "nauc_ndcg_at_10_max": 0.109022,
+        "nauc_ndcg_at_10_std": -0.179232,
+        "nauc_ndcg_at_10_diff1": 0.090936,
+        "nauc_ndcg_at_20_max": 0.055902,
+        "nauc_ndcg_at_20_std": -0.219545,
+        "nauc_ndcg_at_20_diff1": 0.056387,
+        "nauc_ndcg_at_100_max": 0.007695,
+        "nauc_ndcg_at_100_std": -0.243414,
+        "nauc_ndcg_at_100_diff1": 0.027337,
+        "nauc_ndcg_at_1000_max": 0.04756,
+        "nauc_ndcg_at_1000_std": -0.179507,
+        "nauc_ndcg_at_1000_diff1": 0.022172,
+        "nauc_map_at_1_max": 0.152888,
+        "nauc_map_at_1_std": -0.206546,
+        "nauc_map_at_1_diff1": 0.067181,
+        "nauc_map_at_3_max": 0.133551,
+        "nauc_map_at_3_std": -0.234395,
+        "nauc_map_at_3_diff1": 0.074463,
+        "nauc_map_at_5_max": 0.146198,
+        "nauc_map_at_5_std": -0.219463,
+        "nauc_map_at_5_diff1": 0.090968,
+        "nauc_map_at_10_max": 0.127698,
+        "nauc_map_at_10_std": -0.211415,
+        "nauc_map_at_10_diff1": 0.105176,
+        "nauc_map_at_20_max": 0.091347,
+        "nauc_map_at_20_std": -0.227329,
+        "nauc_map_at_20_diff1": 0.070797,
+        "nauc_map_at_100_max": 0.07235,
+        "nauc_map_at_100_std": -0.228508,
+        "nauc_map_at_100_diff1": 0.059918,
+        "nauc_map_at_1000_max": 0.073293,
+        "nauc_map_at_1000_std": -0.224687,
+        "nauc_map_at_1000_diff1": 0.058591,
+        "nauc_recall_at_1_max": 0.152888,
+        "nauc_recall_at_1_std": -0.206546,
+        "nauc_recall_at_1_diff1": 0.067181,
+        "nauc_recall_at_3_max": 0.158256,
+        "nauc_recall_at_3_std": -0.198852,
+        "nauc_recall_at_3_diff1": 0.135204,
+        "nauc_recall_at_5_max": 0.205718,
+        "nauc_recall_at_5_std": -0.170234,
+        "nauc_recall_at_5_diff1": 0.168633,
+        "nauc_recall_at_10_max": 0.175269,
+        "nauc_recall_at_10_std": -0.125935,
+        "nauc_recall_at_10_diff1": 0.14925,
+        "nauc_recall_at_20_max": 0.045362,
+        "nauc_recall_at_20_std": -0.209459,
+        "nauc_recall_at_20_diff1": 0.063763,
+        "nauc_recall_at_100_max": -0.093978,
+        "nauc_recall_at_100_std": -0.289285,
+        "nauc_recall_at_100_diff1": -0.012271,
+        "nauc_recall_at_1000_max": 0.205614,
+        "nauc_recall_at_1000_std": 0.315539,
+        "nauc_recall_at_1000_diff1": -0.052651,
+        "nauc_precision_at_1_max": 0.043162,
+        "nauc_precision_at_1_std": -0.197054,
+        "nauc_precision_at_1_diff1": 0.033,
+        "nauc_precision_at_3_max": -0.034286,
+        "nauc_precision_at_3_std": -0.231688,
+        "nauc_precision_at_3_diff1": 0.004616,
+        "nauc_precision_at_5_max": -0.012642,
+        "nauc_precision_at_5_std": -0.198358,
+        "nauc_precision_at_5_diff1": 0.016286,
+        "nauc_precision_at_10_max": -0.036443,
+        "nauc_precision_at_10_std": -0.174946,
+        "nauc_precision_at_10_diff1": 0.034214,
+        "nauc_precision_at_20_max": -0.125512,
+        "nauc_precision_at_20_std": -0.214498,
+        "nauc_precision_at_20_diff1": -0.051533,
+        "nauc_precision_at_100_max": -0.177078,
+        "nauc_precision_at_100_std": -0.18669,
+        "nauc_precision_at_100_diff1": -0.090969,
+        "nauc_precision_at_1000_max": -0.126668,
+        "nauc_precision_at_1000_std": -0.038646,
+        "nauc_precision_at_1000_diff1": -0.124093,
+        "nauc_mrr_at_1_max": 0.047079,
+        "nauc_mrr_at_1_std": -0.179281,
+        "nauc_mrr_at_1_diff1": 0.127084,
+        "nauc_mrr_at_3_max": 0.041883,
+        "nauc_mrr_at_3_std": -0.214568,
+        "nauc_mrr_at_3_diff1": 0.083644,
+        "nauc_mrr_at_5_max": 0.058587,
+        "nauc_mrr_at_5_std": -0.204153,
+        "nauc_mrr_at_5_diff1": 0.077505,
+        "nauc_mrr_at_10_max": 0.055233,
+        "nauc_mrr_at_10_std": -0.185858,
+        "nauc_mrr_at_10_diff1": 0.065808,
+        "nauc_mrr_at_20_max": 0.053161,
+        "nauc_mrr_at_20_std": -0.19455,
+        "nauc_mrr_at_20_diff1": 0.076221,
+        "nauc_mrr_at_100_max": 0.04739,
+        "nauc_mrr_at_100_std": -0.200545,
+        "nauc_mrr_at_100_diff1": 0.074901,
+        "nauc_mrr_at_1000_max": 0.047993,
+        "nauc_mrr_at_1000_std": -0.199493,
+        "nauc_mrr_at_1000_diff1": 0.074695,
+        "main_score": 0.17158,
+        "hf_subset": "default",
+        "languages": [
+          "eng-Latn"
+        ]
+      }
+    ]
+  },
+  "evaluation_time": 55.14279148916425,
+  "kg_co2_emissions": null
+}