Benchmarks updates & Added Blink benchmark support

.gitignore

@@ -19,7 +19,6 @@ temp
 profile_default/
 ipython_config.py
 logs/
-scripts/
 wandb/
 SimSun.ttf
 submissions/
@@ -30,6 +29,7 @@ cache_dir
 ckpt
 pretrained/
 LLaVA/
+LLaVA-NeXT/
 *logs
 *.isorted
 temp/
@@ -41,6 +41,5 @@ Video-MME/
 VATEX/
 lmms_eval/tasks/vatex/__pycache__/utils.cpython-310.pyc
 lmms_eval/tasks/mlvu/__pycache__/utils.cpython-310.pyc
-
-scripts/
-.env
+.env
+lm-evaluation-harness/

lmms-eval.reqs

@@ -0,0 +1,185 @@
+accelerate                    1.2.0
+aiofiles                      23.2.1
+aiohappyeyeballs              2.4.4
+aiohttp                       3.11.10
+aiosignal                     1.3.1
+altair                        5.5.0
+annotated-types               0.7.0
+anyio                         4.7.0
+async-timeout                 5.0.1
+attrs                         24.2.0
+av                            14.0.1
+bitsandbytes                  0.45.0
+black                         24.1.0
+certifi                       2024.8.30
+cfgv                          3.4.0
+chardet                       5.2.0
+charset-normalizer            3.4.0
+click                         8.1.7
+colorama                      0.4.6
+contourpy                     1.3.1
+cycler                        0.12.1
+DataProperty                  1.0.1
+datasets                      2.20.0
+decord                        0.6.0
+dill                          0.3.7
+distlib                       0.3.9
+distro                        1.9.0
+docker-pycreds                0.4.0
+einops                        0.6.1
+einops-exts                   0.0.4
+et_xmlfile                    2.0.0
+evaluate                      0.4.3
+exceptiongroup                1.2.2
+fastapi                       0.115.6
+ffmpy                         0.4.0
+filelock                      3.16.1
+fonttools                     4.55.2
+frozenlist                    1.5.0
+fsspec                        2023.10.0
+ftfy                          6.3.1
+gitdb                         4.0.11
+GitPython                     3.1.43
+gradio                        4.16.0
+gradio_client                 0.8.1
+h11                           0.14.0
+hf_transfer                   0.1.8
+httpcore                      0.16.3
+httpx                         0.24.0
+huggingface-hub               0.26.5
+identify                      2.6.3
+idna                          3.10
+importlib_resources           6.4.5
+isort                         5.13.2
+Jinja2                        3.1.4
+jiter                         0.8.2
+joblib                        1.4.2
+jsonlines                     4.0.0
+jsonschema                    4.23.0
+jsonschema-specifications     2024.10.1
+kiwisolver                    1.4.7
+latex2mathml                  3.77.0
+llava                         1.2.2.post1 /lustre/fshomisc/home/rech/genrce01/ued79zb/repos/LLaVA
+lmms_eval                     0.3.0       /lustre/fshomisc/home/rech/genrce01/ued79zb/repos/lmms-eval
+loguru                        0.7.3
+lxml                          5.3.0
+markdown-it-py                3.0.0
+markdown2                     2.5.1
+MarkupSafe                    2.1.5
+matplotlib                    3.9.3
+mbstrdecoder                  1.1.3
+mdurl                         0.1.2
+mpmath                        1.3.0
+multidict                     6.1.0
+multiprocess                  0.70.15
+mypy-extensions               1.0.0
+narwhals                      1.17.0
+networkx                      3.4.2
+nltk                          3.9.1
+nodeenv                       1.9.1
+numexpr                       2.10.2
+numpy                         1.26.4
+nvidia-cublas-cu12            12.1.3.1
+nvidia-cuda-cupti-cu12        12.1.105
+nvidia-cuda-nvrtc-cu12        12.1.105
+nvidia-cuda-runtime-cu12      12.1.105
+nvidia-cudnn-cu12             8.9.2.26
+nvidia-cufft-cu12             11.0.2.54
+nvidia-curand-cu12            10.3.2.106
+nvidia-cusolver-cu12          11.4.5.107
+nvidia-cusparse-cu12          12.1.0.106
+nvidia-nccl-cu12              2.18.1
+nvidia-nvjitlink-cu12         12.4.127
+nvidia-nvtx-cu12              12.1.105
+openai                        1.57.2
+opencv-python-headless        4.10.0.84
+openpyxl                      3.1.5
+orjson                        3.10.12
+packaging                     24.2
+pandas                        2.2.3
+pathspec                      0.12.1
+pathvalidate                  3.2.1
+peft                          0.14.0
+pillow                        10.4.0
+pip                           24.2
+platformdirs                  4.3.6
+portalocker                   3.0.0
+pre_commit                    4.0.1
+propcache                     0.2.1
+protobuf                      3.20.0
+psutil                        6.1.0
+pyarrow                       18.1.0
+pyarrow-hotfix                0.6
+pybind11                      2.13.6
+pycocoevalcap                 1.2
+pycocotools                   2.0.8
+pydantic                      2.10.3
+pydantic_core                 2.27.1
+pydub                         0.25.1
+Pygments                      2.18.0
+pyparsing                     3.2.0
+pytablewriter                 1.2.0
+python-dateutil               2.9.0.post0
+python-multipart              0.0.19
+pytz                          2024.2
+PyYAML                        6.0.2
+referencing                   0.35.1
+regex                         2024.11.6
+requests                      2.32.3
+rfc3986                       1.5.0
+rich                          13.9.4
+rpds-py                       0.22.3
+ruff                          0.8.2
+sacrebleu                     2.4.3
+safetensors                   0.4.5
+scikit-learn                  1.2.2
+scipy                         1.14.1
+semantic-version              2.10.0
+sentence-transformers         3.3.1
+sentencepiece                 0.1.99
+sentry-sdk                    2.19.2
+setproctitle                  1.3.4
+setuptools                    75.1.0
+shellingham                   1.5.4
+shortuuid                     1.0.13
+six                           1.17.0
+smmap                         5.0.1
+sniffio                       1.3.1
+sqlitedict                    2.1.0
+starlette                     0.41.3
+svgwrite                      1.4.3
+sympy                         1.13.1
+tabledata                     1.3.3
+tabulate                      0.9.0
+tcolorpy                      0.1.6
+tenacity                      8.3.0
+threadpoolctl                 3.5.0
+tiktoken                      0.8.0
+timm                          0.6.13
+tokenizers                    0.21.0
+tomli                         2.2.1
+tomlkit                       0.12.0
+torch                         2.1.2
+torchvision                   0.16.2
+tqdm                          4.67.1
+tqdm-multiprocess             0.0.11
+transformers                  4.47.0
+transformers-stream-generator 0.0.5
+triton                        2.1.0
+typepy                        1.3.2
+typer                         0.15.1
+typing_extensions             4.12.2
+tzdata                        2024.2
+urllib3                       2.2.3
+uvicorn                       0.32.1
+virtualenv                    20.28.0
+wandb                         0.19.0
+wavedrom                      2.0.3.post3
+wcwidth                       0.2.13
+websockets                    11.0.3
+wheel                         0.44.0
+xxhash                        3.5.0
+yarl                          1.18.3
+yt-dlp                        2024.12.6
+zss                           1.2.0
+zstandard                     0.23.0

lmms_eval/evaluator.py

@@ -168,8 +168,9 @@ def simple_evaluate(
     if task_manager is None:
         task_manager = TaskManager(verbosity, model_name=model)
 
+    # FOR DEBUG PURPOSES -> COMMENT THIS
     task_dict = get_task_dict(tasks, task_manager)
-
+    
     if isinstance(model, str):
         if model_args is None:
             model_args = ""
@@ -590,7 +591,7 @@ def evaluate(
             num_fewshot,
             higher_is_better,
         ) = consolidate_results(eval_tasks)
-
+  
         ### Calculate group metrics ###
         if bool(results):
             results, versions, show_group_table, *_ = consolidate_group_results(results, versions, task_dict)

lmms_eval/models/__init__.py

@@ -28,6 +28,8 @@
     "llama_vid": "LLaMAVid",
     "llama_vision": "LlamaVision",
     "llava": "Llava",
+    "llava_next": "LlavaNext",
+    "llava_v6": "Llava_v6",
     "llava_hf": "LlavaHf",
     "llava_onevision": "Llava_OneVision",
     "llava_onevision_moviechat": "Llava_OneVision_MovieChat",
@@ -65,6 +67,14 @@
     "vllm": "VLLM",
     "xcomposer2_4KHD": "XComposer2_4KHD",
     "xcomposer2d5": "XComposer2D5",
+    "oryx": "Oryx",
+    "llama_vision": "LlamaVision",
+    "aria": "Aria",
+    "pangea": "Pangea",
+    "pixtral": "Pixtral",
+    "pixtral_v6": "Pixtral_v6",
+    "aya": "Aya",
+    "aya_v6": "Aya_v6",
     "egogpt": "EgoGPT",
     "internvideo2_5": "InternVideo2_5",
     "videochat_flash": "VideoChat_Flash",