EvolvingLMMs-Lab
diff --git a/Diff for: ‎docs/README.md
+2-1 b/Diff for: ‎docs/README.md
+2-1
diff --git a/Diff for: ‎lmms_eval/__main__.py
+6-23 b/Diff for: ‎lmms_eval/__main__.py
+6-23
diff --git a/Diff for: ‎lmms_eval/api/metrics.py
+1-4 b/Diff for: ‎lmms_eval/api/metrics.py
+1-4
diff --git a/Diff for: ‎lmms_eval/api/model.py
+2-2 b/Diff for: ‎lmms_eval/api/model.py
+2-2
diff --git a/Diff for: ‎lmms_eval/api/registry.py
+1-2 b/Diff for: ‎lmms_eval/api/registry.py
+1-2
diff --git a/Diff for: ‎lmms_eval/api/task.py
+2-2 b/Diff for: ‎lmms_eval/api/task.py
+2-2
diff --git a/Diff for: ‎lmms_eval/evaluator.py
+2-4 b/Diff for: ‎lmms_eval/evaluator.py
+2-4
diff --git a/Diff for: ‎lmms_eval/logging_utils.py
+2-5 b/Diff for: ‎lmms_eval/logging_utils.py
+2-5
diff --git a/Diff for: ‎lmms_eval/models/__init__.py
+8-6 b/Diff for: ‎lmms_eval/models/__init__.py
+8-6
diff --git a/Diff for: ‎lmms_eval/models/batch_gpt4.py
+3-4 b/Diff for: ‎lmms_eval/models/batch_gpt4.py
+3-4
diff --git a/Diff for: ‎lmms_eval/models/claude.py
+5-5 b/Diff for: ‎lmms_eval/models/claude.py
+5-5
diff --git a/Diff for: ‎lmms_eval/models/from_log.py
+1-2 b/Diff for: ‎lmms_eval/models/from_log.py
+1-2
diff --git a/Diff for: ‎lmms_eval/models/fuyu.py
+1-4 b/Diff for: ‎lmms_eval/models/fuyu.py
+1-4
diff --git a/Diff for: ‎lmms_eval/models/gemini_api.py
+2-2 b/Diff for: ‎lmms_eval/models/gemini_api.py
+2-2
diff --git a/Diff for: ‎lmms_eval/models/gpt4v.py
+2-2 b/Diff for: ‎lmms_eval/models/gpt4v.py
+2-2
diff --git a/Diff for: ‎lmms_eval/models/idefics2.py
+2-2 b/Diff for: ‎lmms_eval/models/idefics2.py
+2-2
diff --git a/Diff for: ‎lmms_eval/models/instructblip.py
+2-3 b/Diff for: ‎lmms_eval/models/instructblip.py
+2-3
diff --git a/Diff for: ‎lmms_eval/models/internvl.py
+1-2 b/Diff for: ‎lmms_eval/models/internvl.py
+1-2
diff --git a/Diff for: ‎lmms_eval/models/llama_vid.py
+1-2 b/Diff for: ‎lmms_eval/models/llama_vid.py
+1-2
@@ -8,4 +8,5 @@ Majority of this documentation is adapted from [lm-eval-harness](https://github.
 
 * To learn about the command line flags, see the [commands](commands.md)
 * To learn how to add a new moddel,  see the [Model Guide](model_guide.md).
-* For a crash course on adding new tasks to the library, see our [Task Guide](task_guide.md).
+* For a crash course on adding new tasks to the library, see our [Task Guide](task_guide.md).
+* If you need to upload your datasets into correct HF format with viewer supported, please refer to [tools](https://github.com/EvolvingLMMs-Lab/lmms-eval/tree/pufanyi/hf_dataset_docs/tools)
@@ -1,12 +1,10 @@
 import os
 import yaml
 import sys
-import copy
 import json
-import logging
+
 import traceback
 import argparse
-import torch
 import numpy as np
 import datetime
 
@@ -25,10 +23,7 @@
 from lmms_eval.tasks import initialize_tasks, include_path, get_task_dict
 from lmms_eval.api.registry import ALL_TASKS
 from lmms_eval.logging_utils import WandbLogger
-from lmms_eval.utils import PathFormatter
-
-
-eval_logger = logging.getLogger("lmms-eval")
+from loguru import logger as eval_logger
 
 
 def _handle_non_serializable(o):
@@ -166,9 +161,10 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
         print("└───────────────────────────────────────────────────────────────────────────────┘")
         sys.exit(1)
 
-    set_loggers(args)
-    eval_logger = logging.getLogger("lmms-eval")
-    eval_logger.setLevel(getattr(logging, f"{args.verbosity}"))
+    # reset logger
+    eval_logger.remove()
+    eval_logger.add(sys.stdout, colorize=True, level=args.verbosity)
+    eval_logger.add(sys.stderr, level=args.verbosity)
     eval_logger.info(f"Verbosity set to {args.verbosity}")
     os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
@@ -228,11 +224,6 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
 
 
 def cli_evaluate_single(args: Union[argparse.Namespace, None] = None) -> None:
-    eval_logger = logging.getLogger("lmms-eval")
-    eval_logger.setLevel(getattr(logging, f"{args.verbosity}"))
-    eval_logger.info(f"Verbosity set to {args.verbosity}")
-    os.environ["TOKENIZERS_PARALLELISM"] = "false"
-
     initialize_tasks(args.verbosity)
 
     if args.predict_only:
@@ -350,13 +341,5 @@ def print_results(args, results):
         print(evaluator.make_table(results, "groups"))
 
 
-def set_loggers(args):
-    eval_logger = logging.getLogger("lmms-eval")
-    ch = logging.StreamHandler()
-    formatter = PathFormatter("%(asctime)s [%(pathname)s:%(lineno)d] %(levelname)s %(message)s", "%m-%d %H:%M:%S", timezone=args.timezone)
-    ch.setFormatter(formatter)
-    eval_logger.addHandler(ch)
-
-
 if __name__ == "__main__":
     cli_evaluate()
@@ -9,10 +9,7 @@
 import torch
 
 from lmms_eval.api.registry import register_metric, register_aggregation
-
-import logging
-
-eval_logger = logging.getLogger("lmms-eval")
+from loguru import logger as eval_logger
 
 
 # Register Aggregations First
 
@@ -8,9 +8,9 @@
 from lmms_eval.api.instance import Instance
 from tqdm import tqdm
 from lmms_eval import utils
-import logging
 
-eval_logger = logging.getLogger("lmms-eval")
+
+from loguru import logger as eval_logger
 
 T = TypeVar("T", bound="lmms")
 
 
@@ -1,10 +1,9 @@
 from lmms_eval.api.model import lmms
 
 from typing import Callable, Dict
-import logging
 import evaluate as hf_evaluate
 
-eval_logger = logging.getLogger("lmms-eval")
+from loguru import logger as eval_logger
 
 MODEL_REGISTRY = {}
 
 
@@ -2,7 +2,7 @@
 import ast
 import itertools
 import json
-import logging
+
 import os
 import random
 import re
@@ -37,7 +37,7 @@
 )
 from lmms_eval.filters import build_filter_ensemble
 
-eval_logger = logging.getLogger("lmms-eval")
+from loguru import logger as eval_logger
 
 # HuggingfaceM4/NoCaps contains truncated image in test split
 # Include this inside code block to avoid error
 
@@ -7,7 +7,7 @@
 from tqdm import tqdm
 
 import torch
-import logging
+
 import numpy as np
 from datasets import Image, Sequence
 
@@ -17,8 +17,6 @@
 import lmms_eval.api.metrics
 import lmms_eval.api.registry
 
-import re
-
 from lmms_eval.utils import (
     positional_deprecated,
     run_task_tests,
@@ -28,7 +26,7 @@
     simple_parse_args_string,
 )
 
-eval_logger = logging.getLogger("lmms-eval")
+from loguru import logger as eval_logger
 
 
 @positional_deprecated
 
@@ -1,6 +1,6 @@
 # Code mostly from: https://github.com/EleutherAI/lm-evaluation-harness/pull/1339, credit to: https://github.com/ayulockin
 import copy
-import logging
+
 import re
 import os
 import json
@@ -9,13 +9,10 @@
 import numpy as np
 from datetime import datetime
 from typing import Any, Dict, List, Literal, Tuple, Union
-
 from packaging.version import Version
-
 from lmms_eval import utils
 import tenacity
-
-logger = logging.getLogger(__name__)
+from loguru import logger
 
 try:
     import wandb
 
@@ -1,7 +1,8 @@
-import os
-import hf_transfer
+from loguru import logger
+import sys
 
-os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
+logger.remove()
+logger.add(sys.stdout, level="WARNING")
 
 AVAILABLE_MODELS = {
     "llava": "Llava",
@@ -22,18 +23,19 @@
     "idefics2": "Idefics2",
     "internvl": "InternVLChat",
     "gemini_api": "GeminiAPI",
-    "gemini_model": "GeminiModel",
     "reka": "Reka",
-    "llava_onevision": "Llava_OneVision",
     "from_log": "FromLog",
     "mplug_owl_video": "mplug_Owl",
     "phi3v": "Phi3v",
     "tinyllava": "TinyLlava",
     "llava_onevision": "Llava_OneVision",
+    "llava_hf": "LlavaHf",
+    "longva": "LongVA",
 }
 
 for model_name, model_class in AVAILABLE_MODELS.items():
     try:
         exec(f"from .{model_name} import {model_class}")
-    except ImportError:
+    except ImportError as e:
+        # logger.warning(f"Failed to import {model_class} from {model_name}: {e}")
         pass
@@ -2,7 +2,7 @@
 from copy import deepcopy
 from io import BytesIO
 import base64
-import logging
+
 import os
 import time
 import json
@@ -20,14 +20,13 @@
 from lmms_eval.api.instance import Instance
 from lmms_eval.api.model import lmms
 from lmms_eval.api.registry import register_model
-from lmms_eval import utils
+from loguru import logger as eval_logger
 
 # Conditional imports
 try:
     from decord import VideoReader, cpu
 except ImportError:
-    eval_logger = logging.getLogger("lmms-eval")
-    eval_logger.info("Decord is not installed. Video input will not be supported.")
+    eval_logger.warning("Decord is not installed. Video input will not be supported.")
 
 # Constants and global configurations
 API_TYPE = os.getenv("API_TYPE", "openai")
 
@@ -5,28 +5,28 @@
 import json
 from typing import List, Tuple, Union
 from tqdm import tqdm
-import requests as url_requests
 import time
-import logging
 
 from lmms_eval.api.instance import Instance
 from lmms_eval.api.model import lmms
 from lmms_eval.api.registry import register_model
-from lmms_eval import utils
 
 from accelerate import Accelerator, DistributedType
 
 from PIL import Image
 
 NUM_SECONDS_TO_SLEEP = 5
-eval_logger = logging.getLogger("lmms-eval")
+
+from loguru import logger
+
+eval_logger = logger
 
 try:
     import anthropic
     from decord import VideoReader, cpu
     import numpy as np
 except Exception as e:
-    eval_logger.error(f"Error importing claude: {e}")
+    eval_logger.warning(f"Error importing claude: {e}")
 
 API_URL = os.getenv("ANTHROPIC_API_URL", "https://api.anthropic.com/v1/complete")
 API_KEY = os.getenv("ANTHROPIC_API_KEY", "YOUR_API_KEY")
 
@@ -1,4 +1,3 @@
-import logging
 import json
 import os
 import re
@@ -11,7 +10,7 @@
 from lmms_eval.api.instance import Instance
 from accelerate import Accelerator, DistributedType
 
-eval_logger = logging.getLogger("lmms-eval")
+from loguru import logger as eval_logger
 
 
 @register_model("from_log")
 
@@ -15,11 +15,8 @@
 from tqdm import tqdm
 from accelerate import Accelerator, DistributedType
 from accelerate.state import AcceleratorState
-import logging
 
-import logging
-
-eval_logger = logging.getLogger("lmms-eval")
+from loguru import logger as eval_logger
 
 
 @register_model("fuyu")
 
@@ -1,7 +1,7 @@
 import io
 import os
 import time
-import logging
+
 import json
 
 from PIL import Image
@@ -12,7 +12,7 @@
 from lmms_eval.api.instance import Instance
 from accelerate import Accelerator, DistributedType
 
-eval_logger = logging.getLogger("lmms-eval")
+from loguru import logger as eval_logger
 
 try:
     import google.generativeai as genai
 
@@ -7,7 +7,7 @@
 from tqdm import tqdm
 import requests as url_requests
 import time
-import logging
+
 
 from lmms_eval.api.instance import Instance
 from lmms_eval.api.model import lmms
@@ -26,7 +26,7 @@
 
 API_TYPE = os.getenv("API_TYPE", "openai")
 NUM_SECONDS_TO_SLEEP = 30
-eval_logger = logging.getLogger("lmms-eval")
+from loguru import logger as eval_logger
 
 if API_TYPE == "openai":
     API_URL = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions")
 
@@ -1,5 +1,5 @@
 import torch
-import logging
+
 from tqdm import tqdm
 from lmms_eval import utils
 from lmms_eval.api.instance import Instance
@@ -14,7 +14,7 @@
 
 warnings.filterwarnings("ignore")
 
-eval_logger = logging.getLogger("lmms-eval")
+from loguru import logger as eval_logger
 
 DEFAULT_IMAGE_TOKEN = "<image>"
 try:
 
@@ -1,5 +1,5 @@
 import torch
-import logging
+
 import copy
 from tqdm import tqdm
 from lmms_eval import utils
@@ -20,8 +20,7 @@
 
 warnings.filterwarnings("ignore")
 
-eval_logger = logging.getLogger("lmms-eval")
-transformers.logging.set_verbosity_error()
+from loguru import logger as eval_logger
 
 
 @register_model("instructblip")
 
@@ -1,4 +1,3 @@
-import logging
 import os
 from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
 from accelerate.state import AcceleratorState
@@ -26,7 +25,7 @@
 import sys
 
 sys.path.append(os.path.join(str(wd), "InternVL", "internvl_chat"))
-eval_logger = logging.getLogger("lmms-eval")
+from loguru import logger as eval_logger
 
 if not hasattr(eval_logger, "internvl_warning_logged"):
     eval_logger.internvl_warning_logged = False
 
@@ -1,4 +1,3 @@
-import logging
 import os
 from accelerate import Accelerator, DistributedType, InitProcessGroupKwargs
 from accelerate.state import AcceleratorState
@@ -22,7 +21,7 @@
 
 import subprocess
 
-eval_logger = logging.getLogger("lmms-eval")
+from loguru import logger as eval_logger
 
 try:
     from llamavid.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN