Better task list_with_num

Author: kcz358

Diff for: lmms_eval/__main__.py

@@ -244,14 +244,17 @@ def cli_evaluate_single(args: Union[argparse.Namespace, None] = None) -> None:
             "\n" + "=" * 70 + "\n" + "\n\tYou are trying to check all the numbers in each task." + "\n\tThis action will download the complete dataset." + "\n\tIf the results are not clear initially, call this again." + "\n\n" + "=" * 70
         )
         eval_logger.info(log_message)
-        task_dict = get_task_dict([task for task in sorted(ALL_TASKS)], model_name="llava")
-        for task_name in task_dict.keys():
-            task_obj = task_dict[task_name]
-            if type(task_obj) == tuple:
-                group, task_obj = task_obj
-                if task_obj is None:
-                    continue
-            eval_logger.info(f"\nTask : {task_obj.config.task}\n - #num : {len(task_obj.test_docs()) if task_obj.has_test_docs() else len(task_obj.validation_docs())}")
+        for task_name in sorted(ALL_TASKS):
+            try:
+                task_dict = get_task_dict([task_name], model_name="llava")
+                task_obj = task_dict[task_name]
+                if type(task_obj) == tuple:
+                    group, task_obj = task_obj
+                    if task_obj is None:
+                        continue
+                eval_logger.info(f"\nTask : {task_obj.config.task}\n - #num : {len(task_obj.test_docs()) if task_obj.has_test_docs() else len(task_obj.validation_docs())}")
+            except Exception as e:
+                eval_logger.debug(f"\nTask : {task_name} fail to load \n Exception : \n {e}")
         sys.exit()
     else:
         tasks_list = args.tasks.split(",")