Eval kit support

nemo_skills/dataset/eval_kit/__init__.py

@@ -0,0 +1,45 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# VLMEvalKit integration module.
+# Benchmarks are referenced as eval_kit.<VLMEvalKit_dataset_name>, e.g. eval_kit.MMBench_DEV_EN
+# The sub-benchmark name after eval_kit. is dynamically resolved and passed to VLMEvalKit.
+
+GENERATION_MODULE = "nemo_skills.inference.eval.eval_kit"
+METRICS_TYPE = "eval_kit"
+GENERATION_ARGS = ""
+NUM_SAMPLES = 0  # VLMEvalKit inference is deterministic; no random seeds
+
+# No JSONL input file; VLMEvalKit manages its own data via build_dataset()
+SKIP_INPUT_FILE = True
+
+# Note: SELF_CONTAINED_TASK is NOT set here because it depends on model_type.
+# For mcore mode (Megatron in-process), the pipeline sets self_contained_task=True
+# at runtime based on ++model_type=mcore in extra_arguments.
+# For vllm mode, the standard NeMo Skills server/client flow is used.
+
+
+def get_extra_generation_args(benchmark):
+    """Return extra generation args for the given benchmark name.
+
+    Extracts the VLMEvalKit dataset name from the dotted benchmark name
+    (e.g. eval_kit.MMBench_DEV_EN -> ++vlm_dataset=MMBench_DEV_EN).
+    """
+    if "." not in benchmark:
+        raise ValueError(
+            f"eval_kit benchmark must be in 'eval_kit.<dataset_name>' format, got '{benchmark}'. "
+            f"Example: eval_kit.MMBench_DEV_EN, eval_kit.LibriSpeech_test_clean"
+        )
+    sub = benchmark.split(".", 1)[1]
+    return f" ++vlm_dataset={sub} "

nemo_skills/dataset/utils.py

@@ -161,6 +161,13 @@ def _load_external_dataset(dataset_path):
 
 def get_default_dataset_module(dataset):
     data_path = "/nemo_run/code/nemo_skills/dataset"
+
+    # For dotted names like eval_kit.MMBench_DEV_EN, import the parent package.
+    # The sub-benchmark part is handled by the module's get_extra_generation_args().
+    if dataset.startswith("eval_kit."):
+        dataset_module = importlib.import_module("nemo_skills.dataset.eval_kit")
+        return dataset_module, data_path
+
     dataset_module = importlib.import_module(f"nemo_skills.dataset.{dataset}")
 
     return dataset_module, data_path

nemo_skills/evaluation/evaluator/audio.py

@@ -505,13 +505,35 @@ def evaluate_sample(sample: dict[str, Any], config: AudioEvaluatorConfig) -> dic
     """Evaluate single sample based on task_type. Returns dict of updates to merge."""
     updates = {}
     task_type = sample.get("task_type", "unknown")
-    generation = sample["generation"].strip()
+    generation_raw = sample.get("generation")
+    generation = generation_raw.strip() if isinstance(generation_raw, str) else ""
     expected_answer = sample.get("expected_answer", "").strip()
 
     # Strip helpful prefixes for ASR tasks (e.g., "The audio says: ...")
     if config.strip_helpful_prefixes:
         generation = strip_helpful_prefixes(generation)
 
+    # Normalise AudioBench speech-translation task types (ST-EN-ZH -> Translation)
+    _ASR_TYPES = {"ASR", "ASR-ZH", "ASR-PC", "ASR_LEADERBOARD"}
+    _TRANSLATION_TYPES = {"AST", "Translation"}
+    # AudioBench speech translation types: ST-{src}-{tgt}
+    if task_type.startswith("ST-"):
+        _TRANSLATION_TYPES.add(task_type)
+
+    if task_type in (_ASR_TYPES | _TRANSLATION_TYPES | {"CER"}) and not generation:
+        base = {
+            "is_correct": False,
+            "error": "missing_generation",
+        }
+        if task_type in _TRANSLATION_TYPES:
+            return {**base, "bleu": 0.0}
+        if task_type == "CER":
+            return {**base, "cer": 1.0}
+        if task_type == "ASR-PC":
+            return {**base, "wer": 1.0, "wer_c": 1.0, "wer_pc": 1.0, "per": 1.0}
+        # ASR / ASR-ZH / ASR_LEADERBOARD
+        return {**base, "wer": 1.0}
+
     if task_type == "ASR-PC":
         mode = resolve_asr_normalization_mode(config)
         metrics = evaluate_asr_pc(
@@ -522,7 +544,7 @@ def evaluate_sample(sample: dict[str, Any], config: AudioEvaluatorConfig) -> dic
         )
         updates.update(metrics)
 
-    elif task_type == "ASR":
+    elif task_type in {"ASR", "ASR-ZH"}:
         mode = resolve_asr_normalization_mode(config)
         metrics = evaluate_asr(expected_answer, generation, normalization_mode=mode)
         updates.update(metrics)
@@ -544,7 +566,7 @@ def evaluate_sample(sample: dict[str, Any], config: AudioEvaluatorConfig) -> dic
                 updates[f"wer_{metric_suffix}"] = ref_metrics["wer"]
                 updates[f"is_correct_{metric_suffix}"] = ref_metrics["is_correct"]
 
-    elif task_type in ["AST", "Translation"]:
+    elif task_type in _TRANSLATION_TYPES:
         metrics = evaluate_translation(expected_answer, generation)
         updates.update(metrics)
 
@@ -561,6 +583,13 @@ def evaluate_sample(sample: dict[str, Any], config: AudioEvaluatorConfig) -> dic
         metrics = evaluate_pc_rate(expected_answer, generation)
         updates.update(metrics)
 
+    elif task_type == "MathQA":
+        # AudioBench MathQA: exact string match after normalization
+        gen_norm = generation.strip().lower()
+        ref_norm = expected_answer.strip().lower()
+        updates["is_correct"] = gen_norm == ref_norm
+        updates["predicted_answer"] = generation
+
     else:
         if "requires_judge" not in sample:
             updates["requires_judge"] = True

nemo_skills/evaluation/metrics/eval_kit_metrics.py

@@ -0,0 +1,95 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from pathlib import Path
+
+from nemo_skills.evaluation.metrics.base import BaseMetrics
+
+
+class EvalKitMetrics(BaseMetrics):
+    """Metrics class for VLMEvalKit benchmarks.
+
+    VLMEvalKit computes its own aggregate metrics during evaluation.
+    This class reads pre-computed aggregates from eval_kit_metrics.json
+    (written by EvalKitGenerationTask) rather than computing per-sample metrics.
+    The per-sample JSONL is still read by ComputeMetrics for the update() loop,
+    but we only count entries here -- the real metrics come from the JSON file.
+
+    Note: ComputeMetrics only calls setup() on the "_all_" calculator.  When
+    the data contains ``subset_for_metrics``, additional per-subset calculator
+    instances are created but never receive a setup() call.  We use a
+    class-level ``_shared_metrics_file`` so that those subset instances can
+    still locate the eval_kit_metrics.json discovered by the "_all_" instance.
+    """
+
+    # Shared across all instances so subset calculators can find the file
+    # even though only the "_all_" calculator receives setup().
+    _shared_metrics_file: Path | None = None
+
+    def __init__(self, **kwargs):
+        super().__init__(compute_no_answer=False)
+        self.eval_kit_metrics_file = None
+
+    def setup(self, input_files):
+        """Find the eval_kit_metrics.json in the same directory as the input files."""
+        if input_files:
+            # input_files are like ['/path/to/eval-results/eval_kit.MMBench_DEV_EN/output.jsonl']
+            metrics_dir = Path(input_files[0]).parent
+            candidate = metrics_dir / "eval_kit_metrics.json"
+            if candidate.exists():
+                self.eval_kit_metrics_file = candidate
+                EvalKitMetrics._shared_metrics_file = candidate
+            else:
+                # Reset stale shared path so a previous run's file isn't reused.
+                EvalKitMetrics._shared_metrics_file = None
+
+    def update(self, predictions):
+        """Count entries but don't compute per-sample metrics."""
+        self.total += 1
+
+    def get_metrics(self):
+        """Return pre-computed VLMEvalKit aggregate metrics."""
+        metrics_dict = {}
+
+        # Load pre-computed metrics from VLMEvalKit.
+        # Fall back to the class-level shared file for subset calculators
+        # that never received a setup() call.
+        eval_kit_results = {}
+        effective_file = self.eval_kit_metrics_file or EvalKitMetrics._shared_metrics_file
+        if effective_file and effective_file.exists():
+            with open(effective_file, "rt", encoding="utf-8") as f:
+                eval_kit_results = json.load(f)
+
+        # Build the metrics in NeMo Skills format
+        agg_dict = {"num_entries": self.total}
+
+        # Flatten VLMEvalKit results into the metrics dict
+        for key, value in eval_kit_results.items():
+            if isinstance(value, dict):
+                # Nested results (e.g., per-category scores)
+                for sub_key, sub_value in value.items():
+                    if isinstance(sub_value, (int, float)):
+                        agg_dict[f"{key}_{sub_key}"] = sub_value
+            elif isinstance(value, (int, float)):
+                agg_dict[key] = value
+
+        metrics_dict["greedy"] = agg_dict
+        return metrics_dict
+
+    def metrics_to_print(self):
+        return None
+
+    def evaluations_to_print(self):
+        return ["greedy"]

nemo_skills/evaluation/metrics/map_metrics.py

@@ -30,6 +30,7 @@
     SweBenchMetrics,
 )
 from nemo_skills.evaluation.metrics.critpt_metrics import CritPtMetrics
+from nemo_skills.evaluation.metrics.eval_kit_metrics import EvalKitMetrics
 from nemo_skills.evaluation.metrics.gradingbench_metrics import GradingBenchMetrics
 from nemo_skills.evaluation.metrics.hleaa_metrics import HLEAAMetrics
 from nemo_skills.evaluation.metrics.icpc_metrics import ICPCMetrics
@@ -87,6 +88,7 @@
     "compute-eval": ComputeEvalMetrics,
     "gradingbench": GradingBenchMetrics,
     "critpt": CritPtMetrics,
+    "eval_kit": EvalKitMetrics,
     "specdec": SpecdecMetrics,
 }
 

nemo_skills/evaluation/metrics/translation_metrics.py

@@ -16,7 +16,6 @@
 from collections import defaultdict
 
 import numpy as np
-from sacrebleu import corpus_bleu
 
 from nemo_skills.evaluation.metrics.base import BaseMetrics, as_float
 
@@ -35,6 +34,8 @@ class TranslationMetrics(BaseMetrics):
     # TODO: add support for other translation metrics, such as MetricX
 
     def get_metrics(self):
+        from sacrebleu import corpus_bleu
+
         metrics_dict = {}
         for key in self.translation_dict:
             src_lang, tgt_lang = key.split("->")