Added AAI-Omniscience Benchmark

docs/evaluation/other-benchmarks.md

@@ -11,3 +11,71 @@ More details are coming soon!
 
 - Benchmark is defined in [`nemo_skills/dataset/arena-hard/__init__.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/arena-hard/__init__.py)
 - Original benchmark source is [here](https://github.com/lmarena/arena-hard-auto).
+
+### AA-Omniscience
+
+This is a benchmark developed by AA to measure hallucinations in LLMs and penalize confidently-false answers.
+
+- Benchmark is defined in [`nemo_skills/dataset/omniscience/__init__.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/omniscience/__init__.py)
+- Original benchmark and leaderboard are defined [here](https://artificialanalysis.ai/evaluations/omniscience), and data is [here](https://huggingface.co/datasets/ArtificialAnalysis/AA-Omniscience-Public)
+
+#### Notes:
+- Note that this benchmark can be quite sensitive to temperature and other sampling parameters, so make sure your settings align well with downstream conditions.
+- Also note that there still may be some variance between the public set and the full dataset; however, this set may be used as a way to compare hallucination rates between different checkpoints/models.
+
+#### Configuration: gpt-oss-20b with default judge (gemini-2.5-flash-preview-09-2025)
+- Make sure to set `DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET=24576` in your environment variables and nemo-skills config to max judge reasoning when using reasoning_effort='high'.
+
+```python
+from nemo_skills.pipeline.cli import wrap_arguments, eval
+
+eval(
+    ctx=wrap_arguments(
+        f"++inference.temperature=0.6 "
+        f"++inference.top_p=1.0 "
+        f"++inference.top_k=-1 "
+        f"++inference.tokens_to_generate=131072 "
+        f"++inference.reasoning_effort='high' "
+    ),
+    cluster="slurm",
+    expname="aa-omniscience-eval",
+    model="openai/gpt-oss-20b",
+    server_gpus=8,
+    server_nodes=1,
+    server_type="vllm",
+    server_args="--async-scheduling",
+    benchmarks="omniscience",
+    output_dir="/workspace/experiments/aa-omniscience-eval",
+    data_dir="/workspace/data_dir",
+    extra_judge_args="++inference.reasoning_effort='high' ++inference.temperature=1.0 ++inference.top_p=0.95 ++inference.top_k=64 " # set max reasoning effort and default temp for judge
+)
+```
+
+#### Configuration: gpt-oss-20b with custom judge (gpt-oss-120b)
+```python
+from nemo_skills.pipeline.cli import wrap_arguments, eval
+
+eval(
+    ctx=wrap_arguments(
+        f"++inference.temperature=0.6 "
+        f"++inference.top_p=1.0 "
+        f"++inference.top_k=-1 "
+        f"++inference.tokens_to_generate=131072 "
+        f"++inference.reasoning_effort='high' "
+    ),
+    cluster="slurm",
+    expname="aa-omniscience-eval",
+    model="openai/gpt-oss-20b",
+    server_gpus=8,
+    server_nodes=1,
+    server_type="vllm",
+    server_args="--async-scheduling",
+    judge_model="openai/gpt-oss-120b",
+    judge_server_type="vllm",
+    judge_server_gpus=8,
+    judge_server_args="--async-scheduling  --reasoning-parser GptOss",
+    benchmarks="omniscience",
+    output_dir="/workspace/experiments/aa-omniscience-eval",
+    data_dir="/workspace/data_dir"
+)
+```

nemo_skills/dataset/omniscience/__init__.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2026, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# settings that define how evaluation should be done by default (all can be changed from cmdline)
+DATASET_GROUP = "math"
+METRICS_TYPE = "omniscience"
+GENERATION_ARGS = "++prompt_config=eval/aai/omni"
+EVAL_SPLIT = "text"
+
+JUDGE_PIPELINE_ARGS = {
+    "model": "gemini-2.5-flash-preview-09-2025",
+    "server_type": "gemini",
+    "server_address": "https://generativelanguage.googleapis.com",
+}
+JUDGE_ARGS = "++prompt_config=judge/aa-omni-judge ++generation_key=judgement ++add_generation_stats=False"

nemo_skills/dataset/omniscience/prepare.py

@@ -0,0 +1,80 @@
+# Copyright (c) 2026, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+from pathlib import Path
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+TOPIC_TO_SPLIT_MAP = {
+    "Humanities and Social Sciences": "humanities",
+    "Health": "health",
+    "Software Engineering": "swe",
+    "Science Engineering and Mathematics": "stem",
+    "Law": "law",
+    "Finance": "finance",
+}
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-s",
+        "--splits",
+        default=["text", "humanities", "health", "swe", "stem", "law", "finance"],
+        nargs="+",
+        choices=["text", "humanities", "health", "swe", "stem", "law", "finance"],
+    )
+    return parser.parse_args()
+
+
+def format_entry(entry) -> dict:
+    return {
+        "id": entry["question_id"],
+        "domain": entry["domain"],
+        "topic": entry["topic"],
+        "question": entry["question"],
+        "expected_answer": entry["answer"],
+    }
+
+
+def write_jsonl(data: list[dict], path: str):
+    with open(path, "w", encoding="utf-8") as f:
+        for d in data:
+            f.write(json.dumps(d) + "\n")
+
+
+if __name__ == "__main__":
+    args = parse_args()
+
+    dataset = load_dataset("ArtificialAnalysis/AA-Omniscience-Public", split="train")
+    jsonl_data = [format_entry(d) for d in dataset]
+    output_dir = Path(__file__).absolute().parent
+
+    split_set = set(args.splits)
+    splits = {
+        "text": dataset,
+        **{
+            TOPIC_TO_SPLIT_MAP.get(t, str(t).lower()): dataset.filter(lambda x: x["domain"] == t)
+            for t in dataset.unique("domain")
+        },
+    }
+    splits = {k: v for k, v in splits.items() if k in split_set}
+
+    for split, data in tqdm(splits.items(), total=len(splits)):
+        output_file = output_dir / f"{split}.jsonl"
+        formatted_data = [format_entry(entry) for entry in data]
+        write_jsonl(formatted_data, output_file)

nemo_skills/evaluation/metrics/map_metrics.py

@@ -38,6 +38,7 @@
 from nemo_skills.evaluation.metrics.math_metrics import MathMetrics
 from nemo_skills.evaluation.metrics.mmau_pro_metrics import MMAUProMetrics
 from nemo_skills.evaluation.metrics.mrcr_metrics import MRCRMetrics
+from nemo_skills.evaluation.metrics.omni_metrics import OmniMetrics
 from nemo_skills.evaluation.metrics.ruler_metrics import RulerMetrics
 from nemo_skills.evaluation.metrics.simpleqa_metrics import SimpleQAMetrics
 from nemo_skills.evaluation.metrics.translation_metrics import TranslationMetrics
@@ -72,6 +73,7 @@
     "mmau_pro_closed_form": MMAUProMetrics,
     "mmau_pro_open_ended": MMAUProMetrics,
     "mmau_pro_instruction_following": MMAUProMetrics,
+    "omniscience": OmniMetrics,
     "compute-eval": ComputeEvalMetrics,
 }
 

nemo_skills/evaluation/metrics/omni_metrics.py

@@ -0,0 +1,142 @@
+# Copyright (c) 2026, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import defaultdict
+
+from nemo_skills.evaluation.metrics.math_metrics import BaseMetrics, as_int, as_percentage
+
+
+class OmniMetrics(BaseMetrics):
+    def __init__(self, compute_no_answer: bool = True, answer_key: str = "generation"):
+        super().__init__(compute_no_answer=compute_no_answer)
+        self.answer_key = answer_key
+
+    # use same RM code as MathMetrics
+    def _compute_reward_at_k(self, predictions: list[dict]):
+        score_dicts = [self._get_score_dict(pred) for pred in predictions]
+
+        for k in range(1, len(predictions) + 1):
+            for score_method in score_dicts[0].keys():
+                # Get valid answers and their results for this field
+                valid_answers_and_results = [
+                    (elem[self.answer_key], correctness_dict[score_method], elem["reward_model_score"])
+                    for elem, correctness_dict in zip(predictions[:k], score_dicts[:k])
+                    if elem[self.answer_key] is not None
+                ]
+
+                # If no valid answers, it's incorrect
+                if not valid_answers_and_results:
+                    is_correct = False
+                else:
+                    is_correct_best = sorted(valid_answers_and_results, key=lambda x: x[2], reverse=True)[0][1]
+                    self.eval_dict[f"rm_best@{k}"][score_method] += is_correct_best
+
+                    answer_to_score_dict = defaultdict(float)
+                    answer_to_correctness_dict = {}
+                    for predicted_answer, is_correct, reward_score in valid_answers_and_results:
+                        answer_to_score_dict[predicted_answer] += reward_score
+                        answer_to_correctness_dict[predicted_answer] = is_correct
+
+                    top_cum_reward_answer = sorted(
+                        list(answer_to_score_dict.items()), key=lambda x: x[1], reverse=True
+                    )[0][0]
+                    is_correct_majority = answer_to_correctness_dict[top_cum_reward_answer]
+                    self.eval_dict[f"rm_majority@{k}"][score_method] += is_correct_majority
+
+            no_answer = all(elem[self.answer_key] is None for elem in predictions[:k])
+            self.eval_dict[f"rm_best@{k}"]["no_answer"] += no_answer
+            self.eval_dict[f"rm_majority@{k}"]["no_answer"] += no_answer
+
+    def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | float]:
+        correctness_dict = {}
+        if "judgement" in prediction:
+            judgement = prediction["judgement"].strip()
+            correctness_dict["judge_correct"] = int(judgement.lower() == "a")
+            correctness_dict["judge_incorrect"] = -int(
+                judgement.lower() == "b"
+            )  # negate incorrect so pass@k minimizes it
+            correctness_dict["judge_partially_correct"] = int(judgement.lower() == "c")
+            correctness_dict["judge_abstained"] = int(judgement.lower() == "d")
+
+            # keep counter of all entries where no answer is correct
+            correctness_dict["non_correct"] = -int(judgement.lower() in ["b", "c", "d"])
+        return correctness_dict
+
+    def get_metrics(self):
+        metrics = super().get_metrics()
+
+        for agg_method, agg_metric_dict in metrics.items():
+            correct, incorrect, part_correct, abstained = (
+                agg_metric_dict["judge_correct"],
+                -agg_metric_dict["judge_incorrect"],  # multiply negated judge_incorrect to get minimized incorrect pct
+                agg_metric_dict["judge_partially_correct"],
+                agg_metric_dict["judge_abstained"],
+            )
+            non_correct = -agg_metric_dict["non_correct"]
+
+            # convert pcts back to counts
+            if isinstance(correct, float):
+                correct *= self.total / 100
+            if isinstance(incorrect, float):
+                incorrect *= self.total / 100
+            if isinstance(part_correct, float):
+                part_correct *= self.total / 100
+            if isinstance(abstained, float):
+                abstained *= self.total / 100
+            if isinstance(non_correct, float):
+                non_correct *= self.total / 100
+
+            # compute omni index between max correct and min incorrect (for pass@k)
+            metrics[agg_method]["judge_omni_index"] = 100 * (correct - incorrect) / self.total if self.total > 0 else 0
+
+            # compute hallucination rate with min incorrect and min non_correct
+            metrics[agg_method]["judge_omni_hallucination"] = 100 * incorrect / non_correct if non_correct > 0 else 0
+        return metrics
+
+    def get_incorrect_sample(self, prediction: dict) -> dict:
+        copy_prediction = prediction.copy()
+        if "judgement" in prediction:
+            copy_prediction["judgement"] = "B"
+            copy_prediction["judge_correct"] = 0
+            copy_prediction["judge_incorrect"] = -1
+            copy_prediction["judge_partially_correct"] = 0
+            copy_prediction["judge_abstained"] = 0
+            copy_prediction["non_correct"] = -1
+        return copy_prediction
+
+    def update(self, predictions):
+        super().update(predictions)
+        self._compute_pass_at_k(predictions, None)
+        if "reward_model_score" in predictions[0]:
+            self._compute_reward_at_k(predictions=predictions)
+
+    # print the same evaluations/metrics as math but ignoring majority/rm since that doesn't really exist with omniscience
+    def evaluations_to_print(self):
+        return [
+            f"pass@1[avg-of-{self.max_k}]",
+            f"pass@{self.max_k}",
+        ]
+
+    def metrics_to_print(self):
+        metrics_to_print = {
+            "num_entries": as_int,
+            "avg_tokens": as_int,
+            "gen_seconds": as_int,
+            "judge_correct": as_percentage,
+            "judge_omni_index": as_percentage,
+            "judge_omni_hallucination": as_percentage,
+        }
+        if self.compute_no_answer:
+            metrics_to_print["no_answer"] = as_percentage
+        return metrics_to_print

nemo_skills/prompt/config/eval/aai/omni.yaml

@@ -0,0 +1,7 @@
+# https://huggingface.co/datasets/ArtificialAnalysis/AA-Omniscience-Public
+
+system: |-
+  You are answering questions about {domain}, and in particular {topic}. You will be given a question, answer with JUST the answer (no explanation). If you do not know the answer, or you need more context or tools to answer the question, be clear about this - it is better that you say this than get the wrong answer.
+
+user: |-
+  {question}