Moving evaluation inside generation class and enforcing empty generations when remove_thinking=True

docs/agentic_inference/parallel_thinking.md

@@ -57,6 +57,7 @@ ns eval \
   --server_gpus 2 \
   --server_type vllm \
   --output_dir /experiments/qwen3_8b/gensynthesis \
+  ++remove_thinking=True \
   ++inference.tokens_to_generate=16384 \
   ++parallel_thinking.mode=gensynthesis \
   ++server.enable_soft_fail=True \
@@ -91,6 +92,7 @@ eval(
     ctx=wrap_arguments(
         "++inference.tokens_to_generate=16384 "
         "++inference.temperature=0.6 "
+        "++remove_thinking=True "
     ),
     cluster="local",
     benchmarks="livecodebench:8",
@@ -110,6 +112,7 @@ eval(
         "++parallel_thinking.mode=genselect "
         "++parallel_thinking.solution_key=completion "
         "++parallel_thinking.generation_dir=/workspace/qwen3_4b_evals/eval-results/livecodebench "
+        "++remove_thinking=True "
     ),
     cluster="local",
     benchmarks="livecodebench:8",

docs/basics/index.md

@@ -236,7 +236,8 @@ run_cmd( # (1)!
 eval(
     ctx=wrap_arguments( # (2)!
         "++inference.tokens_to_generate=16000 "
-        "++inference.temperature=0.6"
+        "++inference.temperature=0.6 "
+        "++remove_thinking=True "
     ),
     cluster=cluster,
     model=f"{output_dir}/QwQ-32B",

docs/evaluation/code.md

@@ -221,17 +221,18 @@ ns eval \
     --split=test_v6_2408_2505 \
     --data_dir=<DATA_DIR> \
     --output_dir=<OUTPUT_DIR> \
-    --extra_eval_args="++eval_config.interpreter=python" \
+    ++remove_thinking=True \
+    ++eval_config.interpreter=python \
     ++inference.temperature=0.6 \
     ++inference.top_p=0.95 \
     ++inference.tokens_to_generate=65536
 ```
 
 ##### Pypy3 Evaluation
 
-To run with the Pypy3 interpreter, we need to use sandbox. Therefore, pass these flags `--with_sandbox --keep_mounts_for_sandbox` and modify the `--extra_eval_args` flag as shown below.
+To run with the Pypy3 interpreter, we need to use sandbox. Therefore, pass these flags `--with_sandbox --keep_mounts_for_sandbox` and also add the following arguments
 ```
---extra_eval_args="++eval_config.interpreter=pypy3 ++eval_config.test_file=<DATA_DIR>/livecodebench/test_v6_2408_2505.jsonl"
+++eval_config.interpreter=pypy3 ++eval_config.test_file=<DATA_DIR>/livecodebench/test_v6_2408_2505.jsonl
 ```
 
 ##### Verifying Results
@@ -331,6 +332,7 @@ ns eval \
     --split=test_python \
     --data_dir=<DATA_DIR> \
     --output_dir=<OUTPUT_DIR> \
+    ++remove_thinking=True \
     ++inference.temperature=0.6 \
     ++inference.top_p=0.95 \
     ++inference.tokens_to_generate=32768

docs/evaluation/index.md

@@ -50,6 +50,10 @@ ns prepare_data ruler --setup=llama_128k --tokenizer_path=meta-llama/Llama-3.1-8
 
 ## Running evaluation
 
+!!! warning
+    For correct evaluation of reasoning models, either provide reasoning parser in server args (e.g. `--server_args="--reasoning-parser ..."` for vllm)
+    or set `++remove_thinking=True` as well as an appropriate `++thinking_end` string (which defaults to `</think>`).
+
 ```bash
 ns eval \
     --cluster=local \
@@ -166,7 +170,7 @@ Different benchmarks have different evaluation options that you can customize. H
 code execution timeout for scicode benchmark
 
 ```bash
-    --extra_eval_args="++eval_config.timeout=60"
+    ++eval_config.timeout=60
 ```
 
 ## Using data on cluster
@@ -223,14 +227,11 @@ Inside [`nemo_skills/dataset/gsm8k/__init__.py`](https://github.com/NVIDIA-NeMo/
 # settings that define how evaluation should be done by default (all can be changed from cmdline)
 DATASET_GROUP = 'math'
 METRICS_TYPE = "math"
-EVAL_ARGS = "++eval_type=math"
-GENERATION_ARGS = "++prompt_config=generic/math"
+GENERATION_ARGS = "++eval_type=math ++prompt_config=generic/math"
 ```
 
 The prompt config and default generation arguments are passed to the
-[nemo_skills/inference/generate.py](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/inference/generate.py) and
-the default eval args are passed to the
-[nemo_skills/evaluation/evaluate_results.py](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/evaluation/evaluate_results.py).
+[nemo_skills/inference/generate.py](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/inference/generate.py).
 The dataset group is used by [nemo_skills/dataset/prepare.py](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/prepare.py)
 to help download only benchmarks from a particular group if `--dataset_groups` parameter is used.
 Finally, the metrics type is used to pick a metrics class from [nemo_skills/evaluation/metrics/map_metrics.py](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/evaluation/metrics/map_metrics.py)

docs/evaluation/multilingual.md

@@ -65,6 +65,7 @@ Some reference numbers for reference and commands for reproduction:
         --server_type=vllm \
         --num_chunks=32 \
         --server_gpus=2 \
+        ++remove_thinking=True \
         ++inference.temperature=0.6 \
         ++inference.top_k=20 \
         ++inference.tokens_to_generate=38912

docs/evaluation/natural-math.md

@@ -11,7 +11,7 @@ We also support arbitrary regex based extraction. E.g., if you use a custom prom
 at the end of the solution, you can use these parameters to match the extraction logic to that prompt
 
 ```bash
-    --extra_eval_args="++eval_config.extract_from_boxed=False ++eval_config.extract_regex='Final answer: (.+)$'"
+    ++eval_config.extract_from_boxed=False ++eval_config.extract_regex='Final answer: (.+)$'
 ```
 
 !!! warning

docs/evaluation/robustness.md

@@ -28,10 +28,12 @@ Note that every prompt is a separate job, and all parameters are shared for all
 ```python
 from nemo_skills.pipeline.cli import wrap_arguments, robust_eval
 robust_eval(ctx=wrap_arguments(
-        f"++inference.temperature=0.6 "
-        f"++inference.top_p=0.95 "
+        "++inference.temperature=0.6 "
+        "++inference.top_p=0.95 "
+        "++remove_thinking=True "
     ),
-    prompt_set_config='robustness/prompt_set_config', # OR nemo_skills/prompt/config/robutness/prompt_set_config OR absolute path to .yaml file
+    # OR nemo_skills/prompt/config/robutness/prompt_set_config OR absolute path to .yaml file
+    prompt_set_config='robustness/prompt_set_config',
     cluster=cluster_config,
     model="Qwen/Qwen3-8B",
     server_type='vllm',

docs/evaluation/tool-calling.md

@@ -62,6 +62,7 @@ ns eval \
   --server_gpus 2 \
   --server_type vllm \
   --output_dir /workspace/qwen3-4b-client-parsing/ \
+  ++remove_thinking=True \
   ++inference.tokens_to_generate=8192 \
   ++model_name=Qwen/Qwen3-4B-FC \
 ```
@@ -87,10 +88,11 @@ The following command evaluates the `Qwen3-4B` model which uses a standard tool-
 ns eval \
   --benchmarks bfcl_v3 \
   --cluster dfw \
-  --model /hf_models/Qwen3-4B \
+  --model Qwen/Qwen3-4B \
   --server_gpus 2 \
   --server_type vllm \
   --output_dir /workspace/qwen3-4b-server-parsing/ \
+  ++remove_thinking=True \
   ++inference.tokens_to_generate=8192 \
   ++use_client_parsing=False \
   --server_args="--enable-auto-tool-choice --tool-call-parser hermes"
@@ -110,6 +112,7 @@ ns eval \
     --server_gpus=2 \
     --server_type=vllm \
     --output_dir=/workspace/llama_nemotron_49b_1_5_tool_calling/ \
+    ++remove_thinking=True \
     ++inference.tokens_to_generate=65536 \
     ++inference.temperature=0.6 \
     ++inference.top_p=0.95 \

docs/pipelines/generation.md

@@ -122,8 +122,8 @@ ns generate \
        --server_nodes=2 \
        --num_random_seeds=32 \
        --output_dir=/workspace/synthetic-math-solutions \
-       --eval_args="++eval_type=math" \
        --input_file=/nemo_run/code/nemo_skills/dataset/math/train.jsonl \
+       ++eval_type=math \
        ++prompt_config=generic/math-base \
        ++examples_type=math_text_detailed \
        ++inference.endpoint_type=text \
@@ -139,7 +139,7 @@ and a tokenizer for the base model
 (we found Llama 3.1 follows few-shots much better without chat tokens).
 Finally, we are specifying few shot examples which come from
 [here](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/prompt/few_shot_examples/math.py)
-and asking the script to evaluate the generated solutions by providing `--eval_args`.
+and asking the script to evaluate the generated solutions by providing `++eval_type=math`.
 
 An example prompt (printed by the generate script) for that job is below.
 

docs/releases/opencodereasoning/evaluation.md

@@ -23,6 +23,7 @@ ns eval \
     --benchmarks=livecodebench:8 \
     --split=test_v6_2408_2505 \
     --server_gpus=1 \
+    ++remove_thinking=True \
     ++inference.tokens_to_generate=64000
 ```
 

docs/releases/openmathinstruct2/dataset.md

@@ -29,8 +29,8 @@ ns generate \
     --server_nodes=2 \
     --num_random_seeds=512 \
     --output_dir=/workspace/solution-augmentation/math \
-    --eval_args="++eval_type=math" \
     --input_file=/nemo_run/code/nemo_skills/dataset/math/train.jsonl \
+    ++eval_type=math \
     ++prompt_config=generic/math-base \
     ++examples_type=math_text_detailed \
     ++inference.endpoint_type=text \
@@ -49,8 +49,8 @@ ns generate \
     --server_nodes=2 \
     --num_random_seeds=64 \
     --output_dir=/workspace/solution-augmentation/gsm8k \
-    --eval_args="++eval_type=math" \
     --input_file=/nemo_run/code/nemo_skills/dataset/gsm8k/train.jsonl \
+    ++eval_type=math \
     ++prompt_config=generic/math-base \
     ++examples_type=gsm8k_text_detailed \
     ++inference.endpoint_type=text \

docs/releases/openmathreasoning/evaluation.md

@@ -36,6 +36,7 @@ ns eval \
     --server_gpus=1 \
     --output_dir=/workspace/openmath-nemotron-1.5b-eval-cot \
     --benchmarks=comp-math-24-25:64 \
+    ++remove_thinking=True \
     ++prompt_config=generic/math \
     ++inference.tokens_to_generate=32768 \
     ++inference.temperature=0.6
@@ -53,6 +54,7 @@ ns eval \
     --output_dir=/workspace/openmath-nemotron-1.5b-eval-cot \
     --benchmarks=hle:64 \
     --split=math \
+    ++remove_thinking=True \
     ++prompt_config=generic/math \
     ++inference.tokens_to_generate=32768 \
     ++inference.temperature=0.6 \
@@ -102,6 +104,7 @@ ns eval \
     --server_gpus=1 \
     --num_jobs=1 \
     --with_sandbox \
+    ++remove_thinking=True \
     ++code_tags=openmath \
     ++prompt_config=openmath/tir \
     ++inference.endpoint_type=text \
@@ -125,6 +128,7 @@ ns eval \
     --server_gpus=1 \
     --num_jobs=1 \
     --with_sandbox \
+    ++remove_thinking=True \
     ++code_tags=openmath \
     ++prompt_config=generic/math \
     ++inference.endpoint_type=text \

docs/tutorials/posts/llama-nemotron-super-v1.5-evals.md

@@ -103,10 +103,10 @@ ns eval \
     --output_dir=/workspace/llama_nemotron_49b_1_5/ \
     --benchmarks=scicode:16,math-500:16,aime24:16,aime25:16 \
     --server_gpus=2 \
+    ++remove_thinking=True \
     ++inference.tokens_to_generate=65536 \
     ++inference.temperature=0.6 \
-    ++inference.top_p=0.95 \
-    ++system_message=''
+    ++inference.top_p=0.95
 ```
 
 For GPQA and MMLU-Pro, we additionally specify the exact prompt on which we evaluate the benchmark:
@@ -118,11 +118,11 @@ ns eval \
     --output_dir=/workspace/llama_nemotron_49b_1_5/ \
     --benchmarks=mmlu-pro:16 \
     --server_gpus=2 \
+    ++remove_thinking=True \
     ++prompt_config=eval/aai/mcq-10choices-boxed \
     ++inference.tokens_to_generate=65536 \
     ++inference.temperature=0.6 \
-    ++inference.top_p=0.95 \
-    ++system_message=''
+    ++inference.top_p=0.95
 
 ns eval \
     --cluster=local \
@@ -131,11 +131,11 @@ ns eval \
     --output_dir=/workspace/llama_nemotron_49b_1_5/ \
     --benchmarks=gpqa:16 \
     --server_gpus=2 \
+    ++remove_thinking=True \
     ++prompt_config=eval/aai/mcq-4choices-boxed \
     ++inference.tokens_to_generate=65536 \
     ++inference.temperature=0.6 \
-    ++inference.top_p=0.95 \
-    ++system_message=''
+    ++inference.top_p=0.95
 ```
 
 For LiveCodeBench, we additionally specify the exact split on which we evaluate the benchmark. In the following command, we evaluate the model on the 166 problems from the 1 October 2024 to 1 March 2025 subset from release_v5. To evaluate on the Artificial Analysis Index (AAI) split, set split to `test_v5_2407_2412`:
@@ -149,10 +149,10 @@ ns eval \
     --benchmarks=livecodebench:16 \
     --split=test_v5_2410_2502 \
     --server_gpus=2 \
+    ++remove_thinking=True \
     ++inference.tokens_to_generate=65536 \
     ++inference.temperature=0.6 \
-    ++inference.top_p=0.95 \
-    ++system_message=''
+    ++inference.top_p=0.95
 ```
 
 #### Command for HLE Eval (Reasoning on)
@@ -173,10 +173,10 @@ ns eval \
     --server_gpus=2 \
     --judge_model="o3-mini-20250131" \
     --extra_judge_args="++inference.tokens_to_generate=4096 ++max_concurrent_requests=8" \
+    ++remove_thinking=True \
     ++inference.tokens_to_generate=65536 \
     ++inference.temperature=0.6 \
-    ++inference.top_p=0.95 \
-    ++system_message=''
+    ++inference.top_p=0.95
 ```
 
 !!! note
@@ -197,10 +197,10 @@ ns eval \
     --server_gpus=2 \
     --server_type=vllm \
     --output_dir=/workspace/llama_nemotron_49b_1_5_tool_calling/ \
+    ++remove_thinking=True \
     ++inference.tokens_to_generate=65536 \
     ++inference.temperature=0.6 \
     ++inference.top_p=0.95 \
-    ++system_message='' \
     ++use_client_parsing=False \
     --server_args="--tool-parser-plugin \"/workspace/Llama-3_3-Nemotron-Super-49B-v1_5/llama_nemotron_toolcall_parser_no_streaming.py\" \
                     --tool-call-parser \"llama_nemotron_json\" \
@@ -211,7 +211,8 @@ ns eval \
 
 For RULER we need to use the same `data_dir` in the evaluation command as we used in the data preparation. We also
 need to use the data preparation `setup` as part of the benchmark name. Finally it's important not to specify
-`++inference.tokens_to_generate` as RULER has a fixed value of this parameter for each task.
+`++inference.tokens_to_generate` as well as not specify `++remove_thinking=True` as
+RULER has predefined setup for those parameters.
 
 ```bash hl_lines="6-7"
 ns eval \
@@ -223,8 +224,7 @@ ns eval \
     --data_dir=/workspace/ns-data \
     --server_gpus=2 \
     ++inference.temperature=0.6 \
-    ++inference.top_p=0.95 \
-    ++system_message=''
+    ++inference.top_p=0.95
 ```
 
 ### Reasoning-on Results

docs/tutorials/posts/nemotron-nano-v2-evals.md

@@ -121,6 +121,7 @@ ns eval \
     --server_type=vllm \
     --server_gpus=1 \
     --server_args="--mamba_ssm_cache_dtype float32 " \
+    ++remove_thinking=True \
     ++inference.tokens_to_generate=32768 \
     ++inference.temperature=0.6 \
     ++inference.top_p=0.95 \
@@ -138,6 +139,7 @@ ns eval \
     --server_type=vllm \
     --server_gpus=1 \
     --server_args="--mamba_ssm_cache_dtype float32 " \
+    ++remove_thinking=True \
     ++inference.tokens_to_generate=32768 \
     ++inference.temperature=0.6 \
     ++inference.top_p=0.95 \
@@ -156,6 +158,7 @@ ns eval \
     --server_type=vllm \
     --server_gpus=1 \
     --server_args="--mamba_ssm_cache_dtype float32 " \
+    ++remove_thinking=True \
     ++inference.tokens_to_generate=32768 \
     ++inference.temperature=0.6 \
     ++inference.top_p=0.95 \
@@ -175,6 +178,7 @@ ns eval \
     --server_type=vllm \
     --server_gpus=1 \
     --server_args="--mamba_ssm_cache_dtype float32 " \
+    ++remove_thinking=True \
     ++inference.tokens_to_generate=32768 \
     ++inference.temperature=0.6 \
     ++inference.top_p=0.95 \
@@ -200,6 +204,7 @@ ns eval \
     --server_args="--mamba_ssm_cache_dtype float32 " \
     --judge_model="o3-mini-20250131" \
     --extra_judge_args="++inference.tokens_to_generate=4096 ++max_concurrent_requests=8" \
+    ++remove_thinking=True \
     ++inference.tokens_to_generate=32768 \
     ++inference.temperature=0.6 \
     ++inference.top_p=0.95 \
@@ -228,6 +233,7 @@ ns eval \
                    --tool-parser-plugin \"/workspace/NVIDIA-Nemotron-Nano-9B-v2/nemotron_toolcall_parser_no_streaming.py\" \
                    --tool-call-parser \"nemotron_json\" \
                    --enable-auto-tool-choice" \
+    ++remove_thinking=True \
     ++use_client_parsing=False \
     ++inference.tokens_to_generate=32768 \
     ++inference.temperature=0.6 \

docs/tutorials/posts/omr-simple-recipe.md

@@ -234,6 +234,7 @@ ns eval \
     --benchmarks=aime24:8,aime25:8 \
     --output_dir=/workspace/evals/after-training \
     --num_jobs=1 \
+    ++remove_thinking=True \
     ++inference.tokens_to_generate=16384
 # summarize results, after the evaluation job is done
 ns summarize_results --cluster=local /workspace/evals/after-training --wandb_name=after-training-evals