Update IOI to support interleaving + small improvements

dockerfiles/sandbox/start-with-nginx.sh

@@ -12,12 +12,18 @@ echo "Workers: $NUM_WORKERS, Nginx port: $NGINX_PORT"
 # Override nginx config for multi-worker mode (single mode uses original config)
 echo "Configuring nginx for multi-worker load balancing..."
 
-# Force session affinity settings: 1 process per worker with minimal cheaper
-UWSGI_PROCESSES=1
-UWSGI_CHEAPER=1
-export UWSGI_PROCESSES
-export UWSGI_CHEAPER
-echo "Forced UWSGI settings for session affinity: PROCESSES=$UWSGI_PROCESSES, CHEAPER=$UWSGI_CHEAPER"
+# Respect pre-set environment variables; fall back to sane defaults
+# If the caller exported UWSGI_PROCESSES / UWSGI_CHEAPER (or the helper
+# variables used by start_local_sandbox.sh), keep those.  Otherwise pick
+# defaults that work for most cases.
+
+# shell parameter expansion with default value
+: "${UWSGI_PROCESSES:=2}"   # default to 2 processes per worker if unset
+: "${UWSGI_CHEAPER:=1}"     # default cheaper value
+
+export UWSGI_PROCESSES UWSGI_CHEAPER
+
+echo "UWSGI settings: PROCESSES=$UWSGI_PROCESSES, CHEAPER=$UWSGI_CHEAPER"
 
 # Validate and fix uwsgi configuration
 if [ -z "$UWSGI_PROCESSES" ]; then

nemo_skills/code_execution/local_sandbox/start_local_sandbox.sh

@@ -22,5 +22,7 @@ docker build --tag=${SANDBOX_NAME} --build-arg="NUM_WORKERS=$((`nproc --all`))"
 echo "Multi-worker mode: Starting $((`nproc --all`)) workers with session affinity"
 docker run --network=host \
     --memory=${NEMO_SKILLS_SANDBOX_MEM_LIMIT:-"16g"} \
+    ${UWSGI_CPU_AFFINITY:+-e UWSGI_CPU_AFFINITY=${UWSGI_CPU_AFFINITY}} \
+    ${UWSGI_PROCESSES:+-e UWSGI_PROCESSES=${UWSGI_PROCESSES}} \
     --restart unless-stopped \
     --name=local-sandbox ${SANDBOX_NAME}

nemo_skills/dataset/ioi24/__init__.py

@@ -17,3 +17,6 @@
 DATASET_GROUP = "code"
 METRICS_TYPE = "ioi"
 EVAL_ARGS = "++eval_type=ioi"
+
+# environment variables required by this benchmark
+ENV_VARS = ["UWSGI_PROCESSES=1024", "UWSGI_CPU_AFFINITY=8", "UWSGI_CHEAPER=1023", "NUM_WORKERS=1"]

nemo_skills/dataset/ioi24/prepare.py

@@ -42,14 +42,11 @@
             entries.append(
                 {
                     "id": x,
-                    "run": run_code,
-                    "compile": compile_code,
                     "name": item["name"],
                     "ioi_id": item["id"],
                     "subtask": item["subtask"],
                     "question": item["problem"],
-                    "score": item["score"],
-                    "grader_files": item["grader_files"],
+                    "subtask_score": item["score"],
                 }
             )
 
@@ -78,8 +75,11 @@
             tests[test_name] = test_cases[problem_id][test_name]
         final_structure[problem_id][subtask] = {
             "tests": tests,
-            "score": entry["score"],
+            "subtask_score": entry["score"],
             "score_precision": entry["score_precision"],
+            "run": run_code,
+            "compile": compile_code,
+            "grader_files": entry["grader_files"],
         }
 
     with open(os.path.join(data_dir, f"{args.split}_metadata.json"), "w") as f:

nemo_skills/dataset/ioi25/__init__.py

@@ -0,0 +1,22 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# settings that define how evaluation should be done by default (all can be changed from cmdline)
+GENERATION_ARGS = "++prompt_config=generic/default"
+DATASET_GROUP = "code"
+METRICS_TYPE = "ioi"
+EVAL_ARGS = "++eval_type=ioi"
+
+# environment variables required by this benchmark
+ENV_VARS = ["UWSGI_PROCESSES=1024", "UWSGI_CPU_AFFINITY=8", "UWSGI_CHEAPER=1023", "NUM_WORKERS=1"]

nemo_skills/evaluation/evaluator/__init__.py

@@ -26,7 +26,7 @@
 )
 from nemo_skills.evaluation.evaluator.ifbench import eval_ifbench
 from nemo_skills.evaluation.evaluator.ifeval import eval_if
-from nemo_skills.evaluation.evaluator.ioi import eval_ioi
+from nemo_skills.evaluation.evaluator.ioi import IOIEvaluator
 from nemo_skills.evaluation.evaluator.livecodebench import eval_livecodebench
 from nemo_skills.evaluation.evaluator.math import (
     Lean4ProofEvaluator,
@@ -58,7 +58,6 @@ def dummy_eval(cfg):
     "livecodebench_pro": eval_livecodebench_pro,
     "scicode": eval_scicode,
     "mrcr": eval_mrcr,
-    "ioi": eval_ioi,
     "bigcodebench": eval_bigcodebench,
     "ojbench": eval_ojbench,
     "human_eval_infilling": eval_human_eval_infilling,
@@ -70,6 +69,7 @@ def dummy_eval(cfg):
     "lean4-proof": Lean4ProofEvaluator,
     "lean4-statement": Lean4StatementEvaluator,
     # Other evaluators can be added here as they're converted to classes
+    "ioi": IOIEvaluator,
 }
 
 # Validation: Ensure no overlap between class and function maps