Update IOI to support interleaving + small improvements

dockerfiles/sandbox/start-with-nginx.sh

@@ -12,12 +12,21 @@ echo "Workers: $NUM_WORKERS, Nginx port: $NGINX_PORT"
 # Override nginx config for multi-worker mode (single mode uses original config)
 echo "Configuring nginx for multi-worker load balancing..."
 
-# Force session affinity settings: 1 process per worker with minimal cheaper
-UWSGI_PROCESSES=1
-UWSGI_CHEAPER=1
-export UWSGI_PROCESSES
-export UWSGI_CHEAPER
-echo "Forced UWSGI settings for session affinity: PROCESSES=$UWSGI_PROCESSES, CHEAPER=$UWSGI_CHEAPER"
+
+# Allow callers to opt-out of single-process state-preserving mode where each worker is given one process
+: "${STATEFUL_SANDBOX:=1}"
+if [ "$STATEFUL_SANDBOX" -eq 1 ]; then
+    UWSGI_PROCESSES=1
+    UWSGI_CHEAPER=1
+else
+    # In stateless mode, honour caller-supplied values
+    : "${UWSGI_PROCESSES:=1}"
+    : "${UWSGI_CHEAPER:=1}"
+fi
+
+export UWSGI_PROCESSES UWSGI_CHEAPER
+
+echo "UWSGI settings: PROCESSES=$UWSGI_PROCESSES, CHEAPER=$UWSGI_CHEAPER"
 
 # Validate and fix uwsgi configuration
 if [ -z "$UWSGI_PROCESSES" ]; then

nemo_skills/code_execution/local_sandbox/start_local_sandbox.sh

@@ -22,5 +22,7 @@ docker build --tag=${SANDBOX_NAME} --build-arg="NUM_WORKERS=$((`nproc --all`))"
 echo "Multi-worker mode: Starting $((`nproc --all`)) workers with session affinity"
 docker run --network=host \
     --memory=${NEMO_SKILLS_SANDBOX_MEM_LIMIT:-"16g"} \
+    ${UWSGI_CPU_AFFINITY:+-e UWSGI_CPU_AFFINITY=${UWSGI_CPU_AFFINITY}} \
+    ${UWSGI_PROCESSES:+-e UWSGI_PROCESSES=${UWSGI_PROCESSES}} \
     --restart unless-stopped \
     --name=local-sandbox ${SANDBOX_NAME}

nemo_skills/dataset/ioi24/__init__.py

@@ -17,3 +17,12 @@
 DATASET_GROUP = "code"
 METRICS_TYPE = "ioi"
 EVAL_ARGS = "++eval_type=ioi"
+
+# environment variables required by this benchmark
+SANDBOX_ENV_VARS = [
+    "UWSGI_PROCESSES=1024",
+    "UWSGI_CPU_AFFINITY=8",
+    "UWSGI_CHEAPER=1023",
+    "NUM_WORKERS=1",
+    "STATEFUL_SANDBOX=0",
+]

nemo_skills/dataset/ioi24/prepare.py

@@ -42,14 +42,11 @@
             entries.append(
                 {
                     "id": x,
-                    "run": run_code,
-                    "compile": compile_code,
                     "name": item["name"],
                     "ioi_id": item["id"],
                     "subtask": item["subtask"],
                     "question": item["problem"],
-                    "score": item["score"],
-                    "grader_files": item["grader_files"],
+                    "subtask_score": item["score"],
                 }
             )
 
@@ -78,8 +75,11 @@
             tests[test_name] = test_cases[problem_id][test_name]
         final_structure[problem_id][subtask] = {
             "tests": tests,
-            "score": entry["score"],
+            "subtask_score": entry["score"],
             "score_precision": entry["score_precision"],
+            "run": run_code,
+            "compile": compile_code,
+            "grader_files": entry["grader_files"],
         }
 
     with open(os.path.join(data_dir, f"{args.split}_metadata.json"), "w") as f:

nemo_skills/dataset/ioi25/__init__.py

@@ -0,0 +1,32 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+todo: We are working on providing the data files that are necessary to run IOI25 evaluation.
+"""
+
+# settings that define how evaluation should be done by default (all can be changed from cmdline)
+GENERATION_ARGS = "++prompt_config=generic/default"
+DATASET_GROUP = "code"
+METRICS_TYPE = "ioi"
+EVAL_ARGS = "++eval_type=ioi"
+
+# environment variables required by this benchmark
+SANDBOX_ENV_VARS = [
+    "UWSGI_PROCESSES=1024",
+    "UWSGI_CPU_AFFINITY=8",
+    "UWSGI_CHEAPER=1023",
+    "NUM_WORKERS=1",
+    "STATEFUL_SANDBOX=0",
+]

nemo_skills/evaluation/evaluator/__init__.py

@@ -26,7 +26,7 @@
 )
 from nemo_skills.evaluation.evaluator.ifbench import eval_ifbench
 from nemo_skills.evaluation.evaluator.ifeval import eval_if
-from nemo_skills.evaluation.evaluator.ioi import eval_ioi
+from nemo_skills.evaluation.evaluator.ioi import IOIEvaluator
 from nemo_skills.evaluation.evaluator.livecodebench import eval_livecodebench
 from nemo_skills.evaluation.evaluator.math import (
     Lean4ProofEvaluator,
@@ -58,7 +58,6 @@ def dummy_eval(cfg):
     "livecodebench_pro": eval_livecodebench_pro,
     "scicode": eval_scicode,
     "mrcr": eval_mrcr,
-    "ioi": eval_ioi,
     "bigcodebench": eval_bigcodebench,
     "ojbench": eval_ojbench,
     "human_eval_infilling": eval_human_eval_infilling,
@@ -70,6 +69,7 @@ def dummy_eval(cfg):
     "lean4-proof": Lean4ProofEvaluator,
     "lean4-statement": Lean4StatementEvaluator,
     # Other evaluators can be added here as they're converted to classes
+    "ioi": IOIEvaluator,
 }
 
 # Validation: Ensure no overlap between class and function maps