Add multi-node sandbox support for SLURM clusters

dockerfiles/Dockerfile.sandbox

@@ -91,9 +91,9 @@ RUN mkdir -p /data && pip install gdown && \
 
 COPY nemo_skills/code_execution/local_sandbox/local_sandbox_server.py /app/main.py
 
-# Copy nginx configuration template and startup scripts
+# Copy nginx configuration templates
 COPY dockerfiles/sandbox/nginx.conf.template /etc/nginx/nginx.conf.template
-COPY dockerfiles/sandbox/start-with-nginx.sh /start-with-nginx.sh
+COPY dockerfiles/sandbox/nginx-worker-proxy.conf.template /etc/nginx/nginx-worker-proxy.conf.template
 
 # =============================================================================
 # Network Blocking for Sandboxed Code Execution (Defense in Depth)
@@ -127,7 +127,9 @@ RUN gcc -shared -fPIC -o /usr/lib/libblock_network.so /tmp/block_network.c -ldl
     rm /tmp/block_network.c && \
     echo "Built libblock_network.so for network isolation"
 
-# Make scripts executable
+# Copy startup script late in Dockerfile for better cache utilization
+# (start-with-nginx.sh changes more frequently than dependencies above)
+COPY dockerfiles/sandbox/start-with-nginx.sh /start-with-nginx.sh
 RUN chmod +x /start-with-nginx.sh
 
 # Set the working directory to /app
@@ -152,4 +154,4 @@ ENV UWSGI_PROCESSES=$UWSGI_PROCESSES
 ENV LISTEN_PORT=6000
 RUN echo "uwsgi_read_timeout 14400s;" > /etc/nginx/conf.d/custom_timeout.conf
 
-CMD /start-with-nginx.sh
+CMD ["/start-with-nginx.sh"]

dockerfiles/sandbox/nginx-worker-proxy.conf.template

@@ -0,0 +1,44 @@
+events {
+    worker_connections 1024;
+}
+
+http {
+    # Proxy all requests to the master node's nginx load balancer.
+    # Worker nodes don't route to individual workers — the master's
+    # consistent-hash upstream handles session affinity.
+    upstream master_lb {
+        server ${MASTER_NODE}:${NGINX_PORT};
+    }
+
+    server {
+        listen ${NGINX_PORT};
+        server_name localhost;
+
+        client_max_body_size 10M;
+        client_body_buffer_size 128k;
+
+        location / {
+            proxy_pass http://master_lb;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-Session-ID $http_x_session_id;
+            proxy_connect_timeout 1200s;
+            proxy_send_timeout 1200s;
+            proxy_read_timeout 1200s;
+            proxy_buffering off;
+        }
+
+        location /nginx-status {
+            stub_status on;
+            access_log off;
+            allow 127.0.0.1;
+            allow ::1;
+            deny all;
+        }
+    }
+
+    access_log /var/log/nginx/access.log;
+    error_log /var/log/nginx/error.log warn;
+}

dockerfiles/sandbox/nginx.conf.template

@@ -15,12 +15,16 @@ http {
         default   $http_x_session_id;
     }
 
-    # Define upstream servers (dynamically populated)
+    # Define upstream servers (dynamically populated by start-with-nginx.sh)
+    # Supports both single-node (localhost TCP) and multi-node (cross-node TCP) modes:
+    #   Single-node: server 127.0.0.1:50001 ...;
+    #   Multi-node:  server node1:50001 ...;  server node2:50001 ...;
     upstream sandbox_workers {
         # Use consistent hashing on real session_id or random request_id
+        # This ensures requests with the same X-Session-ID always go to the same worker
         hash $hash_key consistent;
 
-        # Worker servers will be inserted here by sed
+        # Worker servers will be inserted here (TCP endpoints)
 ${UPSTREAM_SERVERS}
     }