fix(launch): node-local vLLM and FlashInfer caches by default

docs/scaling.md

@@ -181,6 +181,8 @@ The `rl`, `sft`, and `inference` entrypoints all submit to SLURM when a `[slurm]
 
 > **The prime-rl checkout and its `uv` venv must live on a shared filesystem** visible to every node. The generated sbatch script runs a single `uv sync --all-extras --all-packages` on the batch node (not once per node), so all ranks share that one environment — a node-local venv would leave the other nodes stale.
 
+Compile and JIT caches are node-local by default: the launcher sets `TRITON_CACHE_DIR=/tmp/triton-<uid>` for every component, and `VLLM_CACHE_ROOT=/tmp/.vllm-cache-<uid>` and `FLASHINFER_WORKSPACE_BASE=/tmp/.flashinfer-cache-<uid>` for inference. Ranks writing one cache on a shared filesystem can crash or hang. Override them in `[env_vars]` or a component's `env_vars`.
+
 ### Activation
 
 A SLURM config is usually a thin overlay that adds `[slurm]` (and `[deployment]` for multi-node) on top of a base config. Configs are composed left-to-right via the `@` CLI syntax — see [Configuration § TOML Composition](configuration.md#toml-composition):

examples/advanced/glm-4.5-air/search.toml

@@ -159,11 +159,6 @@ env.taskset.task.judge.model = "Qwen/Qwen3-235B-A22B-Instruct-2507"
 
 [inference]
 
-# vLLM + FlashInfer caches on node-local /tmp.
-[inference.env_vars]
-VLLM_CACHE_ROOT = "/tmp/.vllm-cache"
-FLASHINFER_WORKSPACE_BASE = "/tmp/.flashinfer-cache"
-
 [inference.vllm]
 max_model_len = 131072
 tensor_parallel_size = 8

examples/advanced/glm-4.5-air/swe-budget.toml

@@ -91,11 +91,6 @@ thinking_retention = "all"
 
 [inference]
 
-# vLLM + FlashInfer caches on node-local /tmp (see [env_vars]).
-[inference.env_vars]
-VLLM_CACHE_ROOT = "/tmp/.vllm-cache"
-FLASHINFER_WORKSPACE_BASE = "/tmp/.flashinfer-cache"
-
 [inference.vllm]
 max_model_len = 131072
 tensor_parallel_size = 8

examples/advanced/glm-4.5-air/swe.toml

@@ -108,11 +108,6 @@ env.agent.timeout.rollout = 3600
 
 [inference]
 
-# vLLM + FlashInfer caches on node-local /tmp (see [env_vars]).
-[inference.env_vars]
-VLLM_CACHE_ROOT = "/tmp/.vllm-cache"
-FLASHINFER_WORKSPACE_BASE = "/tmp/.flashinfer-cache"
-
 [inference.vllm]
 gpu_memory_utilization = 0.85
 max_model_len = 131072

examples/advanced/glm-4.5-air/terminal.toml

@@ -154,11 +154,6 @@ env.taskset.id = "terminal-bench-2"
 
 [inference]
 
-# vLLM + FlashInfer caches on node-local /tmp.
-[inference.env_vars]
-VLLM_CACHE_ROOT = "/tmp/.vllm-cache"
-FLASHINFER_WORKSPACE_BASE = "/tmp/.flashinfer-cache"
-
 [inference.vllm]
 max_model_len = 131072
 tensor_parallel_size = 8

src/prime_rl/utils/process.py

@@ -35,6 +35,10 @@
     "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:False",
     "VLLM_ENGINE_READY_TIMEOUT_S": "4200",
     "UCX_TLS": "all",
+    # vLLM's AOT compile cache and FlashInfer's JIT cache default to $HOME; a partial cache shared across
+    # nodes makes TP ranks load different compiled artifacts and hang.
+    "VLLM_CACHE_ROOT": f"/tmp/.vllm-cache-{os.getuid()}",
+    "FLASHINFER_WORKSPACE_BASE": f"/tmp/.flashinfer-cache-{os.getuid()}",
 }