From 98b09e2e2da37b15b230611d5b7431eb58c0f0e6 Mon Sep 17 00:00:00 2001 From: Albert Cui Date: Mon, 20 Jul 2026 19:48:50 -0400 Subject: [PATCH] fix(models): redirect LoRA sidecar state to a writable volume The adapters sidecar runs the nmp-api image via `nemo services run --sidecars adapters`, but the pod's securityContext runs it as the vLLM uid (2000), which does not own that image's baked $HOME (/home/nvs, uid 1000). `nemo services run` writes per-instance lock/descriptor state under $XDG_STATE_HOME (default ~/.local/state), so the sidecar hit PermissionError creating /home/nvs/.local and crash-looped to ERROR, blocking LoRA adapter inference. Point the sidecar's XDG_STATE_HOME at the writable scratch volume (/scratch/.nmp-state), outside the /scratch/loras subtree the adapters controller GCs. `nemo services run` already honors XDG_STATE_HOME, so no CLI change is needed. Re-homed onto the deployments_plugin cutover (#705), which removed the k8s_nim_operator backend the original fix patched; the redirect now lives in deployments_plugin/compiler.py::_lora_sidecar and is covered by that backend's compiler unit test. The XDG_STATE_HOME redirect itself was validated end-to-end on minikube earlier (a uid-2000 sidecar crash-loops without it and starts cleanly with it); this commit emits that same env from the new backend. Signed-off-by: Albert Cui --- .../controllers/backends/deployments_plugin/compiler.py | 7 +++++++ .../backends/deployments_plugin/test_compiler.py | 3 +++ 2 files changed, 10 insertions(+) diff --git a/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/compiler.py b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/compiler.py index 4ddecb15fc..d4e35b47d4 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/compiler.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/compiler.py @@ -62,6 +62,12 @@ _WEIGHTS_MOUNT = "/model-store" _SCRATCH_MOUNT = "/scratch" _LORA_MOUNT = "/scratch/loras" +# The adapters sidecar's `nemo services run` writes instance state under $XDG_STATE_HOME +# (default ~/.local/state), but the pod runs it as the vLLM uid (2000), which does not own +# the nmp-api image's $HOME (/home/nvs, uid 1000) -- so the default path is unwritable and +# the sidecar crash-loops. Redirect it to the writable scratch volume, outside the +# /scratch/loras subtree the adapters controller GCs. +_LORA_SIDECAR_STATE_DIR = f"{_SCRATCH_MOUNT}/.nmp-state" _SCRATCH_VOLUME_SIZE = "1Gi" @@ -145,6 +151,7 @@ def _lora_sidecar( "NMP_MODEL_ENTITY_WORKSPACE": entity_workspace, "NMP_MODEL_ENTITY_NAME": entity_name, "NMP_BASE_URL": platform.base_url, + "XDG_STATE_HOME": _LORA_SIDECAR_STATE_DIR, } if engine == ENGINE_VLLM: sidecar_env["VLLM_LORA_BASE_MODEL_OVERRIDE"] = MODEL_STORE_PATH diff --git a/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_compiler.py b/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_compiler.py index 8d4334379a..e7bd907ad1 100644 --- a/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_compiler.py +++ b/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_compiler.py @@ -300,6 +300,9 @@ def test_lora_uses_native_sidecar_on_k8s_and_container_on_docker() -> None: assert sidecar.image == "registry/nmp-api:tag" env = {item.name: item.value for item in sidecar.env} assert env["NIM_PEFT_SOURCE"] == "/scratch/loras" + # Sidecar `nemo services run` state must land on the writable scratch volume, not the + # nmp-api image's $HOME (unwritable under the pod's vLLM uid). See _lora_sidecar. + assert env["XDG_STATE_HOME"] == "/scratch/.nmp-state" assert env["VLLM_LORA_BASE_MODEL_OVERRIDE"] == "/model-store" assert env["NMP_BASE_URL"] == "http://platform.example:8080" assert env["VLLM_ENDPOINT"] == "http://127.0.0.1:8000"