fix(hermes): keep lazy installer probe offline

.github/workflows/managed-images.yaml

@@ -383,6 +383,7 @@ jobs:
           done
 
       - name: Set up Docker Buildx
+        id: buildx
         uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
 
       - name: Set up Node.js
@@ -418,13 +419,34 @@ jobs:
           fi
           if [ "$diff_status" -eq 1 ]; then
             echo "::notice::${DISPLAY_NAME} base Dockerfile changed; building the exact PR base locally"
+            local_base_archive="$RUNNER_TEMP/pr-base.docker.tar"
+            local_base_oci_archive="$RUNNER_TEMP/pr-base.oci.tar"
+            local_base_oci="$RUNNER_TEMP/pr-base.oci"
             docker buildx build \
               --platform linux/amd64 \
-              --load \
+              --provenance=false \
+              --sbom=false \
               --file "$BASE_DOCKERFILE" \
               --tag "$LOCAL_BASE_REFERENCE" \
+              --output "type=docker,dest=${local_base_archive}" \
+              --output "type=oci,dest=${local_base_oci_archive}" \
               .
+            docker load --input "$local_base_archive"
+            mkdir -p "$local_base_oci"
+            tar -C "$local_base_oci" -xf "$local_base_oci_archive"
+            local_base_oci_digest="$(
+              jq -er '
+                .manifests
+                | if length == 1 then .[0].digest else error("not one image") end
+              ' "$local_base_oci/index.json"
+            )"
+            if [[ ! "$local_base_oci_digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then
+              echo "ERROR: local PR base OCI layout returned an invalid digest." >&2
+              exit 1
+            fi
             printf 'ref=%s\n' "$LOCAL_BASE_REFERENCE" >> "$GITHUB_OUTPUT"
+            printf 'local=true\n' >> "$GITHUB_OUTPUT"
+            printf 'oci=%s@%s\n' "$local_base_oci" "$local_base_oci_digest" >> "$GITHUB_OUTPUT"
             printf '### %s PR base\n\nLocally built from `%s` at `%s`.\n' \
               "$DISPLAY_NAME" "$BASE_DOCKERFILE" "$CANDIDATE_SHA" \
               >> "$GITHUB_STEP_SUMMARY"
@@ -467,6 +489,7 @@ jobs:
             exit 1
           fi
           printf 'ref=%s\n' "$reference" >> "$GITHUB_OUTPUT"
+          printf 'local=false\n' >> "$GITHUB_OUTPUT"
           printf '### %s PR base\n\n`%s`\n' "$DISPLAY_NAME" "$reference" \
             >> "$GITHUB_STEP_SUMMARY"
 
@@ -483,7 +506,38 @@ jobs:
             --build-arg "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1" \
             --build-arg "NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER=root"
 
-      - name: Build PR managed image locally
+      - name: Build PR managed image from local base
+        if: steps.base.outputs.local == 'true'
+        shell: bash
+        env:
+          BASE_IMAGE: ${{ steps.base.outputs.ref }}
+          CANDIDATE_SHA: ${{ github.event.pull_request.head.sha }}
+          DOCKERFILE: ${{ matrix.dockerfile }}
+          IMAGE_REFERENCE: ${{ matrix.image }}:${{ github.event.pull_request.head.sha }}
+        run: |
+          set -euo pipefail
+          # The base resolver loads a changed base into Docker's local image
+          # store. Keep this consumer on that same store: a Buildx container
+          # builder otherwise treats the local-only reference as Docker Hub.
+          docker build \
+            --platform linux/amd64 \
+            --file "$DOCKERFILE" \
+            --tag "$IMAGE_REFERENCE" \
+            --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \
+            --label "org.opencontainers.image.revision=${CANDIDATE_SHA}" \
+            --label "io.nvidia.nemoclaw.agent=${{ matrix.agent }}" \
+            --label "io.nvidia.nemoclaw.managed-image.contract=1" \
+            --label "io.nvidia.nemoclaw.managed-image.platform=linux/amd64" \
+            --label "io.nvidia.nemoclaw.managed-image.startup-profile=1" \
+            --label "io.nvidia.nemoclaw.managed-image.capabilities=1" \
+            --label "io.nvidia.nemoclaw.managed-image.cohort=ghrun-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \
+            --build-arg "BASE_IMAGE=${BASE_IMAGE}" \
+            --build-arg "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1" \
+            --build-arg "NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER=root" \
+            .
+
+      - name: Build PR managed image from registry base
+        if: steps.base.outputs.local != 'true'
         uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
         with:
           context: .
@@ -670,9 +724,11 @@ jobs:
         if: github.event.pull_request.head.repo.full_name == github.repository
         uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
         with:
+          builder: ${{ steps.buildx.outputs.name }}
           context: .
           file: ${{ matrix.dockerfile }}
           platforms: linux/amd64
+          build-contexts: ${{ steps.base.outputs.local == 'true' && format('nemoclaw-pr-base=oci-layout://{0}', steps.base.outputs.oci) || '' }}
           outputs: type=image,name=${{ matrix.repository }},push-by-digest=true,name-canonical=true,push=true
           labels: |
             org.opencontainers.image.source=https://github.com/${{ github.repository }}
@@ -684,7 +740,7 @@ jobs:
             io.nvidia.nemoclaw.managed-image.capabilities=1
             io.nvidia.nemoclaw.managed-image.cohort=ghrun-${{ github.run_id }}-${{ github.run_attempt }}
           build-args: |
-            BASE_IMAGE=${{ steps.base.outputs.ref }}
+            BASE_IMAGE=${{ steps.base.outputs.local == 'true' && 'nemoclaw-pr-base' || steps.base.outputs.ref }}
             NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1
             NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER=root
           cache-from: type=registry,ref=ghcr.io/nvidia/nemoclaw/${{ matrix.agent }}-sandbox:buildcache-linux-amd64

agents/hermes/Dockerfile

@@ -1153,9 +1153,13 @@ RUN set -eu; \
 # uv creates the Hermes virtual environment without pip. Seed pip from Python's
 # bundled ensurepip payload while this layer is still root, then verify the
 # sandbox can read but cannot modify the virtual environment used by the lazy
-# installer. The Hermes lazy installer still installs the approved dependency
-# as the sandbox user.
-RUN /opt/hermes/.venv/bin/python -I -m ensurepip --upgrade --default-pip \
+# installer. Dockerfile.base separately installs and imports the official
+# Hindsight client. This layer uses a build-only wheel with the same package
+# identity to isolate the lazy-installer and durable-target contract from PyPI.
+# The nested write-denial shell returns success after every prohibited mutation
+# fails. Create the wheel before resetting the target so the sandbox installer
+# is the next Python process to use it.
+RUN --network=none /opt/hermes/.venv/bin/python -I -m ensurepip --upgrade --default-pip \
     && /opt/hermes/.venv/bin/python -I -m pip --version \
     && chmod 644 /opt/hermes/.venv/.lock \
     && test "$(stat -c '%U:%G %a' /opt/hermes/.venv/.lock)" = "root:root 644" \
@@ -1192,28 +1196,43 @@ RUN /opt/hermes/.venv/bin/python -I -m ensurepip --upgrade --default-pip \
             if printf "" >> /opt/hermes/.venv/lib/python3.13/site-packages/pip/__init__.py 2>/dev/null; then exit 1; fi; \
             if printf "" >> /opt/hermes/.venv/bin/python 2>/dev/null; then exit 1; fi; \
             if printf "" > /opt/hermes/.venv/lib64/.nemoclaw-sandbox-write-probe 2>/dev/null; then exit 1; fi; \
-            if ln -sf /usr/bin/false /opt/hermes/.venv/bin/python 2>/dev/null; then exit 1; fi' \
+            if ln -sf /usr/bin/false /opt/hermes/.venv/bin/python 2>/dev/null; then exit 1; fi; \
+            exit 0' \
     && test ! -e /opt/hermes/.venv/lib/.nemoclaw-sandbox-write-probe \
+    && install -d -o root -g root -m 0755 /tmp/nemoclaw-hindsight-probe \
+    && /opt/hermes/.venv/bin/python -I -c \
+        "import zipfile; entries = {'hindsight_client/__init__.py': 'NEMOCLAW_BUILD_PROBE_FIXTURE = True\n', 'hindsight_client-0.6.1.dist-info/METADATA': 'Metadata-Version: 2.4\nName: hindsight-client\nVersion: 0.6.1\nSummary: NemoClaw offline lazy-installer build probe\n\n', 'hindsight_client-0.6.1.dist-info/WHEEL': 'Wheel-Version: 1.0\nGenerator: NemoClaw build probe\nRoot-Is-Purelib: true\nTag: py3-none-any\n', 'hindsight_client-0.6.1.dist-info/RECORD': 'hindsight_client/__init__.py,,\nhindsight_client-0.6.1.dist-info/METADATA,,\nhindsight_client-0.6.1.dist-info/WHEEL,,\nhindsight_client-0.6.1.dist-info/RECORD,,\n'}; wheel = zipfile.ZipFile('/tmp/nemoclaw-hindsight-probe/hindsight_client-0.6.1-py3-none-any.whl', 'w', compression=zipfile.ZIP_DEFLATED); [wheel.writestr(name, data) for name, data in entries.items()]; wheel.close()" \
+    && chown root:root /tmp/nemoclaw-hindsight-probe/hindsight_client-0.6.1-py3-none-any.whl \
+    && chmod 0444 /tmp/nemoclaw-hindsight-probe/hindsight_client-0.6.1-py3-none-any.whl \
+    && rm -rf /sandbox/.hermes/lazy-packages \
+    && install -d -o sandbox -g sandbox -m 0750 /sandbox/.hermes/lazy-packages \
+    && chmod u=rwx,g=rx,o=,g-s /sandbox/.hermes/lazy-packages \
     && test "$(stat -c '%U:%G %a' /sandbox/.hermes/lazy-packages)" = "sandbox:sandbox 750" \
+    && test -z "$(find /sandbox/.hermes/lazy-packages -mindepth 1 -print -quit)" \
     && HOME=/sandbox \
+        PIP_FIND_LINKS=/tmp/nemoclaw-hindsight-probe \
+        PIP_NO_INDEX=1 \
         UV_CACHE_DIR=/sandbox/.hermes/cache/uv \
+        UV_FIND_LINKS=/tmp/nemoclaw-hindsight-probe \
         UV_NO_CACHE=1 \
+        UV_OFFLINE=1 \
         HERMES_HOME=/sandbox/.hermes \
         HERMES_LAZY_INSTALL_TARGET=/sandbox/.hermes/lazy-packages \
         /usr/bin/setpriv --reuid=sandbox --regid=sandbox --init-groups -- \
         /opt/hermes/.venv/bin/python -I -c \
         "from tools.lazy_deps import ensure; ensure('memory.hindsight', prompt=False)" \
+    && rm -rf /tmp/nemoclaw-hindsight-probe \
     && HERMES_LAZY_INSTALL_TARGET=/sandbox/.hermes/lazy-packages \
         /usr/bin/setpriv --reuid=sandbox --regid=sandbox --init-groups -- \
         /opt/hermes/.venv/bin/python -I -c \
-        "from tools.lazy_deps import activate_durable_lazy_target; activate_durable_lazy_target(); import importlib.metadata as m; assert m.version('hindsight-client') == '0.6.1'" \
+        "from tools.lazy_deps import activate_durable_lazy_target; activate_durable_lazy_target(); import hindsight_client, importlib.metadata as m; from pathlib import Path; target = Path('/sandbox/.hermes/lazy-packages').resolve(); assert m.version('hindsight-client') == '0.6.1'; assert hindsight_client.NEMOCLAW_BUILD_PROBE_FIXTURE is True; assert Path(hindsight_client.__file__).resolve().is_relative_to(target)" \
     && lazy_plan='{"version":1,"readOnlyRoots":["lazy-packages"],"confidentialRoots":[],"readOnlyPrefixes":[],"confidentialPrefixes":[],"writableSubpaths":[]}' \
     && /usr/local/lib/nemoclaw/state-dir-guard.py lock \
         --config-dir /sandbox/.hermes --plan-json "$lazy_plan" \
     && /usr/bin/setpriv --reuid=gateway --regid=gateway --init-groups -- \
         env HERMES_LAZY_INSTALL_TARGET=/sandbox/.hermes/lazy-packages \
         /opt/hermes/.venv/bin/python -I -c \
-        "from tools.lazy_deps import activate_durable_lazy_target; activate_durable_lazy_target(); import hindsight_client" \
+        "from tools.lazy_deps import activate_durable_lazy_target; activate_durable_lazy_target(); import hindsight_client; from pathlib import Path; assert hindsight_client.NEMOCLAW_BUILD_PROBE_FIXTURE is True; assert Path(hindsight_client.__file__).resolve().is_relative_to(Path('/sandbox/.hermes/lazy-packages').resolve())" \
     && if /usr/bin/setpriv --reuid=gateway --regid=gateway --init-groups -- \
         sh -c ': > /sandbox/.hermes/lazy-packages/.nemoclaw-gateway-write-probe' 2>/dev/null; then \
         echo "ERROR: gateway can modify locked Hermes lazy packages" >&2; exit 1; \
@@ -1230,7 +1249,9 @@ RUN /opt/hermes/.venv/bin/python -I -m ensurepip --upgrade --default-pip \
     && test -f /sandbox/.hermes/lazy-packages/.nemoclaw-sandbox-unlock-probe \
     && rm -rf /sandbox/.hermes/lazy-packages \
     && install -d -o sandbox -g sandbox -m 0750 /sandbox/.hermes/lazy-packages \
+    && chmod u=rwx,g=rx,o=,g-s /sandbox/.hermes/lazy-packages \
     && test "$(stat -c '%U:%G %a' /sandbox/.hermes/lazy-packages)" = "sandbox:sandbox 750" \
+    && rm -rf /sandbox/.cache \
     && chown sandbox:sandbox /sandbox/.hermes \
     && chmod 3770 /sandbox/.hermes \
     && test "$(stat -c '%U:%G %a' /sandbox/.hermes)" = "sandbox:sandbox 3770"

agents/hermes/Dockerfile.base

@@ -555,6 +555,41 @@ RUN printf '%s\n' \
         "import multipart; assert multipart.__version__ == '0.0.32', multipart.__version__" \
     && rm -rf /root/.cache/uv
 
+# Keep official-package compatibility separate from the final image's offline
+# lazy-installer contract. BuildKit verifies both reviewed PyPI artifacts before
+# any build command can consume them. Install only those hash-locked wheels into
+# a temporary durable target without network or dependency resolution, import
+# them against the exact base environment, and remove the probe before
+# publishing the base image.
+COPY --chmod=0444 agents/hermes/hindsight-client-probe-requirements.txt /tmp/nemoclaw-hindsight-client-probe-requirements.txt
+ADD --chmod=0444 --checksum=sha256:9fdda176ab50f7cec8d7339c6608c148f0cd9ad7e65d9d76192f2db730bc330a https://files.pythonhosted.org/packages/24/4f/a1d0bc33ef933ecc52e76dc1514163594d25836a5d303c256a61bb61445d/hindsight_client-0.6.1-py3-none-any.whl /tmp/nemoclaw-hindsight-client-artifacts/hindsight_client-0.6.1-py3-none-any.whl
+ADD --chmod=0444 --checksum=sha256:66d2759d1921838256a05a3f80ad7e724936f083e35be5abb5e16eed6be6dc54 https://files.pythonhosted.org/packages/1a/99/84ba7273339d0f3dfa57901b846489d2e5c2cd731470167757f1935fffbd/aiohttp_retry-2.9.1-py3-none-any.whl /tmp/nemoclaw-hindsight-client-artifacts/aiohttp_retry-2.9.1-py3-none-any.whl
+RUN --network=none install -d -o sandbox -g sandbox -m 0750 \
+        /tmp/nemoclaw-hindsight-client-probe \
+        /tmp/nemoclaw-hindsight-client-cache \
+    && chmod 0555 /tmp/nemoclaw-hindsight-client-artifacts \
+    && HOME=/sandbox \
+        UV_CACHE_DIR=/tmp/nemoclaw-hindsight-client-cache \
+        UV_NO_CACHE=1 \
+        /usr/bin/setpriv --reuid=sandbox --regid=sandbox --init-groups -- \
+        /usr/local/bin/uv pip install \
+        --python /opt/hermes/.venv/bin/python \
+        --target /tmp/nemoclaw-hindsight-client-probe \
+        --no-deps --no-cache --offline --no-index \
+        --find-links /tmp/nemoclaw-hindsight-client-artifacts \
+        --require-hashes -r /tmp/nemoclaw-hindsight-client-probe-requirements.txt \
+    && HERMES_LAZY_INSTALL_TARGET=/tmp/nemoclaw-hindsight-client-probe \
+        /usr/bin/setpriv --reuid=sandbox --regid=sandbox --init-groups -- \
+        /opt/hermes/.venv/bin/python -I -c \
+        "from tools.lazy_deps import activate_durable_lazy_target; activate_durable_lazy_target(); import hindsight_client, importlib.metadata as m; from pathlib import Path; target = Path('/tmp/nemoclaw-hindsight-client-probe').resolve(); assert m.version('hindsight-client') == '0.6.1'; assert m.version('aiohttp-retry') == '2.9.1'; assert Path(hindsight_client.__file__).resolve().is_relative_to(target)" \
+    && rm -rf \
+        /tmp/nemoclaw-hindsight-client-artifacts \
+        /tmp/nemoclaw-hindsight-client-probe \
+        /tmp/nemoclaw-hindsight-client-cache \
+        /tmp/nemoclaw-hindsight-client-probe-requirements.txt \
+        /sandbox/.hermes/lazy-packages \
+    && install -d -o sandbox -g sandbox -m 0750 /sandbox/.hermes/lazy-packages
+
 ENV PATH="/usr/local/bin:/opt/hermes/.venv/bin:${PATH}" \
     HERMES_TUI_DIR="/opt/hermes/ui-tui" \
     HERMES_WEB_DIST="/opt/hermes/hermes_cli/web_dist"

agents/hermes/hindsight-client-probe-requirements.txt

@@ -0,0 +1,9 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Build-only compatibility probe. Keep these hashes synchronized with the
+# checksum-pinned PyPI wheels in Dockerfile.base.
+hindsight-client==0.6.1 \
+    --hash=sha256:9fdda176ab50f7cec8d7339c6608c148f0cd9ad7e65d9d76192f2db730bc330a
+aiohttp-retry==2.9.1 \
+    --hash=sha256:66d2759d1921838256a05a3f80ad7e724936f083e35be5abb5e16eed6be6dc54

src/lib/actions/sandbox/stop.test.ts

@@ -9,6 +9,7 @@ import {
   type DockerRuntimeProviderDependencies,
 } from "../../onboard/runtime-provider/docker";
 import { createRuntimeProviderBundleRegistry } from "../../onboard/runtime-provider/registry";
+import { exclusivelyHeldOllamaModel } from "../../inference/ollama/model-ownership";
 import type { SandboxEntry } from "../../state/registry";
 import { teardownSandboxDashboardForward } from "./forward-recovery";
 import { type SandboxStopDeps, stopSandbox } from "./stop";
@@ -72,6 +73,7 @@ function harness(overrides: StopHarnessOverrides = {}) {
     teardownSandboxDashboardForward,
     log,
     warn,
+    exclusivelyHeldOllamaModel,
     withOllamaModelOwnershipLock: (operation) => operation(),
     ...actionOverrides,
   };

src/lib/actions/sandbox/stop.ts

@@ -48,13 +48,17 @@ function unloadOllamaModelsBestEffort(
 ): void {
   if (!sandbox.provider?.includes("ollama")) return;
   try {
-    const ownership = require("../../inference/ollama/model-ownership") as typeof import("../../inference/ollama/model-ownership");
-    const proxy = require("../../inference/ollama/proxy") as typeof import("../../inference/ollama/proxy");
     const withOwnershipLock =
-      deps.withOllamaModelOwnershipLock ?? proxy.withOllamaModelOwnershipLock;
+      deps.withOllamaModelOwnershipLock ??
+      (require("../../inference/ollama/proxy") as typeof import("../../inference/ollama/proxy"))
+        .withOllamaModelOwnershipLock;
+    const exclusivelyHeldOllamaModel =
+      deps.exclusivelyHeldOllamaModel ??
+      (require("../../inference/ollama/model-ownership") as typeof import("../../inference/ollama/model-ownership"))
+        .exclusivelyHeldOllamaModel;
     withOwnershipLock(() => {
       const { sandboxes } = (deps.listSandboxes ?? registry.listSandboxes)();
-      const model = ownership.exclusivelyHeldOllamaModel(sandbox, sandboxes);
+      const model = exclusivelyHeldOllamaModel(sandbox, sandboxes);
       if (!model) return;
       (deps.unloadOllamaModels ?? defaultUnloadOllamaModels)([model]);
     });
@@ -73,6 +77,7 @@ export interface SandboxStopDeps {
   teardownSandboxDashboardForward?: typeof teardownSandboxDashboardForward;
   listSandboxes?: typeof registry.listSandboxes;
   unloadOllamaModels?: (onlyModels: readonly string[]) => void;
+  exclusivelyHeldOllamaModel?: typeof import("../../inference/ollama/model-ownership").exclusivelyHeldOllamaModel;
   withOllamaModelOwnershipLock?: typeof import("../../inference/ollama/proxy").withOllamaModelOwnershipLock;
   log?: (message: string) => void;
   warn?: (message: string) => void;