fix(installer): recover before host preflight

scripts/install.sh

@@ -189,6 +189,40 @@ clone_nemoclaw_ref() {
   )
 }
 
+standalone_installer_payload_needs_checkout() {
+  [[ -n "${DOCKER_HOST-}" ]] || return 1
+  [[ "${NEMOCLAW_BOOTSTRAP_PAYLOAD:-}" != "1" ]] || return 1
+  local source_root="${NEMOCLAW_SOURCE_ROOT:-$(resolve_repo_root)}" arg
+  [[ ! -f "${source_root}/src/lib/domain/docker-host.ts" ]] || return 1
+  for arg in "$@"; do
+    case "$arg" in
+      --help | -h | --version | -v) return 1 ;;
+    esac
+  done
+  return 0
+}
+
+bootstrap_standalone_installer_payload() (
+  local ref bootstrap_tmpdir source_root payload_script docker_host_contract status=0
+  command_exists git || error "git was not found on PATH."
+  ref="$(resolve_release_tag)"
+  bootstrap_tmpdir="$(mktemp -d "${TMPDIR:-/tmp}/nemoclaw-installer-source-XXXXXX")" \
+    || error "Could not prepare the complete NemoClaw installer source."
+  trap 'rm -rf "$bootstrap_tmpdir"' EXIT
+  source_root="${bootstrap_tmpdir}/source"
+  clone_nemoclaw_ref "$ref" "$source_root"
+  payload_script="${source_root}/scripts/install.sh"
+  docker_host_contract="${source_root}/src/lib/domain/docker-host.ts"
+  if [[ ! -s "$payload_script" || ! -s "$docker_host_contract" ]] \
+    || ! grep -q 'NEMOCLAW_VERSIONED_INSTALLER_PAYLOAD=1' "$payload_script" \
+    || ! bash -n "$payload_script"; then
+    error "The selected NemoClaw source does not contain a valid versioned installer and Docker host validation contract."
+  fi
+  NEMOCLAW_INSTALL_REF="$ref" NEMOCLAW_INSTALL_TAG="$ref" NEMOCLAW_BOOTSTRAP_PAYLOAD=1 \
+    bash "$payload_script" "$@" || status=$?
+  return "$status"
+)
+
 # ---------------------------------------------------------------------------
 # Color / style — disabled when NO_COLOR is set or stdout is not a TTY.
 # Uses exact NVIDIA green #76B900 on truecolor terminals; 256-color otherwise.
@@ -834,6 +868,8 @@ print_done() {
   # not clean either.
   if [[ "${_UPGRADE_SANDBOXES_FAILED:-false}" == true ]]; then
     warn "=== Installation completed with warnings ==="
+  elif [[ "${_PREEXISTING_SANDBOX_RECOVERY_UNCONFIRMED:-false}" == true ]]; then
+    warn "=== Installation completed with warnings ==="
   elif [[ "${_PREEXISTING_SANDBOX_ORPHANED:-false}" == true ]]; then
     warn "=== Installation completed with warnings ==="
   else
@@ -843,7 +879,10 @@ print_done() {
   printf "  ${C_GREEN}${C_BOLD}%s${C_RESET}  ${C_DIM}(%ss)${C_RESET}\n" "$_CLI_DISPLAY" "$elapsed"
   printf "\n"
   if [[ "${_PREEXISTING_SANDBOX_RECOVERY_RAN:-false}" == true ]]; then
-    if [[ "${_PREEXISTING_SANDBOX_ORPHANED:-false}" == true ]]; then
+    if [[ "${_PREEXISTING_SANDBOX_RECOVERY_UNCONFIRMED:-false}" == true ]]; then
+      printf "  ${C_YELLOW}The recovery command succeeded, but NemoClaw could not inspect its output.${C_RESET}\n"
+      printf "  ${C_DIM}Run '%s upgrade-sandboxes --check' to verify every recorded sandbox.${C_RESET}\n" "$_CLI_BIN"
+    elif [[ "${_PREEXISTING_SANDBOX_ORPHANED:-false}" == true ]]; then
       # #6520: recovery exited 0 but recorded sandboxes were not found on
       # their own recorded gateway; do not report them as recovered, and give
       # a concrete remediation path instead.
@@ -859,7 +898,9 @@ print_done() {
       printf "  ${C_GREEN}For this terminal:${C_RESET}\n"
       print_cli_path_refresh_actions
     fi
-    if [[ "${_PREEXISTING_SANDBOX_ORPHANED:-false}" == true ]]; then
+    if [[ "${_PREEXISTING_SANDBOX_RECOVERY_UNCONFIRMED:-false}" == true ]]; then
+      printf "  ${C_DIM}Generic onboarding was skipped because recovery verification is incomplete.${C_RESET}\n"
+    elif [[ "${_PREEXISTING_SANDBOX_ORPHANED:-false}" == true ]]; then
       printf "  ${C_DIM}Generic onboarding was skipped because recorded sandboxes exist.${C_RESET}\n"
     else
       printf "  ${C_DIM}No new sandbox onboarding was needed.${C_RESET}\n"
@@ -1275,6 +1316,7 @@ _NEMOCLAW_CLI_INSTALL_MODE=""
 _OPENSHELL_INSTALL_REQUIRED_BEFORE_RECOVERY=false
 _PREEXISTING_SANDBOX_COUNT=0
 _PREEXISTING_SANDBOX_RECOVERY_RAN=false
+_PREEXISTING_SANDBOX_RECOVERY_UNCONFIRMED=false
 # #6520: set when the automatic recovery pass exited 0 but skipped recorded
 # sandboxes it could not observe on the selected gateway (e.g. their gateway
 # and Docker image were removed by a prior uninstall while sandboxes.json was
@@ -3973,9 +4015,12 @@ recover_preexisting_sandboxes_before_onboard() {
   # src/lib/actions/upgrade-sandboxes.ts.
   local recovery_log=""
   recovery_log="$(mktemp "${TMPDIR:-/tmp}/nemoclaw-recovery-XXXXXX" 2>/dev/null)" || recovery_log=""
-  local recovery_status=0 recovery_pass=1
+  local recovery_status=0 recovery_pass=1 orphan_marker_status=0
+  local -a recovery_pipeline_status=()
   if [ -n "$recovery_log" ]; then
     _cleanup_files+=("$recovery_log")
+  else
+    _PREEXISTING_SANDBOX_RECOVERY_UNCONFIRMED=true
   fi
   while [ "$recovery_pass" -le 2 ]; do
     recovery_status=0
@@ -3987,7 +4032,11 @@ recover_preexisting_sandboxes_before_onboard() {
         # pipefail: take the CLI's own status, not tee's — a log-write failure
         # (e.g. ENOSPC on TMPDIR) must not convert a successful recovery into
         # the #5735 failure path.
-        recovery_status=${PIPESTATUS[0]}
+        recovery_pipeline_status=("${PIPESTATUS[@]}")
+        recovery_status="${recovery_pipeline_status[0]:-1}"
+        if [ "${recovery_pipeline_status[1]:-1}" -ne 0 ]; then
+          _PREEXISTING_SANDBOX_RECOVERY_UNCONFIRMED=true
+        fi
       fi
     else
       NEMOCLAW_CONFIRMED_LEGACY_MANAGED_SANDBOXES="${_LEGACY_MANAGED_RECOVERY_NAMES_JSON:-[]}" \
@@ -4006,9 +4055,16 @@ recover_preexisting_sandboxes_before_onboard() {
   done
   if [ "$recovery_status" -eq 0 ]; then
     _PREEXISTING_SANDBOX_RECOVERY_RAN=true
-    if [ -n "$recovery_log" ] \
-      && grep -Fq "recorded sandbox(es) were not found on their recorded gateway" "$recovery_log"; then
-      _PREEXISTING_SANDBOX_ORPHANED=true
+    if [[ "${_PREEXISTING_SANDBOX_RECOVERY_UNCONFIRMED:-false}" != true ]] \
+      && [ -n "$recovery_log" ]; then
+      orphan_marker_status=0
+      grep -Fq "recorded sandbox(es) were not found on their recorded gateway" "$recovery_log" \
+        || orphan_marker_status=$?
+      case "$orphan_marker_status" in
+        0) _PREEXISTING_SANDBOX_ORPHANED=true ;;
+        1) ;;
+        *) _PREEXISTING_SANDBOX_RECOVERY_UNCONFIRMED=true ;;
+      esac
     fi
     rm -f "$recovery_log" 2>/dev/null || true
     return 0
@@ -6236,6 +6292,27 @@ install_nemoclaw_before_onboarding() {
   # `nemoclaw onboard` (the install-ollama / install-vllm branches).
   # install.sh stays focused on dependency setup.
   fix_npm_permissions
+  if [[ -n "${DOCKER_HOST-}" ]]; then
+    local docker_host_contract="${NEMOCLAW_SOURCE_ROOT}/src/lib/domain/docker-host.ts"
+    if ! command_exists node || [[ ! -f "$docker_host_contract" ]]; then
+      error "Could not validate DOCKER_HOST before sandbox recovery. Rerun the installer from a complete NemoClaw source checkout. Sandbox recovery did not start."
+    fi
+    local docker_host_status=0
+    node --experimental-strip-types --no-warnings -e '
+      const validator = require(process.argv[1]).isSupportedGatewayDockerHost;
+      if (typeof validator !== "function") process.exit(11);
+      process.exit(validator(process.argv[2]) ? 0 : 10);
+    ' "$docker_host_contract" "$DOCKER_HOST" >/dev/null 2>&1 || docker_host_status=$?
+    case "$docker_host_status" in
+      0) ;;
+      10)
+        error "DOCKER_HOST is not a supported absolute local Unix socket endpoint. Unset DOCKER_HOST or set it to an absolute local Unix socket URL, such as unix:///var/run/docker.sock. Then rerun the installer. Sandbox recovery did not start."
+        ;;
+      *)
+        error "Could not validate DOCKER_HOST before sandbox recovery. Rerun the installer from a complete NemoClaw source checkout. Sandbox recovery did not start."
+        ;;
+    esac
+  fi
   preinstall_backup_and_retire_legacy_gateway
   install_nemoclaw
   verify_nemoclaw
@@ -6425,33 +6502,48 @@ main() {
     fi
     if should_defer_hermes_onboarding "$_registered_sandbox_count"; then
       info "NVIDIA inference credentials are absent. Hermes onboarding did not run."
-    elif run_installer_host_preflight; then
+    else
       if ! recover_preexisting_sandboxes_before_onboard "$_cli_runner"; then
         finalize_install
         return 1
       fi
+
+      local _run_onboard_after_recovery=false
       if [[ "${_PREEXISTING_SANDBOX_RECOVERY_RAN:-false}" == true ]]; then
-        if [[ "${_PREEXISTING_SANDBOX_ORPHANED:-false}" == true ]]; then
+        if [[ "${_PREEXISTING_SANDBOX_RECOVERY_UNCONFIRMED:-false}" == true ]]; then
+          warn "Recovery output could not be inspected; skipping generic onboarding."
+        elif [[ "${_PREEXISTING_SANDBOX_ORPHANED:-false}" == true ]]; then
           # #6520: do not claim recovery when recorded sandboxes are stranded.
           warn "Some recorded sandboxes could not be recovered; skipping generic onboarding."
         elif [[ "${_SELECTED_EXPRESS_PLATFORM:-}" == "DGX Station" ]] \
           || [[ "${_STATION_EXPRESS_RESUME_LOADED:-}" == "1" ]] \
           || station_express_receipt_retirement_pending; then
-          info "Existing sandboxes recovered; reconciling DGX Station Express onboarding state."
-          run_onboard || fail_onboarding "$?"
-          ONBOARD_RAN=true
+          _run_onboard_after_recovery=true
         else
           info "Existing sandboxes recovered; skipping generic onboarding."
         fi
       else
-        run_onboard || fail_onboarding "$?"
-        ONBOARD_RAN=true
-        restore_onboard_forward_after_post_checks || error "Hermes host forward restore failed."
+        _run_onboard_after_recovery=true
+      fi
+
+      if [[ "$_run_onboard_after_recovery" == true ]]; then
+        if run_installer_host_preflight; then
+          if [[ "${_PREEXISTING_SANDBOX_RECOVERY_RAN:-false}" == true ]]; then
+            info "Existing sandboxes recovered; reconciling DGX Station Express onboarding state."
+          fi
+          run_onboard || fail_onboarding "$?"
+          ONBOARD_RAN=true
+          if [[ "${_PREEXISTING_SANDBOX_RECOVERY_RAN:-false}" != true ]]; then
+            restore_onboard_forward_after_post_checks || error "Hermes host forward restore failed."
+          fi
+        elif [ "${NON_INTERACTIVE:-}" = "1" ]; then
+          error "Skipping onboarding until the host prerequisites above are fixed."
+        elif [[ "${_PREEXISTING_SANDBOX_RECOVERY_RAN:-false}" == true ]]; then
+          error "DGX Station reconciliation did not run. Fix the host prerequisites above, then rerun the installer."
+        else
+          warn "Skipping onboarding until the host prerequisites above are fixed."
+        fi
       fi
-    elif [ "${NON_INTERACTIVE:-}" = "1" ]; then
-      error "Skipping onboarding until the host prerequisites above are fixed."
-    else
-      warn "Skipping onboarding until the host prerequisites above are fixed."
     fi
   else
     warn "Skipping onboarding — could not locate the ${_CLI_BIN} executable on disk."
@@ -6507,5 +6599,10 @@ if [[ "${BASH_SOURCE[0]:-}" == "$0" ]] || { [[ -z "${BASH_SOURCE[0]:-}" ]] && {
     # fall through to direct main(). The legacy newgrp/re-curl path still applies.
     rm -f "${_staged:-}" 2>/dev/null
   fi
+  if standalone_installer_payload_needs_checkout "$@"; then
+    _standalone_payload_status=0
+    bootstrap_standalone_installer_payload "$@" || _standalone_payload_status=$?
+    exit "$_standalone_payload_status"
+  fi
   main "$@"
 fi

test/install/install-onboard-yes.test.ts

@@ -749,8 +749,8 @@ describe("Hermes deferred onboarding", () => {
     const result = runDeferredOnboardingMain({ deferFlag: true, registeredSandboxCount: 1 });
 
     expect(result.result.status, result.output).toBe(0);
-    expect(result.calls).toContain("host-preflight");
     expect(result.calls).toContain("recover-preexisting");
+    expect(result.calls).not.toContain("host-preflight");
     expect(result.calls).not.toContain("onboard");
   });