fix(OMN-15137): provision omnibase_spi via a shared sibling-clone manifest

scripts/runtime_build/ensure_runner_clones.sh

@@ -11,33 +11,39 @@
 # dubious-ownership, FETCH_HEAD EACCES, busy-branch worktree contention).
 # Instead it gets a PRIVATE OMNI_HOME bind mount (DEPLOY_RUNNER_OMNI_HOME in
 # docker/docker-compose.runners.yml) owned by the runner uid. This script
-# guarantees the 5 sibling clones exist under that private OMNI_HOME and are
-# operable by the CURRENT euid, so provisioning is a committed, automatic step
-# inside every entry script -- never a remembered manual step on the host.
+# guarantees the sibling clones (see sibling_clone_manifest.sh for the current
+# set) exist under that private OMNI_HOME and are operable by the CURRENT
+# euid, so provisioning is a committed, automatic step inside every entry
+# script -- never a remembered manual step on the host.
 #
 # Called at the top of the execute path of cut_release_train_tag.sh,
 # refresh_stability_lane.sh, and refresh_dev_lane.sh. Safe to re-run: existing
 # clones are left untouched (the refresh scripts already fetch; this script
 # only guarantees existence + operability).
 #
-# All 5 repos are public, so cloning needs NO credentials. The base URL is
+# All repos are public, so cloning needs NO credentials. The base URL is
 # overridable (RUNNER_CLONE_BASE_URL) so tests can point it at local file://
 # bare fixtures.
 #
+# OMN-15137: the clone set is NOT hardcoded here. It is sourced from
+# sibling_clone_manifest.sh -- the single source of truth shared with
+# stage_workspace.sh's sibling-pin preflight -- so this script can never again
+# provision fewer repos than the pin preflight requires (the omnibase_spi gap
+# this ticket fixes: the preflight referenced OMNI_HOME/omnibase_spi 3 deploy
+# hops downstream of this script, which never created it).
+#
 # Exit codes:
 #   0   all clones present and operable by the current euid
 #   64  precondition failure (OMNI_HOME unset / not a directory, git missing,
 #       clone failed, or a clone is not operable/writable by the current euid)
 
 set -euo pipefail
 
-RUNNER_CLONE_REPOS=(
-    "omnibase_infra"
-    "omnibase_core"
-    "omnibase_compat"
-    "onex_change_control"
-    "omnimarket"
-)
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=./sibling_clone_manifest.sh
+source "${SCRIPT_DIR}/sibling_clone_manifest.sh"
+
+RUNNER_CLONE_REPOS=("${SIBLING_CLONE_MANIFEST[@]}")
 
 RUNNER_CLONE_BASE_URL="${RUNNER_CLONE_BASE_URL:-https://github.com/OmniNode-ai}"
 

scripts/runtime_build/sibling_clone_manifest.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc.
+# SPDX-License-Identifier: MIT
+#
+# sibling_clone_manifest.sh -- OMN-15137: single source of truth for every
+# sibling repo that must have a live clone under the deploy runner's OMNI_HOME.
+#
+# Root cause this file closes: ensure_runner_clones.sh (RUNNER_CLONE_REPOS)
+# and stage_workspace.sh's sibling-pin preflight (PREFLIGHT_REPO_ARGS, which
+# maps 1:1 onto check_sibling_lock_pins.py's DEFAULT_PACKAGE_REPO_DIRS) each
+# hardcoded their OWN independent copy of "the sibling repo set" as two
+# separately maintained bash arrays. omnibase_spi was added to the preflight's
+# set (OMN-12977) but never mirrored into the clone-provisioning set, so
+# check_sibling_lock_pins.py referenced a clone directory
+# (OMNI_HOME/omnibase_spi) that ensure_runner_clones.sh never created -- a
+# silent coverage gap that surfaced only 3 hops deep into the pipeline
+# (OMN-15137: "ERROR: cannot resolve clone pin for omnibase-spi: missing
+# pyproject.toml"), after two unrelated defects (OMN-15122, OMN-15131) were
+# fixed and a run finally reached this step for the first time.
+#
+# Both consuming scripts source THIS file instead of hardcoding their own
+# list, so the two can never drift apart again:
+#   - ensure_runner_clones.sh builds RUNNER_CLONE_REPOS from
+#     SIBLING_CLONE_MANIFEST.
+#   - stage_workspace.sh builds PREFLIGHT_REPO_ARGS from
+#     SIBLING_CLONE_MANIFEST + SIBLING_CLONE_MANIFEST_DIST_NAMES.
+# tests/scripts/test_sibling_clone_manifest_parity.py additionally asserts
+# this file's directory set is IDENTICAL to check_sibling_lock_pins.py's
+# DEFAULT_PACKAGE_REPO_DIRS values (the Python side's own canonical mapping),
+# so a future 7th sibling added on one side and forgotten on the other fails
+# CI immediately instead of failing 3 deploy hops deep on a real runner.
+#
+# NOT the same list as stage_workspace.sh's SIBLING_REPOS (the narrower
+# subset actually vendored as SOURCE into the runtime image via rsync):
+# omnibase_infra is the Docker build context itself (never vendored as a
+# "sibling"), and omnibase_spi is installed from the published wheel via
+# `uv sync` (never staged from a local source tree) -- both still need a
+# live OMNI_HOME clone so the pin-preflight can read their pyproject.toml
+# version + git HEAD SHA against the consuming lock file.
+#
+# Arrays below are INDEX-ALIGNED: SIBLING_CLONE_MANIFEST[i] is the OMNI_HOME
+# directory name; SIBLING_CLONE_MANIFEST_DIST_NAMES[i] is the corresponding
+# uv.lock distribution (package) name for that same repo.
+# shellcheck disable=SC2034  # consumed by scripts that `source` this file
+SIBLING_CLONE_MANIFEST=(
+    "omnibase_infra"
+    "omnibase_core"
+    "omnibase_spi"
+    "omnibase_compat"
+    "onex_change_control"
+    "omnimarket"
+)
+
+# shellcheck disable=SC2034  # consumed by scripts that `source` this file
+SIBLING_CLONE_MANIFEST_DIST_NAMES=(
+    "omnibase-infra"
+    "omnibase-core"
+    "omnibase-spi"
+    "omnibase-compat"
+    "onex-change-control"
+    "omnimarket"
+)

scripts/runtime_build/stage_workspace.sh

@@ -125,15 +125,21 @@ fi
 CONSUMER_LOCK="${CONSUMER_LOCK:-${OMNI_HOME}/omnimarket/uv.lock}"
 PIN_COMPARISON_OUT="workspace/sibling-pin-comparison.json"
 
-# Foundation + sibling packages the build vendors, mapped to OMNI_HOME clone dirs.
-PREFLIGHT_REPO_ARGS=(
-    --repo "omnibase-infra=${OMNI_HOME}/omnibase_infra"
-    --repo "omnibase-core=${OMNI_HOME}/omnibase_core"
-    --repo "omnibase-spi=${OMNI_HOME}/omnibase_spi"
-    --repo "omnibase-compat=${OMNI_HOME}/omnibase_compat"
-    --repo "onex-change-control=${OMNI_HOME}/onex_change_control"
-    --repo "omnimarket=${OMNI_HOME}/omnimarket"
-)
+# Foundation + sibling packages the build vendors, mapped to OMNI_HOME clone
+# dirs. OMN-15137: built from sibling_clone_manifest.sh -- the single source
+# of truth shared with ensure_runner_clones.sh's RUNNER_CLONE_REPOS -- instead
+# of a second independently hardcoded list, so the two can never drift apart
+# again (the omnibase_spi gap this ticket fixes was exactly that drift: this
+# list already named omnibase-spi, but ensure_runner_clones.sh never
+# provisioned OMNI_HOME/omnibase_spi for it to find).
+# shellcheck source=./sibling_clone_manifest.sh
+source "${SCRIPT_DIR}/sibling_clone_manifest.sh"
+PREFLIGHT_REPO_ARGS=()
+for i in "${!SIBLING_CLONE_MANIFEST[@]}"; do
+    PREFLIGHT_REPO_ARGS+=(
+        --repo "${SIBLING_CLONE_MANIFEST_DIST_NAMES[$i]}=${OMNI_HOME}/${SIBLING_CLONE_MANIFEST[$i]}"
+    )
+done
 
 preflight_extra=()
 if [[ "${ALLOW_SIBLING_PIN_DRIFT:-0}" == "1" ]]; then

tests/scripts/test_ensure_runner_clones.py

@@ -0,0 +1,227 @@
+# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc.
+# SPDX-License-Identifier: MIT
+
+"""ensure_runner_clones.sh must provision EVERY sibling the workspace build's
+sibling-pin preflight requires -- not just the 5 it hardcoded before this fix
+(OMN-15137).
+
+Live failure: OMN-14900 stability deploy-hop re-fire iteration N+3 (run
+https://github.com/OmniNode-ai/omnibase_infra/actions/runs/30175447119,
+2026-07-25) got past ensure_runner_clones.sh ("all 5 private clones present
+and operable"), then failed 3 steps later in stage_workspace.sh's sibling-pin
+preflight:
+
+    ERROR: cannot resolve clone pin for omnibase-spi: missing pyproject.toml
+    for omnibase-spi: /data/omninode/runner_omni_home/omnibase_spi/pyproject.toml
+
+Root cause: ``RUNNER_CLONE_REPOS`` in ensure_runner_clones.sh never included
+``omnibase_spi``, even though stage_workspace.sh's preflight
+(``PREFLIGHT_REPO_ARGS`` -> check_sibling_lock_pins.py's
+``DEFAULT_PACKAGE_REPO_DIRS``) has required a live ``OMNI_HOME/omnibase_spi``
+clone since OMN-12977. The two lists were independently hardcoded and drifted
+apart silently.
+
+The fix sources a new shared manifest (``sibling_clone_manifest.sh``) into
+both scripts so they can never diverge again. These tests exercise the
+REAL ``ensure_runner_clones.sh`` script end-to-end against real (local
+``file://``) git remotes -- proving the artifact that runs, not just that the
+source text mentions "omnibase_spi".
+"""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+SCRIPT = REPO_ROOT / "scripts" / "runtime_build" / "ensure_runner_clones.sh"
+MANIFEST = REPO_ROOT / "scripts" / "runtime_build" / "sibling_clone_manifest.sh"
+
+# The full sibling set the workspace build's pin-preflight resolves clones
+# for (check_sibling_lock_pins.py's DEFAULT_PACKAGE_REPO_DIRS). Mirrored here
+# (not imported) so a regression in the manifest itself still fails this test
+# independently -- see test_sibling_clone_manifest_parity.py for the
+# cross-language authoritative-source check.
+EXPECTED_REPOS = (
+    "omnibase_infra",
+    "omnibase_core",
+    "omnibase_spi",
+    "omnibase_compat",
+    "onex_change_control",
+    "omnimarket",
+)
+
+
+def _git(*args: str, cwd: Path) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        ["git", *args],
+        cwd=cwd,
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+
+
+def _make_bare_upstream(upstream_root: Path, repo: str) -> None:
+    """Create a real bare git repo with one commit, clonable via file://."""
+    bare = upstream_root / f"{repo}.git"
+    _git("init", "-q", "--bare", str(bare), cwd=upstream_root)
+
+    work = upstream_root / f".work-{repo}"
+    work.mkdir()
+    _git("init", "-q", cwd=work)
+    _git(
+        "-c",
+        "user.email=test@example.com",
+        "-c",
+        "user.name=test",
+        "commit",
+        "--allow-empty",
+        "-q",
+        "-m",
+        "init",
+        cwd=work,
+    )
+    _git("remote", "add", "origin", str(bare), cwd=work)
+    _git("push", "-q", "origin", "HEAD:refs/heads/main", cwd=work)
+
+
+def _run_ensure_clones(
+    omni_home: Path,
+    base_url: str,
+    *,
+    extra_env: dict[str, str] | None = None,
+) -> subprocess.CompletedProcess[str]:
+    env = {
+        "OMNI_HOME": str(omni_home),
+        "RUNNER_CLONE_BASE_URL": base_url,
+        "PATH": "/usr/bin:/bin:/usr/local/bin:/opt/homebrew/bin",
+        "HOME": str(omni_home.parent),
+    }
+    if extra_env:
+        env.update(extra_env)
+    return subprocess.run(
+        ["bash", str(SCRIPT)],
+        env=env,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+
+
+@pytest.fixture
+def upstream(tmp_path: Path) -> Path:
+    """A directory of real bare git repos for all 6 canonical siblings."""
+    root = tmp_path / "upstream"
+    root.mkdir()
+    for repo in EXPECTED_REPOS:
+        _make_bare_upstream(root, repo)
+    return root
+
+
+def test_manifest_declares_all_six_canonical_siblings() -> None:
+    """sibling_clone_manifest.sh enumerates the full set, including spi."""
+    text = MANIFEST.read_text(encoding="utf-8")
+    for repo in EXPECTED_REPOS:
+        assert f'"{repo}"' in text, f"{repo} missing from sibling_clone_manifest.sh"
+
+
+def test_ensure_runner_clones_provisions_all_six_including_spi(
+    tmp_path: Path, upstream: Path
+) -> None:
+    """RED against the exact OMN-15137 gap: omnibase_spi must be cloned."""
+    omni_home = tmp_path / "omni_home"
+    omni_home.mkdir()
+
+    result = _run_ensure_clones(omni_home, f"file://{upstream}")
+
+    assert result.returncode == 0, result.stderr
+    assert "all 6 private clones present and operable" in result.stderr
+
+    for repo in EXPECTED_REPOS:
+        clone = omni_home / repo
+        assert (clone / ".git").exists(), f"{repo} was not cloned under OMNI_HOME"
+
+    # The specific regression: omnibase_spi must exist and be a real repo.
+    spi = omni_home / "omnibase_spi"
+    rev_parse = _git("rev-parse", "--git-dir", cwd=spi)
+    assert rev_parse.returncode == 0
+
+
+def test_ensure_runner_clones_is_idempotent(tmp_path: Path, upstream: Path) -> None:
+    omni_home = tmp_path / "omni_home"
+    omni_home.mkdir()
+
+    first = _run_ensure_clones(omni_home, f"file://{upstream}")
+    assert first.returncode == 0, first.stderr
+
+    heads_before = {
+        repo: _git("rev-parse", "HEAD", cwd=omni_home / repo).stdout.strip()
+        for repo in EXPECTED_REPOS
+    }
+
+    second = _run_ensure_clones(omni_home, f"file://{upstream}")
+    assert second.returncode == 0, second.stderr
+    assert "clone missing" not in second.stderr
+
+    for repo in EXPECTED_REPOS:
+        head_after = _git("rev-parse", "HEAD", cwd=omni_home / repo).stdout.strip()
+        assert head_after == heads_before[repo]
+
+
+def test_missing_upstream_repo_fails_closed_naming_the_repo(
+    tmp_path: Path, upstream: Path
+) -> None:
+    """If ANY sibling's upstream is unreachable, fail loud and name it --
+    the exact shape of the OMN-15137 defect (a sibling silently never
+    provisioned) must be impossible to reproduce for any repo in the set."""
+    # Remove the omnibase_spi upstream to simulate "not provisioned anywhere"
+    import shutil
+
+    shutil.rmtree(upstream / "omnibase_spi.git")
+
+    omni_home = tmp_path / "omni_home"
+    omni_home.mkdir()
+
+    result = _run_ensure_clones(omni_home, f"file://{upstream}")
+
+    assert result.returncode == 64
+    assert "omnibase_spi" in result.stderr
+    assert "git clone failed" in result.stderr
+    # Must not silently report success.
+    assert "all 6 private clones present" not in result.stderr
+
+
+def test_unwritable_clone_git_dir_fails_closed(tmp_path: Path, upstream: Path) -> None:
+    """A clone whose .git is not writable by the current euid must fail --
+    this is the dubious-ownership / FETCH_HEAD-EACCES failure mode the
+    private-OMNI_HOME provisioning (OMN-14900) exists to catch."""
+    omni_home = tmp_path / "omni_home"
+    omni_home.mkdir()
+
+    first = _run_ensure_clones(omni_home, f"file://{upstream}")
+    assert first.returncode == 0, first.stderr
+
+    spi_git_dir = omni_home / "omnibase_spi" / ".git"
+    spi_git_dir.chmod(0o500)
+    try:
+        result = _run_ensure_clones(omni_home, f"file://{upstream}")
+        assert result.returncode == 64
+        assert "not writable" in result.stderr
+        assert "omnibase_spi" in result.stderr
+    finally:
+        spi_git_dir.chmod(0o700)
+
+
+def test_missing_omni_home_fails_closed() -> None:
+    result = subprocess.run(
+        ["bash", str(SCRIPT)],
+        env={"PATH": "/usr/bin:/bin:/usr/local/bin:/opt/homebrew/bin"},
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 64
+    assert "OMNI_HOME must be set" in result.stderr