diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e01360e9ff..17a55e9a83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,6 +125,28 @@ jobs: - name: LLM topology contract drift validator (OMN-11926) run: PYTHONPATH=src uv run python -m omnibase_infra.validators.llm_topology_validator contracts/llm_endpoints.yaml + # OMN-13026: Transport-mock lint. Bare AsyncMock/MagicMock without + # spec=/spec_set= on EventBus/transport/kafka surfaces. Ratchet: existing + # violations frozen in config/validation/transport_mock_baseline.yaml. New + # violations in any PR are blocked. Drain tracked by per-site tickets. + # SYNC with pre-commit hook 'transport-mock-lint'. + # Clone omnibase_core HEAD so the validator is available before the + # OMN-13026 omnibase_core pin bump lands in the lockfile. + - name: Checkout omnibase_core validator source + uses: actions/checkout@v6 + with: + repository: OmniNode-ai/omnibase_core + ref: dev + path: .ci/omnibase_core + token: ${{ secrets.OMNIBASE_CORE_TOKEN || secrets.GITHUB_TOKEN }} + + - name: Transport-mock lint (OMN-13026) + run: | + PYTHONPATH=.ci/omnibase_core/src uv run python \ + -m omnibase_core.validators.transport_mock_lint \ + --baseline config/validation/transport_mock_baseline.yaml \ + tests/ + - name: Run mypy type checking run: uv run mypy src/omnibase_infra @@ -982,6 +1004,13 @@ jobs: echo "================================================================" uv run python scripts/check_dockerfile_pins.py + - name: Check runtime image identity stamping + run: | + echo "================================================================" + echo "Runtime Image Identity Check (OMN-12965)" + echo "================================================================" + uv run python scripts/check_runtime_image_identity.py + # ============================================================================ # ONEX Compliance Check # Runs `onex compliance check` to validate contracts, handlers, and schemas. diff --git a/.github/workflows/dispatcher-route-coverage.yml b/.github/workflows/dispatcher-route-coverage.yml new file mode 100644 index 0000000000..23a067d6b9 --- /dev/null +++ b/.github/workflows/dispatcher-route-coverage.yml @@ -0,0 +1,134 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# dispatcher-route-coverage.yml — OMN-12858 / OMN-12879 / OMN-12880 +# +# CI gate: for every contract.yaml in omnibase_infra + omnimarket that +# subscribes to a command topic (onex.cmd.*), assert that the contract +# declares a dispatcher route via handler_routing or runtime_dispatch. +# +# Missing wiring causes the command to go to DLQ silently. This gate +# catches that regression before it reaches the runtime. +# +# MOTIVATION — two incidents this gate would have caught: +# +# 1. June 9 2026 DLQ regression: node_generation_consumer subscribed +# onex.cmd.omnimarket.node-generation-requested.v1 but auto-wiring +# produced zero dispatchers after a sole-handler revert was merged. +# Messages silently went to DLQ. (OMN-12858 post-mortem) +# +# 2. June 12 2026 DEL-01 live finding: onex.cmd.omnimarket.delegate-skill.v1 +# was consumed by the dev lane bus but no dispatcher route existed in any +# deployed contract, so delegation requests went to DLQ silently. +# Discovered via manual rpk consumer-group lag probe (DEL-01 evidence). +# +# Coverage scope (OMN-12880): +# - All subscribed topics where topic contains '.cmd.' +# - compatibility_publish_topics are sender-side only; they are NOT +# checked for missing routes (no false positives). +# +# Changed-command-topic mode (OMN-12879): +# - When the PR diff touches contract.yaml files, only those contracts are +# checked. Pre-existing allowlisted violations are not re-reported. +# +# Ratchet / baseline model: +# - Known pre-existing violations are in scripts/check_dispatcher_route_coverage.py +# _ALLOWLISTED_CMD_TOPICS with ticket refs and expiry dates. +# - New violations are NEVER silently added — they must be fixed. + +name: Dispatcher Route Coverage + +on: + pull_request: + branches: [main, dev] + push: + branches: [main] + merge_group: + +permissions: + contents: read + +jobs: + dispatcher-route-coverage: + name: dispatcher-route-coverage + runs-on: >- + ${{ + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) + && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') + || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') + }} + timeout-minutes: 5 + + steps: + - name: Checkout omnibase_infra + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Checkout omnimarket (sibling) + uses: actions/checkout@v6 + with: + repository: OmniNode-ai/omnimarket + ref: dev + path: omnimarket + token: ${{ secrets.OMNI_GITHUB_TOKEN || github.token }} + # Sparse checkout: only the contract files needed for coverage analysis + sparse-checkout: | + src/omnimarket/nodes + sparse-checkout-cone-mode: false + + - name: Install pyyaml (only dep for gate script) + shell: bash + run: | + # The gate script uses only stdlib + pyyaml; skip full uv sync to keep + # this job fast (no Kafka, no DB, no handler imports). + python3 -m pip install --quiet pyyaml + + - name: Collect changed contract paths (OMN-12879) + id: changed_contracts + shell: bash + run: | + set -euo pipefail + # Find contract.yaml files changed in this PR diff vs the base branch. + # For push/merge_group events there is no PR diff; run full scan. + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + BASE="${{ github.event.pull_request.base.sha }}" + HEAD="${{ github.event.pull_request.head.sha }}" + CHANGED=$(git diff --name-only "$BASE" "$HEAD" -- '**/contract.yaml' || true) + if [[ -n "$CHANGED" ]]; then + # Convert to absolute paths so the script can match them + ABS_PATHS=$(echo "$CHANGED" | xargs -I{} realpath --relative-to=. {} 2>/dev/null || echo "$CHANGED") + echo "changed_contracts=$ABS_PATHS" >> "$GITHUB_OUTPUT" + echo "Found changed contracts:" + echo "$CHANGED" | head -20 + else + echo "changed_contracts=" >> "$GITHUB_OUTPUT" + echo "No contract.yaml files changed; full scan will run." + fi + else + echo "changed_contracts=" >> "$GITHUB_OUTPUT" + echo "Non-PR event; running full scan." + fi + + - name: Run dispatcher route-coverage gate + shell: bash + run: | + set -euo pipefail + + CHANGED="${{ steps.changed_contracts.outputs.changed_contracts }}" + EXTRA_DIR="omnimarket/src/omnimarket/nodes" + + ARGS=( + "--contracts-dir" "src/omnibase_infra/nodes" + "--extra-contracts-dir" "$EXTRA_DIR" + ) + + if [[ -n "${CHANGED}" ]]; then + echo "::notice::Running in changed-contract mode (OMN-12879): $(echo "${CHANGED}" | wc -w) paths" + ARGS+=("--changed-contracts" "${CHANGED}") + else + echo "::notice::Running full scan across both contract directories" + fi + + python3 scripts/check_dispatcher_route_coverage.py "${ARGS[@]}" diff --git a/.github/workflows/node-migration-sync.yml b/.github/workflows/node-migration-sync.yml new file mode 100644 index 0000000000..4a9965d4c6 --- /dev/null +++ b/.github/workflows/node-migration-sync.yml @@ -0,0 +1,85 @@ +# Node Migration Vendor Sync Gate (OMN-13124) +# +# WHY THIS EXISTS +# omnimarket projection nodes ship SQL under +# src/omnimarket/nodes//migrations/*.sql +# which must be vendored into omnibase_infra under +# docker/migrations/forward/nodes// (by scripts/sync-node-migrations.sh) +# so the forward-migration runner recreates the projection tables on a clean +# redeploy. The local pre-commit hook (onex-check-node-migration-sync) only +# runs in the infra repo and only when an infra commit is made; an omnimarket +# PR that adds a node migration cannot fire it. OMN-13124's +# node_projection_pattern_learning merged WITHOUT its vendored copy, so +# pattern_learning_artifacts was absent from omnidash_analytics on a clean +# redeploy and the golden-chain pattern_learning chain only passed via a live +# hot-fix (golden-chain sweep run12 Finding). This gate is the enforcement +# layer: every infra PR (and merge_group) re-runs the vendor --check against +# the current omnimarket dev tip, so an un-vendored or drifted node migration +# fails CI before merge. + +name: node-migration-sync + +on: + pull_request: + branches: [main, dev] + push: + branches: [main] + merge_group: + +permissions: + contents: read + +jobs: + node-migration-sync: + name: node-migration-sync + runs-on: >- + ${{ + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository) + && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') + || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') + }} + timeout-minutes: 5 + + steps: + - name: Checkout omnibase_infra + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Checkout omnimarket (sibling — node migration source of truth) + uses: actions/checkout@v6 + with: + repository: OmniNode-ai/omnimarket + ref: dev + path: omnimarket-src + token: ${{ secrets.OMNI_GITHUB_TOKEN || github.token }} + # Sparse checkout: only the node tree carrying migrations/*.sql. + sparse-checkout: | + src/omnimarket/nodes + sparse-checkout-cone-mode: false + + - name: Run node migration vendor sync --check + shell: bash + env: + # The script's resolution order prefers OMNIMARKET_SRC (a repo root that + # contains src/omnimarket/nodes). Point it at the sibling checkout so the + # gate compares the vendored tree against the omnimarket dev tip and never + # silently passes on an unresolvable source. + OMNIMARKET_SRC: ${{ github.workspace }}/omnimarket-src + run: | + set -euo pipefail + echo "::group::node migration vendor sync --check" + if bash scripts/sync-node-migrations.sh --check; then + echo "::endgroup::" + echo "::notice::node migration vendor tree is in sync with omnimarket dev tip." + else + rc=$? + echo "::endgroup::" + if [[ "${rc}" -eq 2 ]]; then + echo "::error::Could not resolve the omnimarket source tree (exit 2). The sibling checkout is missing or empty." + else + echo "::error::Vendored node migrations under docker/migrations/forward/nodes/ are OUT OF SYNC with omnimarket. Run 'scripts/sync-node-migrations.sh' locally and commit the diff. An un-vendored node migration means its projection table will be MISSING after a clean redeploy (golden-chain run12 Finding)." + fi + exit "${rc}" + fi diff --git a/.github/workflows/non-dev-base-guard.yml b/.github/workflows/non-dev-base-guard.yml new file mode 100644 index 0000000000..756db0ad01 --- /dev/null +++ b/.github/workflows/non-dev-base-guard.yml @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# OMN-13021 (retro A-6): Fail PRs whose base is neither dev nor main unless +# the body declares an intentional stack via `Stacked-Parent: #`. +# +# Failure class: feedback_stacked_prs_orphan_from_dev — PRs #1185/#1954 were +# auto-merged INTO parent feature branches, producing misleading MERGED states +# and stranding the work off dev (recurrence of the June 5 incident, 3 tickets +# stranded). Base=main is governed by main-target-guard (OMN-12243/OMN-12477). + +name: Non-dev base guard + +on: + pull_request: + types: [opened, edited, synchronize] + +permissions: + contents: read + +jobs: + non-dev-base-guard: + name: non-dev-base-guard + runs-on: >- + ${{ + (github.event.pull_request.head.repo.full_name != github.repository) + && fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') + || fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') + }} + timeout-minutes: 5 + + steps: + - name: Check PR base branch + env: + BASE_REF: ${{ github.event.pull_request.base.ref }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} + PR_BODY: ${{ github.event.pull_request.body }} + run: | + set -euo pipefail + + if [[ "${BASE_REF}" == "dev" ]]; then + echo "::notice::PR targets dev; guard does not apply." + exit 0 + fi + + if [[ "${BASE_REF}" == "main" ]]; then + echo "::notice::PR targets main; main-target-guard governs that path." + exit 0 + fi + + echo "PR targets feature base '${BASE_REF}' from head '${HEAD_REF}'." + + if echo "${PR_BODY}" | grep -qE 'Stacked-Parent:[[:space:]]*#[0-9]+'; then + echo "::notice::Declared stacked PR (Stacked-Parent found). Allowed." + echo "::warning::Stacked PRs merge INTO their parent branch, not dev." + echo "::warning::Do NOT arm auto-merge on this PR; land the parent into dev first." + exit 0 + fi + + echo "::error::PR base '${BASE_REF}' is not the integration branch (dev)." + echo "::error::Auto-merge on this PR would fold it INTO '${BASE_REF}' and strand it off dev" + echo "::error::(feedback_stacked_prs_orphan_from_dev — the #1185/#1954 incident class)." + echo "::error::Either retarget to dev, or declare an intentional stack by adding a body line:" + echo "::error:: Stacked-Parent: #" + exit 1 diff --git a/.github/workflows/reusable-runtime-boot.yml b/.github/workflows/reusable-runtime-boot.yml index 63f2fb0292..14c963be72 100644 --- a/.github/workflows/reusable-runtime-boot.yml +++ b/.github/workflows/reusable-runtime-boot.yml @@ -287,13 +287,52 @@ jobs: echo "E2E_REDPANDA_ADVERTISE_HOST=${E2E_REDPANDA_ADVERTISE_HOST}" } >> "$GITHUB_ENV" - POSTGRES_PORT="${POSTGRES_PORT}" \ - KAFKA_PORT="${KAFKA_PORT}" \ - REDPANDA_ADMIN_PORT="${REDPANDA_ADMIN_PORT}" \ - REDPANDA_SCHEMA_REGISTRY_PORT="${REDPANDA_SCHEMA_REGISTRY_PORT}" \ - REDPANDA_PANDAPROXY_PORT="${REDPANDA_PANDAPROXY_PORT}" \ - E2E_REDPANDA_ADVERTISE_HOST="${E2E_REDPANDA_ADVERTISE_HOST}" \ - "${compose_cmd[@]}" -p "${OMNIBASE_INFRA_COMPOSE_PROJECT}" -f docker/docker-compose.e2e.yml up -d postgres redpanda + compose_up() { + POSTGRES_PORT="${POSTGRES_PORT}" \ + KAFKA_PORT="${KAFKA_PORT}" \ + REDPANDA_ADMIN_PORT="${REDPANDA_ADMIN_PORT}" \ + REDPANDA_SCHEMA_REGISTRY_PORT="${REDPANDA_SCHEMA_REGISTRY_PORT}" \ + REDPANDA_PANDAPROXY_PORT="${REDPANDA_PANDAPROXY_PORT}" \ + E2E_REDPANDA_ADVERTISE_HOST="${E2E_REDPANDA_ADVERTISE_HOST}" \ + "${compose_cmd[@]}" -p "${OMNIBASE_INFRA_COMPOSE_PROJECT}" -f docker/docker-compose.e2e.yml up -d postgres redpanda + } + + retry_compose_down() { + "${compose_cmd[@]}" -p "${OMNIBASE_INFRA_COMPOSE_PROJECT}" -f docker/docker-compose.e2e.yml down -v --remove-orphans || true + docker network rm "${OMNIBASE_INFRA_NETWORK}" "${OMNIBASE_INFRA_COMPOSE_PROJECT}_default" 2>/dev/null || true + } + + compose_started=false + for compose_attempt in 1 2 3; do + if compose_up; then + compose_started=true + break + fi + compose_status=$? + if [[ "${compose_attempt}" -eq 3 ]]; then + exit "${compose_status}" + fi + echo "::warning::compose infra bring-up failed on attempt ${compose_attempt}; repicking host ports and retrying" + retry_compose_down + export POSTGRES_PORT="$(pick_free_port)" + export KAFKA_PORT="$(pick_free_port)" + export REDPANDA_ADMIN_PORT="$(pick_free_port)" + export REDPANDA_SCHEMA_REGISTRY_PORT="$(pick_free_port)" + export REDPANDA_PANDAPROXY_PORT="$(pick_free_port)" + { + echo "POSTGRES_PORT=${POSTGRES_PORT}" + echo "PG_PORT=${POSTGRES_PORT}" + echo "KAFKA_PORT=${KAFKA_PORT}" + echo "REDPANDA_ADMIN_PORT=${REDPANDA_ADMIN_PORT}" + echo "REDPANDA_SCHEMA_REGISTRY_PORT=${REDPANDA_SCHEMA_REGISTRY_PORT}" + echo "REDPANDA_PANDAPROXY_PORT=${REDPANDA_PANDAPROXY_PORT}" + } >> "$GITHUB_ENV" + done + + if [[ "${compose_started}" != "true" ]]; then + echo "::error::compose infra bring-up failed before health checks" + exit 1 + fi capture_compose_infra_diagnostics() { { diff --git a/.github/workflows/validator-runtime-profiles.yml b/.github/workflows/validator-runtime-profiles.yml new file mode 100644 index 0000000000..b42cf4a1e5 --- /dev/null +++ b/.github/workflows/validator-runtime-profiles.yml @@ -0,0 +1,82 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +--- +# Runtime Profiles — ValidatorRuntimeProfiles gate [OMN-12957] +# +# Wires omnibase_core.validation.validator_runtime_profiles as a required +# status check on omnibase_infra contracts. Enforces three rules: +# - runtime_profiles_missing (OMN-9886) +# - runtime_profile_unregistered (OMN-12957) — declared profile not in the +# canonical REGISTERED_RUNTIME_PROFILES registry +# - runtime_profile_no_consumer_lane (OMN-12957) — subscribing REDUCER/EFFECT +# names no consumer-attached profile, so its subscriptions are never drained +# +# Pre-existing violators are frozen in +# config/validation/runtime_profiles_allowlist.yaml (drain tracked by OMN-12982); the +# workflow passes that repo-local path explicitly. The gate blocks NEW orphans. +# +# Required status check context: "Runtime Profiles / validate". + +name: Runtime Profiles + +on: + push: + branches: [main, develop, dev, "hotfix/**"] + pull_request: + branches: [main, develop, dev] + merge_group: + branches: [main, develop, dev] + +env: + PYTHON_VERSION: "3.12" + UV_VERSION: "0.8.3" + +concurrency: + group: runtime-profiles-${{ github.ref }} + cancel-in-progress: true + +jobs: + validate: + name: validate + # OMNI_RUNNER_SELECTOR_V1 - trusted events default to self-hosted omnibase-ci; pull_request defaults to ubuntu-latest + runs-on: >- + ${{ + github.event_name != 'pull_request' + && fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') + || fromJSON(vars.OMNI_PUBLIC_PR_RUNS_ON_JSON || '["ubuntu-latest"]') + }} + timeout-minutes: 10 + + steps: + - name: Checkout omnibase_infra + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: ${{ env.UV_VERSION }} + + - name: Install dependencies + run: uv sync --all-extras + + - name: Validate runtime_profiles on omnibase_infra contracts + run: | + uv run python - <<'PY' + from pathlib import Path + + from omnibase_core.validation.validator_runtime_profiles import ( + ValidatorRuntimeProfiles, + ) + + result = ValidatorRuntimeProfiles( + allowlist_path=Path("config/validation/runtime_profiles_allowlist.yaml") + ).validate(Path("src")) + for issue in result.issues: + print(f"[{issue.severity.value}] {issue.file_path}:{issue.line_number}: {issue.message}") + raise SystemExit(0 if result.is_valid else 1) + PY diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f2862358dc..c9f4833521 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,11 +21,18 @@ # pre-commit install --hook-type pre-push # Install pre-push hooks # pre-commit run --all-files # Run all pre-commit hooks repos: - # Stub implementation detection — sourced from omnibase_core (OMN-4473) + # Stub implementation detection + transport-mock lint — sourced from omnibase_core. + # Rev bumped to OMN-13026 (transport-mock-lint hook added). - repo: https://github.com/OmniNode-ai/omnibase_core - rev: 30a06f85bb5849902d33c8d0bdb90fc6cba0c346 # pragma: allowlist secret + rev: cfaa6ec863c3026b5e65196151ed76cdad13dae3 # pragma: allowlist secret hooks: - id: check-stub-implementations + # OMN-13026: bare AsyncMock/MagicMock without spec=/spec_set= on + # EventBus/transport/kafka surfaces. Ratchet: existing violations frozen + # in config/validation/transport_mock_baseline.yaml (218 violations, 63 files). + # New violations blocked. Drain tracked by OMN-13026 per-site tickets. + - id: transport-mock-lint + args: [--baseline, config/validation/transport_mock_baseline.yaml] # Hardcoded topic guard — sourced from onex_change_control (OMN-5256/OMN-5259) # Pre-existing violations excluded pending migration to canonical constants (OMN-5259) - repo: https://github.com/OmniNode-ai/onex_change_control @@ -106,6 +113,16 @@ repos: files: ^src/omnibase_infra/nodes/[^/]+/handlers/.*\.py$ pass_filenames: false stages: [pre-commit] + # OMN-12965: enforce runtime image identity stamping (version + revision). + # A blank/placeholder identity degrades every proof packet. Guards both the + # Dockerfile workspace-mode identity guard and the cmd_up build-arg wiring. + - id: check-runtime-image-identity + name: Enforce runtime image identity labels (OMN-12965) + entry: uv run --frozen python scripts/check_runtime_image_identity.py + language: system + files: ^(docker/Dockerfile\.runtime|src/omnibase_infra/docker/catalog/cli\.py)$ + pass_filenames: false + stages: [pre-commit] # OMN-11069: Block new os.environ/os.getenv reads outside approved overlay modules - id: check-env-reads name: Block new os.environ reads (use overlay-resolved config) @@ -132,14 +149,22 @@ repos: pass_filenames: false always_run: false stages: [pre-commit] - # OMN-12559: vendored omnimarket node migrations must stay in sync with - # source. Skips silently when omnimarket source is not checked out. + # OMN-12559 / OMN-13124: vendored omnimarket node migrations must stay in + # sync with source. MUST always_run — the prior files:-scoped trigger only + # fired when a file under docker/migrations/forward/nodes/ (or the sync + # script) was staged. A node migration added in the *omnimarket* repo + # (e.g. node_projection_pattern_learning, OMN-13124) never touches those + # infra paths, so the hook silently passed and the un-vendored migration + # merged — the table was missing in omnidash_analytics on a clean redeploy + # (golden-chain run12 Finding). always_run closes that gap. The script + # itself returns exit 2 (gate fires) when the omnimarket source is + # unresolvable unless SYNC_NODE_MIGRATIONS_SKIP_UNRESOLVABLE=1. - id: onex-check-node-migration-sync name: ONEX Node Migration Vendor Sync Check entry: bash scripts/sync-node-migrations.sh --check language: system pass_filenames: false - files: ^(docker/migrations/forward/nodes/|scripts/sync-node-migrations\.sh) + always_run: true stages: [pre-commit] # Reject any staged .env file at any path depth (OMN-2476) # Uses (^|/)\.env([._-].+)?$ to block root/.env, config/.env, .env.production, .env.staging, @@ -196,6 +221,22 @@ repos: files: '\.yaml$' pass_filenames: false stages: [pre-commit] + # OMN-12957: runtime_profiles contract validator (omnibase_core). + # Enforces three rules on every node contract.yaml: + # - runtime_profiles_missing (OMN-9886) + # - runtime_profile_unregistered (declared profile not in the canonical + # omnibase_core REGISTERED_RUNTIME_PROFILES registry) + # - runtime_profile_no_consumer_lane (subscribing REDUCER/EFFECT names + # no consumer-attached profile, so subscriptions are never drained) + # Pre-existing violators are frozen in validation/runtime_profiles_allowlist.yaml + # (discovered by repo-root walk; drain tracked by OMN-12982). + - id: onex-validate-runtime-profiles + name: Validate runtime_profiles on node contracts (OMN-12957) + entry: uv run --frozen python -m omnibase_core.validation.validator_runtime_profiles src/ + language: system + files: 'contract\.yaml$' + pass_filenames: false + stages: [pre-commit] # Pattern validation - naming conventions - id: onex-validate-patterns name: ONEX Pattern Validation diff --git a/config/validation/runtime_profiles_allowlist.yaml b/config/validation/runtime_profiles_allowlist.yaml new file mode 100644 index 0000000000..b77ebbc385 --- /dev/null +++ b/config/validation/runtime_profiles_allowlist.yaml @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# OMN-12957 baseline allowlist: freezes pre-existing runtime_profiles violators +# so the gate blocks NEW orphans without forcing a mass remediation on demo day. +# Drain tracked by OMN-12982 — fix each node's lane and delete its entry here. +# ValidatorRuntimeProfiles receives this repo-local path explicitly from the +# Runtime Profiles workflow; keep it under config/ so clean-root stays green. +allowlist: + - node_id: node_artifact_change_detector_effect + reason: OMN-12957 baseline freeze; remediate via OMN-12982 (assign correct runtime lane or document runtime-exempt) + - node_id: node_baselines_batch_compute + reason: OMN-12957 baseline freeze; remediate via OMN-12982 (assign correct runtime lane or document runtime-exempt) + - node_id: node_build_loop_write_effect + reason: OMN-12957 baseline freeze; remediate via OMN-12982 (assign correct runtime lane or document runtime-exempt) + - node_id: node_chain_orchestrator + reason: OMN-12957 baseline freeze; remediate via OMN-12982 (assign correct runtime lane or document runtime-exempt) + - node_id: node_chain_retrieval_effect + reason: OMN-12957 baseline freeze; remediate via OMN-12982 (assign correct runtime lane or document runtime-exempt) + - node_id: node_contract_resolver_bridge + reason: OMN-12957 baseline freeze; remediate via OMN-12982 (assign correct runtime lane or document runtime-exempt) + - node_id: node_ledger_projection_compute + reason: OMN-12957 baseline freeze; remediate via OMN-12982 (assign correct runtime lane or document runtime-exempt) + - node_id: node_ledger_write_effect + reason: OMN-12957 baseline freeze; remediate via OMN-12982 (assign correct runtime lane or document runtime-exempt) + - node_id: node_llm_completion_effect + reason: OMN-12957 baseline freeze; remediate via OMN-12982 (assign correct runtime lane or document runtime-exempt) + - node_id: node_llm_embedding_effect + reason: OMN-12957 baseline freeze; remediate via OMN-12982 (assign correct runtime lane or document runtime-exempt) + - node_id: node_merge_sweep_workflow_orchestrator + reason: OMN-12957 baseline freeze; remediate via OMN-12982 (assign correct runtime lane or document runtime-exempt) + - node_id: node_onboarding_orchestrator + reason: OMN-12957 baseline freeze; remediate via OMN-12982 (assign correct runtime lane or document runtime-exempt) + - node_id: node_registration_orchestrator + reason: OMN-12957 baseline freeze; remediate via OMN-12982 (assign correct runtime lane or document runtime-exempt) + - node_id: node_routing_orchestrator + reason: OMN-12957 baseline freeze; remediate via OMN-12982 (assign correct runtime lane or document runtime-exempt) + - node_id: node_rsd_orchestrator + reason: OMN-12957 baseline freeze; remediate via OMN-12982 (assign correct runtime lane or document runtime-exempt) + - node_id: node_runtime_orchestrator + reason: OMN-12957 baseline freeze; remediate via OMN-12982 (assign correct runtime lane or document runtime-exempt) + - node_id: node_scope_workflow_orchestrator + reason: OMN-12957 baseline freeze; remediate via OMN-12982 (assign correct runtime lane or document runtime-exempt) + - node_id: node_topic_migration_executor_effect + reason: OMN-12957 baseline freeze; remediate via OMN-12982 (assign correct runtime lane or document runtime-exempt) + - node_id: node_vector_store_effect + reason: OMN-12957 baseline freeze; remediate via OMN-12982 (assign correct runtime lane or document runtime-exempt) diff --git a/config/validation/transport_mock_baseline.yaml b/config/validation/transport_mock_baseline.yaml new file mode 100644 index 0000000000..8dc501d606 --- /dev/null +++ b/config/validation/transport_mock_baseline.yaml @@ -0,0 +1,63 @@ +tests/integration/handlers/test_handler_no_publish_constraint.py: 1 +tests/integration/nodes/test_handler_mandatory_di_params_integration.py: 1 +tests/integration/nodes/test_node_baseline_capture_effect.py: 2 +tests/integration/nodes/test_node_reward_binder_effect.py: 1 +tests/integration/nodes/test_plugin_wire_dispatchers_single_authority.py: 1 +tests/integration/runtime/test_async_container_pre_resolution_integration.py: 2 +tests/integration/runtime/test_auto_wiring_deferred_subscription_order.py: 1 +tests/integration/runtime/test_event_bus_contract_wiring.py: 18 +tests/integration/runtime/test_freshness_monitor_integration.py: 2 +tests/integration/runtime/test_handler_wiring_async_incompat_quarantine_integration.py: 4 +tests/integration/runtime/test_plugin_managed_subscription_integration.py: 3 +tests/integration/test_auto_wiring_async_incompat_quarantine_integration.py: 1 +tests/integration/test_auto_wiring_strict_invariant.py: 3 +tests/integration/test_kafka_contract_source_materialize_integration.py: 6 +tests/integration/test_llm_gpu_usage_evidence_integration.py: 1 +tests/integration/test_runtime_crash_loop_fix.py: 2 +tests/integration/test_runtime_health_monitor_live_event_bus.py: 1 +tests/integration/test_runtime_host_plugin_managed_gate.py: 2 +tests/services/session/test_consumer.py: 4 +tests/unit/adapters/test_scoped_adapters.py: 1 +tests/unit/event_bus/test_readiness.py: 4 +tests/unit/mixins/test_ack_listener_loop.py: 12 +tests/unit/nodes/node_baseline_capture/test_handler_baseline_capture.py: 4 +tests/unit/nodes/node_contract_registry_reducer/test_contract_registration_event_router.py: 1 +tests/unit/nodes/node_llm_inference_effect/services/test_service_llm_metrics_publisher.py: 2 +tests/unit/nodes/node_registration_orchestrator/test_plugin_wire_dispatchers_deferred.py: 1 +tests/unit/nodes/node_registration_orchestrator/test_topic_routing_e2e.py: 1 +tests/unit/nodes/test_handler_runtime_error_triage.py: 10 +tests/unit/nodes/test_node_runtime_orchestrator.py: 2 +tests/unit/runtime/auto_wiring/test_handler_wiring_async_container_resolution.py: 1 +tests/unit/runtime/auto_wiring/test_handler_wiring_async_incompat_quarantine.py: 1 +tests/unit/runtime/auto_wiring/test_handler_wiring_handle_async_dispatch.py: 3 +tests/unit/runtime/auto_wiring/test_handler_wiring_inference_intent_failure.py: 1 +tests/unit/runtime/auto_wiring/test_plugin_managed_subscription.py: 6 +tests/unit/runtime/auto_wiring/test_raw_event_projection_wiring.py: 1 +tests/unit/runtime/auto_wiring/test_wiring_subscribe.py: 5 +tests/unit/runtime/health/test_health_published_events_map_wiring.py: 3 +tests/unit/runtime/test_contract_registration_event_router.py: 11 +tests/unit/runtime/test_dispatch_result_applier_topic_routing.py: 18 +tests/unit/runtime/test_domain_plugin_discovery_seam.py: 1 +tests/unit/runtime/test_event_bus_subcontract_wiring.py: 1 +tests/unit/runtime/test_freshness_monitor.py: 2 +tests/unit/runtime/test_handler_source_resolver.py: 2 +tests/unit/runtime/test_handler_wiring_resolver_integration.py: 2 +tests/unit/runtime/test_kernel.py: 1 +tests/unit/runtime/test_projection_ordering.py: 12 +tests/unit/runtime/test_publisher_topic_scoped.py: 1 +tests/unit/runtime/test_request_response_wiring.py: 1 +tests/unit/runtime/test_runtime_host_architecture_validation.py: 1 +tests/unit/runtime/test_runtime_idempotency_guard.py: 1 +tests/unit/runtime/test_runtime_scheduler.py: 1 +tests/unit/runtime/test_runtime_scheduler_valkey.py: 1 +tests/unit/runtime/test_service_dispatch_result_applier.py: 22 +tests/unit/runtime/test_transition_notification_outbox.py: 2 +tests/unit/runtime/test_transition_notification_publisher.py: 3 +tests/unit/runtime/test_util_wiring.py: 3 +tests/unit/services/contract_publisher/test_service.py: 3 +tests/unit/services/observability/baselines/test_service_batch_compute_baselines.py: 2 +tests/unit/services/registry_api/test_service_update_feature_flag.py: 2 +tests/unit/services/test_circuit_breaker_event_publisher.py: 2 +tests/unit/services/test_service_llm_endpoint_health.py: 1 +tests/unit/services/test_service_runtime_health_monitor.py: 6 +tests/unit/services/test_timeout_emitter.py: 1 diff --git a/contracts/OMN-12848.yaml b/contracts/OMN-12848.yaml index 9895a82c0b..19684b5dc2 100644 --- a/contracts/OMN-12848.yaml +++ b/contracts/OMN-12848.yaml @@ -1,25 +1,14 @@ ---- schema_version: '1.0.0' ticket_id: OMN-12848 -title: 'stability-test rebuild lost node-generation-requested dispatch route → DLQ + cachetools missing - in workspace image' +title: 'stability-test rebuild lost node-generation-requested dispatch route → DLQ + cachetools missing in workspace image' summary: >- - Fix two defects blocking the close-the-loop demo. (1) Routing regression: a sole handler entry that - declares an event_model and subscribes to more than one topic (node_generation_consumer subscribing - to node-generation-requested AND node-deploy after OMN-12826) fell through _topics_for_handler_entry - to return () and registered ZERO dispatch routes, so generation commands were consumed then DLQ'd ("No - dispatcher found for category 'command'"). A sole handler unambiguously owns every subscribe topic; - only competing multi-handler entries keep the ambiguity guard. (2) Workspace-mode image missing cachetools - (omnimemory transitive dep) → node_semantic_analyzer_compute / node_agent_learning_retrieval_effect - failed to load. Added cachetools to the runtime external-deps install + a build-time import smoke check - so a missing transitive dep fails the build instead of degrading at runtime. + Fix two defects blocking the close-the-loop demo. (1) Routing regression: a sole handler entry that declares an event_model and subscribes to more than one topic (node_generation_consumer subscribing to node-generation-requested AND node-deploy after OMN-12826) fell through _topics_for_handler_entry to return () and registered ZERO dispatch routes, so generation commands were consumed then DLQ'd ("No dispatcher found for category 'command'"). A sole handler unambiguously owns every subscribe topic; only competing multi-handler entries keep the ambiguity guard. (2) Workspace-mode image missing cachetools (omnimemory transitive dep) → node_semantic_analyzer_compute / node_agent_learning_retrieval_effect failed to load. Added cachetools to the runtime external-deps install + a build-time import smoke check so a missing transitive dep fails the build instead of degrading at runtime. is_seam_ticket: false interface_change: false interfaces_touched: [] evidence_requirements: - kind: tests - description: _topics_for_handler_entry assigns all topics to a sole handler entry (regression covered); - existing multi-handler ambiguity and per-handler event_type behavior preserved + description: _topics_for_handler_entry assigns all topics to a sole handler entry (regression covered); existing multi-handler ambiguity and per-handler event_type behavior preserved command: uv run pytest tests/unit/runtime/auto_wiring/test_topics_for_handler_entry.py -q - kind: ci description: CI pipeline green on PR @@ -31,17 +20,14 @@ emergency_bypass: dod_evidence: - id: dod-001 description: >- - A single-handler contract with an event_model and multiple subscribe topics gets all topics assigned - (dispatch route registered), fixing the node-generation-requested → DLQ regression. Multi-handler - ambiguity guard and per-handler event_type selection unchanged. + A single-handler contract with an event_model and multiple subscribe topics gets all topics assigned (dispatch route registered), fixing the node-generation-requested → DLQ regression. Multi-handler ambiguity guard and per-handler event_type selection unchanged. source: generated checks: - check_type: command check_value: 'uv run pytest tests/unit/runtime/auto_wiring/test_topics_for_handler_entry.py -q' - id: dod-002 description: >- - Full auto_wiring unit suite remains green after the topic-assignment fix (no regression in dispatcher/route - registration). + Full auto_wiring unit suite remains green after the topic-assignment fix (no regression in dispatcher/route registration). source: generated checks: - check_type: command diff --git a/contracts/OMN-12860.yaml b/contracts/OMN-12860.yaml index 31d415ff6f..d91608dea3 100644 --- a/contracts/OMN-12860.yaml +++ b/contracts/OMN-12860.yaml @@ -4,11 +4,7 @@ schema_version: "1.0.0" ticket_id: "OMN-12860" title: "Infra lane overlay renders logical secret resolver config" summary: >- - Extends runtime policy to declare logical LLM secret references and concrete - secret source descriptors, renders those policy values into runtime env, and - materializes a typed ModelSecretResolverConfig artifact at runtime startup. - This keeps Bifrost route authority on logical refs while allowing dev and - stability-test lanes to use environment variables as local secret sources. + Extends runtime policy to declare logical LLM secret references and concrete secret source descriptors, renders those policy values into runtime env, and materializes a typed ModelSecretResolverConfig artifact at runtime startup. This keeps Bifrost route authority on logical refs while allowing dev and stability-test lanes to use environment variables as local secret sources. is_seam_ticket: true interface_change: true interfaces_touched: @@ -18,10 +14,7 @@ evidence_requirements: - kind: "tests" description: "Secret resolver renderer and runtime policy contract tests pass" command: >- - uv run pytest tests/unit/runtime/test_render_secret_resolver_config.py - tests/unit/docker/test_runtime_entrypoint_volume_bootstrap.py - tests/ci/test_runtime_policy_contract.py tests/ci/test_runtime_env_anchor.py - tests/integration/runtime/test_runtime_policy_contract.py -q + uv run pytest tests/unit/runtime/test_render_secret_resolver_config.py tests/unit/docker/test_runtime_entrypoint_volume_bootstrap.py tests/ci/test_runtime_policy_contract.py tests/ci/test_runtime_env_anchor.py tests/integration/runtime/test_runtime_policy_contract.py -q - kind: "tests" description: "Stability-test compose render exposes contract-owned resolver config path" command: >- @@ -29,21 +22,11 @@ evidence_requirements: - kind: "ci" description: "Touched runtime policy and renderer files pass ruff" command: >- - uv run ruff check src/omnibase_infra/runtime/render_secret_resolver_config.py - src/omnibase_infra/runtime/models/model_runtime_profile_policy.py - scripts/render_runtime_policy_env.py - tests/unit/runtime/test_render_secret_resolver_config.py - tests/unit/docker/test_runtime_entrypoint_volume_bootstrap.py - tests/integration/infra/test_stability_test_runtime_compose_render.py - tests/ci/test_runtime_policy_contract.py tests/ci/test_runtime_env_anchor.py - tests/integration/runtime/test_runtime_policy_contract.py + uv run ruff check src/omnibase_infra/runtime/render_secret_resolver_config.py src/omnibase_infra/runtime/models/model_runtime_profile_policy.py scripts/render_runtime_policy_env.py tests/unit/runtime/test_render_secret_resolver_config.py tests/unit/docker/test_runtime_entrypoint_volume_bootstrap.py tests/integration/infra/test_stability_test_runtime_compose_render.py tests/ci/test_runtime_policy_contract.py tests/ci/test_runtime_env_anchor.py tests/integration/runtime/test_runtime_policy_contract.py - kind: "manual" description: "Post-merge dev/stability deploy captures runtime identity and resolver artifact evidence" command: >- - deploy dev/stability runtime, then record runtime SHA, image digest, - runtime-policy hash, overlay hash, ONEX_SECRET_RESOLVER_CONFIG_PATH, and - logical llm.openrouter.api_key resolution through SecretResolver without - logging secret values + deploy dev/stability runtime, then record runtime SHA, image digest, runtime-policy hash, overlay hash, ONEX_SECRET_RESOLVER_CONFIG_PATH, and logical llm.openrouter.api_key resolution through SecretResolver without logging secret values emergency_bypass: enabled: false justification: "" @@ -55,10 +38,7 @@ dod_evidence: checks: - check_type: "command" check_value: >- - uv run pytest tests/unit/runtime/test_render_secret_resolver_config.py - tests/unit/docker/test_runtime_entrypoint_volume_bootstrap.py - tests/ci/test_runtime_policy_contract.py tests/ci/test_runtime_env_anchor.py - tests/integration/runtime/test_runtime_policy_contract.py -q + uv run pytest tests/unit/runtime/test_render_secret_resolver_config.py tests/unit/docker/test_runtime_entrypoint_volume_bootstrap.py tests/ci/test_runtime_policy_contract.py tests/ci/test_runtime_env_anchor.py tests/integration/runtime/test_runtime_policy_contract.py -q - id: "dod-stability-compose" description: "Stability-test compose render consumes contract-owned resolver config" source: "manual" @@ -72,14 +52,11 @@ dod_evidence: checks: - check_type: "command" check_value: >- - uv run python scripts/render_runtime_policy_env.py --check-env-file - docker/runtime-policy.env + uv run python scripts/render_runtime_policy_env.py --check-env-file docker/runtime-policy.env - id: "dod-deploy" description: "Runtime deploy evidence is required before declaring dev/stability fixed" source: "manual" checks: - check_type: "command" check_value: >- - deploy dev/stability runtime after merge; capture runtime SHA, image - digest, contract hash, overlay hash, config path, and logical - OpenRouter secret-ref resolution evidence + deploy dev/stability runtime after merge; capture runtime SHA, image digest, contract hash, overlay hash, config path, and logical OpenRouter secret-ref resolution evidence diff --git a/contracts/OMN-12940.yaml b/contracts/OMN-12940.yaml index 988be85459..6470f9f2ff 100644 --- a/contracts/OMN-12940.yaml +++ b/contracts/OMN-12940.yaml @@ -4,17 +4,7 @@ schema_version: "1.0.0" ticket_id: "OMN-12940" title: "Recursively unwrap double-wrapped transport envelopes in auto-wiring dispatch" summary: >- - The runtime auto-wiring dispatch layer constructs the contract input_model - itself via model_validate(_extract_dispatch_payload(envelope)) before the - handler runs. _extract_dispatch_payload unwrapped exactly one level, so a - double-wrapped delivery stayed an envelope and model_validate raised - "N validation errors ... Field required" — bypassing the omnimarket - handler-side coerce/unwrap fixes (OMN-12935/12936) entirely. A second site, - the post-handler correlation read in _normalize_handler_result, called bare - UUID() on a non-hex candidate and crashed dispatch. This makes the unwrap - recursive/envelope-aware and routes the correlation read through the existing - _coerce_uuid_or_none guard, unblocking runtime verification of the evidence - pipeline and readiness gate. + The runtime auto-wiring dispatch layer constructs the contract input_model itself via model_validate(_extract_dispatch_payload(envelope)) before the handler runs. _extract_dispatch_payload unwrapped exactly one level, so a double-wrapped delivery stayed an envelope and model_validate raised "N validation errors ... Field required" — bypassing the omnimarket handler-side coerce/unwrap fixes (OMN-12935/12936) entirely. A second site, the post-handler correlation read in _normalize_handler_result, called bare UUID() on a non-hex candidate and crashed dispatch. This makes the unwrap recursive/envelope-aware and routes the correlation read through the existing _coerce_uuid_or_none guard, unblocking runtime verification of the evidence pipeline and readiness gate. is_seam_ticket: false interface_change: false interfaces_touched: [] @@ -22,8 +12,7 @@ evidence_requirements: - kind: "tests" description: "Double-wrapped envelope unwrap + UUID-guard unit tests pass" command: >- - uv run pytest - tests/unit/runtime/auto_wiring/test_handler_wiring_double_wrapped_envelope.py -q + uv run pytest tests/unit/runtime/auto_wiring/test_handler_wiring_double_wrapped_envelope.py -q - kind: "tests" description: "Full auto-wiring unit suite passes (no regression in dispatch path)" command: >- @@ -31,14 +20,10 @@ evidence_requirements: - kind: "ci" description: "Touched files pass ruff and mypy --strict" command: >- - uv run ruff check - src/omnibase_infra/runtime/auto_wiring/handler_wiring.py - tests/unit/runtime/auto_wiring/test_handler_wiring_double_wrapped_envelope.py - && uv run mypy src/omnibase_infra/runtime/auto_wiring/handler_wiring.py --strict + uv run ruff check src/omnibase_infra/runtime/auto_wiring/handler_wiring.py tests/unit/runtime/auto_wiring/test_handler_wiring_double_wrapped_envelope.py && uv run mypy src/omnibase_infra/runtime/auto_wiring/handler_wiring.py --strict - kind: "manual" description: >- - Post-merge stability redeploy re-runs the evidence-pipeline verification - leg and captures a fresh projection row / correlation-ID-backed traversal + Post-merge stability redeploy re-runs the evidence-pipeline verification leg and captures a fresh projection row / correlation-ID-backed traversal emergency_bypass: enabled: false justification: "" @@ -50,8 +35,7 @@ dod_evidence: checks: - check_type: "command" check_value: >- - uv run pytest - tests/unit/runtime/auto_wiring/test_handler_wiring_double_wrapped_envelope.py -q + uv run pytest tests/unit/runtime/auto_wiring/test_handler_wiring_double_wrapped_envelope.py -q - id: "dod-auto-wiring-suite" description: "Full auto-wiring unit suite green (no dispatch-path regression)" source: "manual" @@ -72,7 +56,4 @@ dod_evidence: checks: - check_type: "command" check_value: >- - rebuild stability runtime images from omnibase_infra@dev including this - fix; thin-publish to onex.cmd.omnimarket.evidence-pipeline-start.v1; - assert no ValidationError on dispatcher.auto.* and a fresh projection - row materializes keyed by the correlation id + rebuild stability runtime images from omnibase_infra@dev including this fix; thin-publish to onex.cmd.omnimarket.evidence-pipeline-start.v1; assert no ValidationError on dispatcher.auto.* and a fresh projection row materializes keyed by the correlation id diff --git a/contracts/OMN-13008.yaml b/contracts/OMN-13008.yaml new file mode 100644 index 0000000000..a8266f7f01 --- /dev/null +++ b/contracts/OMN-13008.yaml @@ -0,0 +1,43 @@ +schema_version: '1.0.0' +ticket_id: OMN-13008 +title: 'Automated worktree GC + .201 docker/disk GC + disk watermark alert ratchet' +summary: 'Conservative, keep-list-driven .201 disk maintenance: merged-worktree GC (drives prune-worktrees.sh), docker/image GC, and a df watermark alert ratchet (>85% ticket, >90% bus event), shipped as systemd user timer units plus dry-run-tested GC scripts.' +is_seam_ticket: false +interface_change: false +interfaces_touched: [] +evidence_requirements: + - kind: tests + description: GC plan-safety + dry-run + watermark-event unit tests pass + command: uv run pytest tests/unit/scripts/test_disk_gc_plan.py tests/unit/scripts/test_disk_gc_dry_run.py tests/unit/scripts/test_disk_watermark_event.py -q + - kind: ci + description: CI pipeline green on PR + command: gh pr checks --repo OmniNode-ai/omnibase_infra +emergency_bypass: + enabled: false + justification: '' + follow_up_ticket_id: '' +dod_evidence: + - id: dod-001 + description: Removal-plan resolver never reaps a keep-list repo, kept tag, in-use image, or too-young artifact (pure-planner safety tests) + source: generated + checks: + - check_type: command + check_value: 'uv run pytest tests/unit/scripts/test_disk_gc_plan.py -q' + - id: dod-002 + description: GC shell scripts in default/dry-run mode issue no destructive docker/git/rpk op; --execute correctly forwards the flag + source: generated + checks: + - check_type: command + check_value: 'uv run pytest tests/unit/scripts/test_disk_gc_dry_run.py -q' + - id: dod-003 + description: Disk-watermark bus event builder produces stable typed warning/critical events with a dedupe alert_key + source: generated + checks: + - check_type: command + check_value: 'uv run pytest tests/unit/scripts/test_disk_watermark_event.py -q' + - id: dod-deploy + description: systemd USER timer units (NOT lane containers) installed + enabled on .201 via deploy/disk-gc/install-disk-gc.sh; verified with systemctl --user list-timers + source: generated + checks: + - check_type: command + check_value: 'uv run pytest tests/unit/scripts/test_disk_gc_plan.py tests/unit/scripts/test_disk_gc_dry_run.py -q' diff --git a/contracts/OMN-13011.yaml b/contracts/OMN-13011.yaml new file mode 100644 index 0000000000..cdf155077e --- /dev/null +++ b/contracts/OMN-13011.yaml @@ -0,0 +1,89 @@ +# OMN-13011 contract file for omnibase_infra deploy-gate. +# The canonical ticket contract and PASS receipts live in onex_change_control. +schema_version: "1.0.0" +ticket_id: "OMN-13011" +title: "LANE CENSUS RECONCILIATION ratchet — declared desired-state per lane, scheduled + on-demand, drift = auto-ticket" +summary: >- + The class fix for the recurring lane-drift regression. Nothing reconciled the + declared desired state of a runtime lane against what is actually running, so + the same failure kept recurring with zero signal: volume config drift + (OMN-12945), WORKER_REPLICAS silent zero (OMN-12988/12990), and on 2026-06-11 + prod runtime containers plus the broker network were silently absent for hours + during demo prep. This ships a per-lane DESIRED-STATE census: (a) DECLARED in a + versioned lane manifest (deploy/lane-census/lane-manifest.yaml) — container set, + network, replicas, image-tag pattern per lane (stability-test/prod/judge/dev), + derived from the canonical compose lane files; (b) RECONCILED on a schedule on + .201 by sharing the OMN-13008 systemd timer (a drop-in 4th ExecStart on + onex-disk-gc.service, never a second timer) and on-demand via runtime_sweep; + (c) drift = a typed bus event + Linear auto-ticket naming exactly what is + missing/extra (container_absent, network_detached, replicas_zero, + unexpected_container, oneshot_failed/stuck, image_tag_mismatch). Fail-fast, + no warn-only mode (gates-block policy). Builds on the OMN-12988 deploy-agent + RUNTIME census (deploy-time) as the complementary steady-state reconciler. +is_seam_ticket: false +interface_change: false +interfaces_touched: [] +evidence_requirements: + - kind: "tests" + description: "Pure-planner drift tests incl. the 2026-06-11 prod red fixture pass" + command: >- + uv run pytest tests/unit/scripts/test_lane_census_plan.py -q + - kind: "tests" + description: "Event-builder, manifest-parity, dry-run, and timer-dropin tests pass" + command: >- + uv run pytest + tests/unit/scripts/test_lane_census_event.py + tests/unit/scripts/test_lane_census_manifest_parity.py + tests/unit/scripts/test_lane_census_dry_run.py + tests/unit/scripts/test_lane_census_timer_dropin.py -q + - kind: "ci" + description: "New source passes ruff and mypy --strict" + command: >- + uv run ruff check scripts/lane_census_plan.py scripts/lane_census_event.py + && uv run mypy scripts/lane_census_plan.py scripts/lane_census_event.py --strict + - kind: "manual" + description: >- + On .201, after OMN-13008's install-disk-gc.sh, run + deploy/lane-census/install-lane-census.sh and confirm via + `systemctl --user cat onex-disk-gc.service` that the lane-census ExecStart + is the 4th pass on the shared unit (no second timer) +emergency_bypass: + enabled: false + justification: "" + follow_up_ticket_id: "" +dod_evidence: + - id: "dod-planner-red-fixture" + description: "Pure planner reports network_detached + container_absent for the 2026-06-11 prod outage" + source: "manual" + checks: + - check_type: "command" + check_value: >- + uv run pytest tests/unit/scripts/test_lane_census_plan.py -q + - id: "dod-event-and-ticket" + description: "Typed drift event builds stable alert_key + ticket title/body naming exactly what is wrong" + source: "manual" + checks: + - check_type: "command" + check_value: >- + uv run pytest tests/unit/scripts/test_lane_census_event.py -q + - id: "dod-manifest-parity" + description: "Lane manifest is locked in step with the compose lane files; no service declares replicas 0" + source: "manual" + checks: + - check_type: "command" + check_value: >- + uv run pytest tests/unit/scripts/test_lane_census_manifest_parity.py -q + - id: "dod-dry-run-failfast" + description: "Dry-run never publishes; drift exits 30; no broker => no localhost-default publish" + source: "manual" + checks: + - check_type: "command" + check_value: >- + uv run pytest tests/unit/scripts/test_lane_census_dry_run.py -q + - id: "dod-shared-timer" + description: "Lane-census shares the OMN-13008 onex-disk-gc.service via drop-in; no second timer" + source: "manual" + checks: + - check_type: "command" + check_value: >- + uv run pytest tests/unit/scripts/test_lane_census_timer_dropin.py -q diff --git a/contracts/OMN-13141.yaml b/contracts/OMN-13141.yaml new file mode 100644 index 0000000000..14707c4fe8 --- /dev/null +++ b/contracts/OMN-13141.yaml @@ -0,0 +1,62 @@ +# OMN-13141 contract file for omnibase_infra deploy-gate. +# The canonical ticket contract and PASS receipts live in onex_change_control. +# This file is the per-repo deploy-evidence carrier only. +schema_version: "1.0.0" +ticket_id: "OMN-13141" +title: "fix(OMN-13141): runtime executor skips event_model import for operation_match entries" +summary: >- + Executor companion to OMN-13137's validator half (#1973, merged). _run_event_driven + unconditionally imported entry.event_model_module for every resolved routing entry. + operation_match entries declare no event_model, so event_model_module is empty and + importlib.import_module("") raised ValueError: Empty module name — not caught by the + (ImportError, AttributeError) clause, swallowed by the outer except in run_async -> + result FAILED. The node never booted end-to-end (e.g. node_integration_sweep_orchestrator). + Fix gates the import on a non-empty event_model_module: operation_match forwards the raw + decoded payload (input_model_cls=None); only payload_type_match resolves a typed event + model. _validate_routing (the OMN-13137 half) is left untouched. +is_seam_ticket: false +interface_change: false +interfaces_touched: [] +evidence_requirements: + - kind: "tests" + description: "operation_match boot e2e test passes; payload_type_match regression guard holds" + command: "uv run pytest tests/unit/runtime/test_run_event_driven_operation_match.py -q" + - kind: "ci" + description: "CI gates pass on omnibase_infra PR" + command: "gh pr checks --repo OmniNode-ai/omnibase_infra" +emergency_bypass: + enabled: false + justification: "" + follow_up_ticket_id: "" +dod_evidence: + - id: "dod-contract-artifact" + description: "Repo-local deploy evidence contract is present for OMN-13141" + source: "manual" + checks: + - check_type: "command" + check_value: "test -f contracts/OMN-13141.yaml" + - id: "dod-boot-e2e" + description: >- + operation_match contract boots end-to-end through RuntimeLocal.run_async() + with no empty-module ValueError, and the payload_type_match import path is + unchanged (regression guard). This boot test is the CI gate for the fix. + source: "manual" + checks: + - check_type: "command" + check_value: "uv run pytest tests/unit/runtime/test_run_event_driven_operation_match.py -q" + - id: "dod-runtime-suite" + description: "Full runtime unit suite passes — no regression in the event-driven executor or adapter." + source: "manual" + checks: + - check_type: "command" + check_value: "uv run pytest tests/unit/runtime/ -q" + - id: "dod-deploy" + description: >- + Deploy validation: change is an internal correctness fix to the local-runtime + event-driven executor (_run_event_driven) and its bus adapter. No new Kafka topics, + no schema changes, no container image changes, no API surface changes. Runtime + restart picks up the change automatically on next deploy. + source: "manual" + checks: + - check_type: "command" + check_value: "echo 'deploy-gate: OMN-13141 is an internal runtime-executor correctness fix; no new topics, schema, or container changes; runtime restart picks it up on next deploy'" diff --git a/contracts/OMN-13149.yaml b/contracts/OMN-13149.yaml new file mode 100644 index 0000000000..5102ebe6cc --- /dev/null +++ b/contracts/OMN-13149.yaml @@ -0,0 +1,66 @@ +# OMN-13149 contract file for omnibase_infra deploy-gate. +# The canonical ticket contract and PASS receipts live in onex_change_control. +# This file is the per-repo deploy-evidence carrier only. +schema_version: "1.0.0" +ticket_id: "OMN-13149" +title: "fix(OMN-13149): savings_estimation consumer decodes typed ModelEventMessage instead of calling .get()" +summary: >- + The savings-estimation consumer is wired in service_kernel via event_bus.subscribe; + both the Kafka and in-memory buses deliver a typed ModelEventMessage to the + on_message callback. The pre-fix _savings_on_message did json.loads only when msg + was str/bytes and otherwise passed the value through unchanged, so a ModelEventMessage + reached ServiceSavingsEstimator.ingest_event -> _session_id_for_topic, which calls + payload.get("session_id"). ModelEventMessage has no .get(), raising + AttributeError: 'ModelEventMessage' object has no attribute 'get' at runtime — the + consumer wrote zero rows. Fix adds module-level decode_event_message(message) that + reads the JSON payload off the typed .value field directly (strongly typed, fail-fast: + raises TypeError on a non-object payload — no getattr-default, no dict-style fallback). + The kernel callback now decodes then calls ingest_event(topic, payload). Same + dict/typed-model family as OMN-13141 / golden-chain BUG A. +is_seam_ticket: false +interface_change: false +interfaces_touched: [] +evidence_requirements: + - kind: "tests" + description: "Typed-message decode correlates a session through the real bus; pre-fix path reproduces AttributeError" + command: "uv run pytest tests/unit/services/observability/savings_estimation/test_consumer.py -q" + - kind: "ci" + description: "CI gates pass on omnibase_infra PR" + command: "gh pr checks --repo OmniNode-ai/omnibase_infra" +emergency_bypass: + enabled: false + justification: "" + follow_up_ticket_id: "" +dod_evidence: + - id: "dod-contract-artifact" + description: "Repo-local deploy evidence contract is present for OMN-13149" + source: "manual" + checks: + - check_type: "command" + check_value: "test -f contracts/OMN-13149.yaml" + - id: "dod-typed-message-regression" + description: >- + Through the REAL consumer path (EventBusInmemory.subscribe/publish -> + on_message(ModelEventMessage) -> decode_event_message -> ingest_event) a typed + ModelEventMessage correlates a session; the pre-fix path (typed message straight + to ingest_event) reproduces AttributeError. This is the CI gate for the fix. + source: "manual" + checks: + - check_type: "command" + check_value: "uv run pytest tests/unit/services/observability/savings_estimation/test_consumer.py -q" + - id: "dod-runtime-suite" + description: "Runtime + event-bus + savings unit suites pass — no regression in kernel wiring or estimator." + source: "manual" + checks: + - check_type: "command" + check_value: "uv run pytest tests/unit/runtime/ tests/unit/services/observability/savings_estimation/ -q" + - id: "dod-deploy" + description: >- + Deploy validation: change is an internal correctness fix to the savings-estimation + consumer wiring in service_kernel (_savings_on_message) plus a module-level decode + helper in the consumer. No new Kafka topics, no schema changes, no container image + changes, no API surface changes. Runtime restart picks up the change on next deploy. + source: "manual" + checks: + - check_type: "command" + check_value: "echo 'deploy-gate: OMN-13149 is an internal savings-consumer wiring correctness fix; no new topics, schema, or container changes; runtime restart picks it up on next deploy'" diff --git a/contracts/services/runtime_policy.contract.yaml b/contracts/services/runtime_policy.contract.yaml index 1e6c069d7e..1a57b0ea7b 100644 --- a/contracts/services/runtime_policy.contract.yaml +++ b/contracts/services/runtime_policy.contract.yaml @@ -6,6 +6,13 @@ active_runtime_packages: llm_cloud_endpoint_host_allowlist: - generativelanguage.googleapis.com - api.z.ai + # OMN-12971: Vertex AI OpenAI-compatible endpoint host. The region prefix is + # part of the host (e.g. us-central1-aiplatform.googleapis.com); the suffix + # below allows region-prefixed Vertex hosts for the ADC ($500-credit) path. + - aiplatform.googleapis.com +bifrost_vertex_gemini_endpoint_url: https://us-central1-aiplatform.googleapis.com/v1beta1/projects/gen-lang-client-0084338881/locations/us-central1/endpoints/openapi/chat/completions +google_cloud_project: gen-lang-client-0084338881 +google_cloud_location: us-central1 omnimemory_memgraph_port: 7687 auxiliary_services_omnimemory_enabled: false profiles: @@ -28,6 +35,19 @@ profiles: source: source_type: env source_path: GEMINI_API_KEY + # OMN-12971: Vertex bearer-token ref. ADDITIVE to the Gemini API-key path. + # The logical ref MUST match the cloud-vertex-gemini backend's secret_ref in + # omnimarket bifrost_delegation.yaml (llm.vertex.access_token). The effect + # boundary posts Authorization: Bearer , so the resolved VALUE is a + # short-lived OAuth access token (NOT the raw ADC JSON). source_type=file → + # the ref resolves to the mounted token FILE; the operator/bootstrap mints + # the token from ADC (`gcloud auth application-default print-access-token`) + # and refreshes the file (tokens expire ~1h). The token VALUE is never + # committed — only the ref name and the in-container path live here. + - logical_name: llm.vertex.access_token + source: + source_type: file + source_path: /run/secrets/vertex_access_token services: main: runtime_id: dev-main @@ -54,6 +74,8 @@ profiles: worker: runtime_id: dev-worker runtime_address: runtime://omninode-pc/dev/worker + # OMN-12990: explicit worker replica pin in the ledgered config surface. + replicas: 1 capabilities: - workflow.dispatch - contract.update @@ -81,6 +103,19 @@ profiles: source: source_type: env source_path: GEMINI_API_KEY + # OMN-12971: Vertex bearer-token ref. ADDITIVE to the Gemini API-key path. + # The logical ref MUST match the cloud-vertex-gemini backend's secret_ref in + # omnimarket bifrost_delegation.yaml (llm.vertex.access_token). The effect + # boundary posts Authorization: Bearer , so the resolved VALUE is a + # short-lived OAuth access token (NOT the raw ADC JSON). source_type=file → + # the ref resolves to the mounted token FILE; the operator/bootstrap mints + # the token from ADC (`gcloud auth application-default print-access-token`) + # and refreshes the file (tokens expire ~1h). The token VALUE is never + # committed — only the ref name and the in-container path live here. + - logical_name: llm.vertex.access_token + source: + source_type: file + source_path: /run/secrets/vertex_access_token services: main: runtime_id: stability-test-main @@ -107,6 +142,8 @@ profiles: worker: runtime_id: stability-test-worker runtime_address: runtime://omninode-pc/stability-test/worker + # OMN-12990: explicit worker replica pin in the ledgered config surface. + replicas: 1 capabilities: - workflow.dispatch - contract.update @@ -134,6 +171,19 @@ profiles: source: source_type: env source_path: GEMINI_API_KEY + # OMN-12971: Vertex bearer-token ref. ADDITIVE to the Gemini API-key path. + # The logical ref MUST match the cloud-vertex-gemini backend's secret_ref in + # omnimarket bifrost_delegation.yaml (llm.vertex.access_token). The effect + # boundary posts Authorization: Bearer , so the resolved VALUE is a + # short-lived OAuth access token (NOT the raw ADC JSON). source_type=file → + # the ref resolves to the mounted token FILE; the operator/bootstrap mints + # the token from ADC (`gcloud auth application-default print-access-token`) + # and refreshes the file (tokens expire ~1h). The token VALUE is never + # committed — only the ref name and the in-container path live here. + - logical_name: llm.vertex.access_token + source: + source_type: file + source_path: /run/secrets/vertex_access_token services: main: runtime_id: judge-main @@ -160,6 +210,8 @@ profiles: worker: runtime_id: judge-worker runtime_address: runtime://judge-local/judge/worker + # OMN-12990: explicit worker replica pin in the ledgered config surface. + replicas: 1 capabilities: - workflow.dispatch - contract.update @@ -199,6 +251,8 @@ profiles: worker: runtime_id: prod-worker runtime_address: runtime://omninode-pc/prod/worker + # OMN-12990: explicit worker replica pin in the ledgered config surface. + replicas: 1 capabilities: - workflow.dispatch - contract.update diff --git a/deploy/disk-gc/install-disk-gc.sh b/deploy/disk-gc/install-disk-gc.sh new file mode 100755 index 0000000000..ed0411875b --- /dev/null +++ b/deploy/disk-gc/install-disk-gc.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +# install-disk-gc.sh — Install the .201 disk-maintenance systemd USER timer (OMN-13008). +# +# These are systemd USER units (NOT lane containers). Installing/enabling them is +# scoped to the runtime user and does not touch any docker-compose lane. No sudo. +# +# Usage (run on 192.168.86.201 after pulling latest): +# bash deploy/disk-gc/install-disk-gc.sh # install + enable + start timer +# bash deploy/disk-gc/install-disk-gc.sh --uninstall +# bash deploy/disk-gc/install-disk-gc.sh --status +# +# Prerequisites: +# - systemd user manager available (loginctl enable-linger $USER if running headless) +# - omnibase_infra cloned at $OMNI_HOME/omnibase_infra (default ~/Code/omni_home) +# - omniclaude cloned alongside (for prune-worktrees.sh) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +SERVICE_SRC="${SCRIPT_DIR}/onex-disk-gc.service" +TIMER_SRC="${SCRIPT_DIR}/onex-disk-gc.timer" +USER_UNIT_DIR="${HOME}/.config/systemd/user" +SERVICE_DST="${USER_UNIT_DIR}/onex-disk-gc.service" +TIMER_DST="${USER_UNIT_DIR}/onex-disk-gc.timer" + +if [[ "${1:-}" == "--uninstall" ]]; then + echo "Uninstalling onex-disk-gc user timer..." + systemctl --user stop onex-disk-gc.timer onex-disk-gc.service 2>/dev/null || true + systemctl --user disable onex-disk-gc.timer 2>/dev/null || true + rm -f "$SERVICE_DST" "$TIMER_DST" + systemctl --user daemon-reload + echo "Done. onex-disk-gc uninstalled." + exit 0 +fi + +if [[ "${1:-}" == "--status" ]]; then + systemctl --user list-timers onex-disk-gc.timer --no-pager || true + echo "" + systemctl --user status onex-disk-gc.timer onex-disk-gc.service --no-pager || true + echo "" + echo "Recent logs:" + journalctl --user -u onex-disk-gc.service -n 30 --no-pager || true + exit 0 +fi + +echo "Installing onex-disk-gc systemd USER timer..." + +# Make the GC scripts executable. +chmod +x "${SCRIPT_DIR}/../../scripts/disk-gc.sh" \ + "${SCRIPT_DIR}/../../scripts/disk-watermark-check.sh" \ + "${SCRIPT_DIR}/../../scripts/worktree-gc.sh" 2>/dev/null || true + +mkdir -p "$USER_UNIT_DIR" +cp "$SERVICE_SRC" "$SERVICE_DST" +cp "$TIMER_SRC" "$TIMER_DST" + +systemctl --user daemon-reload +systemctl --user enable --now onex-disk-gc.timer + +echo "" +echo "Done. onex-disk-gc installed and running (user scope)." +echo " Service: ${SERVICE_DST}" +echo " Timer: ${TIMER_DST}" +echo "" +echo "Check: systemctl --user list-timers onex-disk-gc.timer" +echo "Logs: journalctl --user -u onex-disk-gc.service -f" +echo "Run now: systemctl --user start onex-disk-gc.service" +echo "Uninstall: bash deploy/disk-gc/install-disk-gc.sh --uninstall" +echo "" +echo "NOTE: if this host runs headless, enable lingering so the timer fires" +echo "without an active login session: loginctl enable-linger \$USER" diff --git a/deploy/disk-gc/keep-list.yaml b/deploy/disk-gc/keep-list.yaml new file mode 100644 index 0000000000..4dea5e51c7 --- /dev/null +++ b/deploy/disk-gc/keep-list.yaml @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# keep-list.yaml — Versioned protect-list for the .201 conservative docker/disk GC. +# +# OMN-13008. This file is the single source of truth for what disk-gc.sh must NOT +# remove. It is read at runtime by disk-gc.sh; nothing is hardcoded in the script. +# +# How matching works: +# - keep_image_repos: any docker image whose repository name contains one of +# these substrings is NEVER removed, regardless of age. +# Use this for the live runtime image family and any +# image you may need to roll back to. +# - keep_image_tags: any image whose ":tag" matches one of these (exact tag +# match) is NEVER removed. Use for tagged rollback sets. +# - protect_running: when true (default), images referenced by any container +# (running OR stopped) are always kept even if not listed +# above. This is the backstop that prevents pulling the +# rug out from a lane mid-flight. +# +# Maintenance: when promoting a new runtime image generation, add its tag to +# keep_image_tags so the GC keeps the prior generation as a rollback target for +# `superseded_image_keep_generations` cycles. Do NOT hardcode tags in the script. +schema_version: "1.0.0" +# Image repositories (substring match) that are always retained. +keep_image_repos: + - omninode-runtime + - omnibase-infra + - omnibase-intelligence + - omnimarket + - omnidash + - projection-api + # Core infra base images — removing these forces slow re-pulls and can wedge a + # lane restart while /data is already tight. + - postgres + - redpandadata/redpanda + - valkey +# Exact image tags (the part after ':') that are always retained — tagged +# rollback sets. Update on each promotion. +keep_image_tags: + - latest + - stable + - rollback +# Keep images referenced by any container (running or stopped). The live lane +# backstop. Disable only with explicit operator approval. +protect_running: true +# How many superseded (untagged-by-keep_image_tags) generations of a kept repo +# to retain before the oldest become eligible for removal once older than +# superseded_image_max_age_days. Belt-and-suspenders with age. +superseded_image_keep_generations: 2 +# Age threshold (days) below which NOTHING is removed, even dangling images. +# Conservative: a just-built image is never reaped. +min_age_days: 3 diff --git a/deploy/disk-gc/onex-disk-gc.service b/deploy/disk-gc/onex-disk-gc.service new file mode 100644 index 0000000000..6da7535c59 --- /dev/null +++ b/deploy/disk-gc/onex-disk-gc.service @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# onex-disk-gc.service — .201 disk maintenance pass (OMN-13008). +# +# A systemd USER unit (NOT a lane container). One-shot maintenance run, triggered +# by onex-disk-gc.timer. Runs three conservative passes in order: +# 1. worktree-gc.sh — remove merged+clean worktrees (drives prune-worktrees.sh) +# 2. disk-gc.sh — conservative docker/builder/image GC honoring keep-list +# 3. disk-watermark-check.sh — emit bus alert if /data crosses the watermark +# +# Install (on .201, as the runtime user — NO sudo, user scope): +# bash deploy/disk-gc/install-disk-gc.sh +# +# Check: +# systemctl --user status onex-disk-gc.timer +# journalctl --user -u onex-disk-gc.service -f + +[Unit] +Description=ONEX .201 disk maintenance (worktree GC + docker GC + watermark alert) +Documentation=https://github.com/OmniNode-ai/omnibase_infra +After=docker.service + +[Service] +Type=oneshot + +# Resolve OMNI_HOME for the runtime user on .201. +Environment="OMNI_HOME=%h/Code/omni_home" +Environment="PATH=/usr/local/bin:/usr/bin:/bin" + +# KAFKA_BOOTSTRAP_SERVERS for the watermark bus publish is sourced from the +# operator env file (not hardcoded in source). Optional ('-' prefix): if the +# file is absent the watermark check still runs but logs the event instead of +# publishing (fail-soft on the alert publish, never a silent default broker). +EnvironmentFile=-%h/.omnibase/.env + +ExecStartPre=/bin/mkdir -p %h/.local/log/onex + +# 1. Merged-worktree GC (drives the canonical prune-worktrees.sh safety logic). +ExecStart=/bin/bash %h/Code/omni_home/omnibase_infra/scripts/worktree-gc.sh --execute --worktrees-root /data/omninode/omni_worktrees + +# 2. Conservative docker/disk GC honoring the versioned keep-list. +ExecStart=/bin/bash %h/Code/omni_home/omnibase_infra/scripts/disk-gc.sh --execute + +# 3. Watermark alert ratchet (non-zero exit on breach is expected; do not fail the unit). +ExecStart=-/bin/bash %h/Code/omni_home/omnibase_infra/scripts/disk-watermark-check.sh --mount /data + +StandardOutput=journal +StandardError=journal +SyslogIdentifier=onex-disk-gc + +[Install] +WantedBy=default.target diff --git a/deploy/disk-gc/onex-disk-gc.timer b/deploy/disk-gc/onex-disk-gc.timer new file mode 100644 index 0000000000..3dd8db2d67 --- /dev/null +++ b/deploy/disk-gc/onex-disk-gc.timer @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# onex-disk-gc.timer — triggers the .201 disk maintenance pass (OMN-13008). +# +# Systemd USER timer. The watermark check runs hourly so /data alerts fire well +# before detonation; the GC passes are conservative (keep-list + min_age_days) so +# running hourly is safe and self-throttling. +# +# Install on .201 (user scope, no sudo): +# bash deploy/disk-gc/install-disk-gc.sh +# +# Check: +# systemctl --user list-timers onex-disk-gc.timer +# systemctl --user status onex-disk-gc.timer + +[Unit] +Description=ONEX .201 disk maintenance timer (worktree + docker GC + watermark) +Documentation=https://github.com/OmniNode-ai/omnibase_infra + +[Timer] +# First run 5 min after the user manager starts (let docker settle). +OnStartupSec=5min +# Then every hour on a calendar schedule. Calendar recurrence re-arms reliably for +# a oneshot service (OnUnitActiveSec does not always re-elapse once the unit goes +# inactive). Watermark alerts stay timely; GC is idempotent + conservative. +OnCalendar=hourly +# Spread load a little so it never lines up with other top-of-hour jobs. +RandomizedDelaySec=2min +# Fire a missed run on resume (host sleep, reboots). +Persistent=true +Unit=onex-disk-gc.service + +[Install] +WantedBy=timers.target diff --git a/deploy/lane-census/install-lane-census.sh b/deploy/lane-census/install-lane-census.sh new file mode 100755 index 0000000000..5caabd40d9 --- /dev/null +++ b/deploy/lane-census/install-lane-census.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +# install-lane-census.sh — Register the lane-census reconcile on the SHARED +# onex-disk-gc.service systemd USER timer (OMN-13011). +# +# COORDINATION (OMN-13008 / PR #1952): the disk-GC PR owns onex-disk-gc.service + +# onex-disk-gc.timer. This installer does NOT create a second timer — it drops a +# systemd override (onex-disk-gc.service.d/20-lane-census.conf) that appends the +# lane-census ExecStart to that one unit, then reloads. The reconcile runs on the +# same hourly tick as the GC/watermark passes. +# +# Prerequisite: OMN-13008's `bash deploy/disk-gc/install-disk-gc.sh` must have run +# first (it installs the base onex-disk-gc.service/.timer). If the base unit is +# absent this installer fails fast and tells you to run the disk-gc installer. +# +# Usage (on 192.168.86.201, user scope, no sudo): +# bash deploy/lane-census/install-lane-census.sh # install drop-in + reload +# bash deploy/lane-census/install-lane-census.sh --uninstall +# bash deploy/lane-census/install-lane-census.sh --status + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +DROPIN_SRC="${SCRIPT_DIR}/onex-disk-gc.service.d/20-lane-census.conf" +USER_UNIT_DIR="${HOME}/.config/systemd/user" +BASE_SERVICE="${USER_UNIT_DIR}/onex-disk-gc.service" +DROPIN_DST_DIR="${USER_UNIT_DIR}/onex-disk-gc.service.d" +DROPIN_DST="${DROPIN_DST_DIR}/20-lane-census.conf" + +if [[ "${1:-}" == "--uninstall" ]]; then + echo "Removing lane-census drop-in..." + rm -f "$DROPIN_DST" + systemctl --user daemon-reload 2>/dev/null || true + echo "Done. lane-census drop-in removed (base onex-disk-gc unit untouched)." + exit 0 +fi + +if [[ "${1:-}" == "--status" ]]; then + echo "Effective onex-disk-gc.service (base + drop-ins):" + systemctl --user cat onex-disk-gc.service --no-pager 2>/dev/null || \ + echo " (base unit not installed — run deploy/disk-gc/install-disk-gc.sh first)" + exit 0 +fi + +if [[ ! -f "$BASE_SERVICE" ]]; then + echo "ERROR: base onex-disk-gc.service not installed at $BASE_SERVICE" >&2 + echo "Run OMN-13008's installer first: bash deploy/disk-gc/install-disk-gc.sh" >&2 + exit 1 +fi + +echo "Installing lane-census drop-in onto the shared onex-disk-gc.service..." +chmod +x "${SCRIPT_DIR}/../../scripts/lane-census-check.sh" 2>/dev/null || true +mkdir -p "$DROPIN_DST_DIR" +cp "$DROPIN_SRC" "$DROPIN_DST" +systemctl --user daemon-reload + +echo "" +echo "Done. lane-census reconcile registered on the shared onex-disk-gc timer." +echo " Drop-in: $DROPIN_DST" +echo "" +echo "Verify: systemctl --user cat onex-disk-gc.service # base + lane-census ExecStart" +echo "Run now: systemctl --user start onex-disk-gc.service" +echo "Logs: journalctl --user -u onex-disk-gc.service -f" +echo "On-demand: bash scripts/lane-census-check.sh --json" diff --git a/deploy/lane-census/lane-manifest.yaml b/deploy/lane-census/lane-manifest.yaml new file mode 100644 index 0000000000..5919e34ec6 --- /dev/null +++ b/deploy/lane-census/lane-manifest.yaml @@ -0,0 +1,182 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# lane-manifest.yaml — Versioned DESIRED-STATE census for the .201 runtime lanes (OMN-13011). +# +# THE CLASS FIX. We kept regressing the same way — nothing reconciled desired vs +# actual lane state: +# - volume config drift (OMN-12945 family) +# - WORKER_REPLICAS silent zero (OMN-12988 / OMN-12990) +# - 2026-06-11: prod runtime containers + broker network silently absent for +# hours during demo prep, with ZERO alert. +# +# This file is the single source of truth for what each lane MUST look like in +# steady state. scripts/lane_census_plan.py reads it and diffs it against the +# live `docker ps` / `docker network ls` inventory; scripts/lane-census-check.sh +# drives the reconcile, emits a typed bus event, and fails fast (non-zero) on any +# drift. Nothing about lane shape is hardcoded in the scripts — it all lives here. +# +# Maintenance: the container_name and network values below are derived from the +# canonical compose lane files (docker/docker-compose..yml `container_name` +# fields and the `networks:` block). When a lane gains/loses a container or +# renames its network, update BOTH the compose file and this manifest in the same +# PR. tests/unit/scripts/test_lane_census_manifest_parity.py asserts the manifest +# stays in lock-step with the compose files so the manifest can never silently +# drift away from the lane it is supposed to police. +# +# Container kinds: +# service — long-running; MUST be in `docker ps` Running state. Absence = drift. +# oneshot — run-to-completion migration/init container; expected to have Exited +# (0). It is NOT drift for a oneshot to be absent (compose removes it) +# or Exited; it IS drift for a oneshot to be Running-stuck or Exited +# non-zero. Tracked, not required-running. +# +# replicas: the REQUIRED running count for a service container. 1 unless a lane +# explicitly scales out. A service whose replicas resolve to 0 is itself drift +# (the WORKER_REPLICAS silent-zero regression): declaring replicas: 0 here is +# forbidden by the parity test. +# +# image_tag_pattern: a regex every running container's image tag must match for +# the lane. Guards against a lane silently running a stale/wrong generation. The +# special token "${RUNTIME_TAG}" is resolved at check time from the deploy-agent +# resolved runtime version; absence of a resolved tag relaxes the check to the +# default pattern (never a hard failure on an unresolvable tag — that is a +# separate signal, not a census drift). + +schema_version: "1.0.0" + +# Default image-tag pattern applied to a lane that does not override it. +default_image_tag_pattern: '.+' + +lanes: + stability-test: + compose_file: docker/docker-compose.stability-test.yml + compose_project: omnibase-infra-stability-test + network: omnibase-infra-stability-test-network + image_tag_pattern: '.+' + services: + - name: omnibase-infra-stability-test-postgres + kind: service + replicas: 1 + - name: omnibase-infra-stability-test-redpanda + kind: service + replicas: 1 + - name: omnibase-infra-stability-test-redpanda-partition-cap + kind: oneshot + - name: omnibase-infra-stability-test-valkey + kind: service + replicas: 1 + - name: omnibase-infra-stability-test-keycloak + kind: service + replicas: 1 + - name: omnibase-infra-stability-test-forward-migration + kind: oneshot + - name: omnibase-infra-stability-test-migration-gate + kind: oneshot + - name: omnibase-infra-stability-test-intelligence-migration + kind: oneshot + - name: omninode-stability-test-runtime + kind: service + replicas: 1 + - name: omninode-stability-test-runtime-effects + kind: service + replicas: 1 + - name: omnimarket-stability-test-projection-api + kind: service + replicas: 1 + # The worker that OMN-12988 / OMN-12990 silently scaled to 0. Declared + # replicas: 1 here so its absence (replicas 0 => no container) is loud drift. + - name: omninode-stability-test-runtime-worker + kind: service + replicas: 1 + + prod: + compose_file: docker/docker-compose.prod.yml + compose_project: omnibase-infra-prod + network: omnibase-infra-prod-network + image_tag_pattern: '.+' + services: + - name: omnibase-infra-prod-postgres + kind: service + replicas: 1 + - name: omnibase-infra-prod-redpanda + kind: service + replicas: 1 + - name: omnibase-infra-prod-redpanda-partition-cap + kind: oneshot + - name: omnibase-infra-prod-valkey + kind: service + replicas: 1 + - name: omnibase-infra-prod-forward-migration + kind: oneshot + - name: omnibase-infra-prod-migration-gate + kind: oneshot + - name: omnibase-infra-prod-intelligence-migration + kind: oneshot + # The four runtime services that were silently absent on 2026-06-11. + - name: omninode-prod-runtime + kind: service + replicas: 1 + - name: omninode-prod-runtime-effects + kind: service + replicas: 1 + - name: omnimarket-prod-projection-api + kind: service + replicas: 1 + - name: omninode-prod-runtime-worker + kind: service + replicas: 1 + + judge: + compose_file: docker/docker-compose.judge.yml + compose_project: omnibase-infra-judge + network: omnibase-infra-judge-network + image_tag_pattern: '.+' + services: + - name: omnibase-infra-judge-postgres + kind: service + replicas: 1 + - name: omnibase-infra-judge-redpanda + kind: service + replicas: 1 + - name: omnibase-infra-judge-redpanda-partition-cap + kind: oneshot + - name: omnibase-infra-judge-valkey + kind: service + replicas: 1 + - name: omnibase-infra-judge-forward-migration + kind: oneshot + - name: omnibase-infra-judge-migration-gate + kind: oneshot + - name: omnibase-infra-judge-intelligence-migration + kind: oneshot + - name: omninode-judge-runtime + kind: service + replicas: 1 + - name: omninode-judge-runtime-effects + kind: service + replicas: 1 + - name: omnimarket-judge-projection-api + kind: service + replicas: 1 + + # The dev lane (compose project `omnibase-infra`, the default infra stack) uses + # generated compose with non-prefixed container names; its census is the + # deploy-agent RUNTIME-scope verifier (OMN-12988). The lane manifest treats dev + # as OPTIONAL (reconcile-when-present): dev is a developer lane that is legitimately + # down most of the time, so its absence is not a steady-state regression. When dev + # IS up, the listed services must be running. This avoids a noisy false-positive + # ticket every time a developer stops their local stack. + dev: + compose_file: docker/docker-compose.generated.yml + compose_project: omnibase-infra + network: omnibase-infra_default + image_tag_pattern: '.+' + optional: true + services: + - name: omninode-runtime + kind: service + replicas: 1 + - name: omninode-runtime-effects + kind: service + replicas: 1 diff --git a/deploy/lane-census/onex-disk-gc.service.d/20-lane-census.conf b/deploy/lane-census/onex-disk-gc.service.d/20-lane-census.conf new file mode 100644 index 0000000000..fb0494d813 --- /dev/null +++ b/deploy/lane-census/onex-disk-gc.service.d/20-lane-census.conf @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# 20-lane-census.conf — systemd drop-in extending the SHARED onex-disk-gc.service +# with the lane-census reconcile pass (OMN-13011). +# +# COORDINATION (OMN-13008 / PR #1952): the disk-GC PR owns onex-disk-gc.service + +# onex-disk-gc.timer. Rather than install a SECOND timer (per the ticket: "share +# the timer unit rather than adding a second"), this drop-in appends a 4th +# ExecStart to that ONE unit. systemd runs drop-ins after the base unit's own +# ExecStart lines, so the lane-census reconcile runs after the GC passes on the +# same hourly tick. +# +# Why a drop-in instead of editing the base .service: the base unit lives in a +# separate in-flight PR. A drop-in lets both PRs land independently with zero +# merge collision on the unit file. When OMN-13008 merges, install-lane-census.sh +# copies this drop-in into ~/.config/systemd/user/onex-disk-gc.service.d/ and +# reloads — no second timer, no duplicate schedule. +# +# Install (on .201, user scope, after OMN-13008's install-disk-gc.sh has run): +# bash deploy/lane-census/install-lane-census.sh +# +# Check: +# systemctl --user cat onex-disk-gc.service # shows base + this drop-in +# journalctl --user -u onex-disk-gc.service -f + +[Service] +# Reconcile every declared lane against actual. Non-zero exit on drift is the +# expected maintenance signal (the watermark pass uses the same '-' fail-soft +# convention so one drift does not fail the whole maintenance unit). The drift +# event is published to the bus inside the script when KAFKA_BOOTSTRAP_SERVERS +# is present. +ExecStart=-/bin/bash %h/Code/omni_home/omnibase_infra/scripts/lane-census-check.sh diff --git a/docker/Dockerfile.runtime b/docker/Dockerfile.runtime index ba6de76b33..26538c697b 100644 --- a/docker/Dockerfile.runtime +++ b/docker/Dockerfile.runtime @@ -266,6 +266,16 @@ COPY workspace/sibling-repos/ /workspace/sibling-repos/ # Copy provenance helper (always present — used in workspace mode, no-op otherwise) COPY scripts/runtime_build/compute_workspace_provenance.py /workspace/compute_workspace_provenance.py +# Pin-resolution helper imported by the provenance script (OMN-12989): enforces +# that staged siblings honor the consuming repo's uv.lock pins. Must sit beside +# the provenance script so its sys.path-relative import resolves inside the image. +COPY scripts/runtime_build/resolve_workspace_pins.py /workspace/resolve_workspace_pins.py + +# Copy the sibling-pin comparison emitted by the preflight (OMN-12977). A +# committed placeholder makes this COPY always valid; workspace mode overwrites +# it via stage_workspace.sh so compute_workspace_provenance.py can fold the +# expected-vs-actual pin proof into build-provenance.json for deploy verifiers. +COPY workspace/sibling-pin-comparison.json /workspace/sibling-pin-comparison.json # workspace mode installs omnibase_compat, onex_change_control, omnimarket from # the staged local trees. release mode installs from pinned remote sources. @@ -320,7 +330,7 @@ RUN set -eu; \ if [ "${BUILD_SOURCE}" = "workspace" ]; then \ /app/.venv/bin/python /workspace/compute_workspace_provenance.py; \ else \ - printf '{\n "build_source": "release",\n "build_time": "%s",\n "vcs_ref": "%s",\n "proofs": []\n}\n' \ + printf '{\n "build_source": "release",\n "build_time": "%s",\n "vcs_ref": "%s",\n "proofs": [],\n "lock_pin_comparison": []\n}\n' \ "${BUILD_DATE:-unknown}" "${VCS_REF:-unknown}" > /app/build-provenance.json; \ fi @@ -434,6 +444,25 @@ RUN set -eu; \ exit 64; \ fi +# OMN-12965: workspace-mode builds must stamp a real image identity. A blank +# VCS_REF (-> blank org.opencontainers.image.revision) or the placeholder +# RUNTIME_VERSION default (0.1.0) degrades every proof packet, since the runtime +# SHA + image digest are required citations in accepted evidence. Fail the build +# here rather than ship a blank-identity image. Release-mode builds may rely on +# the placeholder default (operator-driven release tooling pins them explicitly), +# so the guard is scoped to workspace mode. +RUN set -eu; \ + if [ "${BUILD_SOURCE}" = "workspace" ]; then \ + if [ -z "${VCS_REF}" ] || [ "${VCS_REF}" = "unknown" ]; then \ + echo "workspace build requires a non-empty VCS_REF for org.opencontainers.image.revision (OMN-12965)" >&2; \ + exit 64; \ + fi; \ + if [ -z "${RUNTIME_VERSION}" ] || [ "${RUNTIME_VERSION}" = "0.1.0" ]; then \ + echo "workspace build requires RUNTIME_VERSION from pyproject (not placeholder 0.1.0) for org.opencontainers.image.version (OMN-12965)" >&2; \ + exit 64; \ + fi; \ + fi + # OCI Image Labels (https://github.com/opencontainers/image-spec/blob/main/annotations.md) LABEL org.opencontainers.image.title="OmniBase Infra Runtime" \ org.opencontainers.image.description="ONEX Infrastructure Runtime Service - Contract-driven kernel for event-based infrastructure orchestration" \ diff --git a/docker/catalog/services/forward-migration.yaml b/docker/catalog/services/forward-migration.yaml index 3e284d525f..6890612a27 100644 --- a/docker/catalog/services/forward-migration.yaml +++ b/docker/catalog/services/forward-migration.yaml @@ -2,7 +2,7 @@ name: forward-migration description: One-shot runner for pending forward SQL migrations image: postgres:16-alpine layer: infrastructure -container_name: omnibase-forward-migration +container_name: omnibase-infra-forward-migration required_env: - POSTGRES_PASSWORD hardcoded_env: diff --git a/docker/catalog/services/migration-gate.yaml b/docker/catalog/services/migration-gate.yaml index c1ba2ae142..2eddc251d5 100644 --- a/docker/catalog/services/migration-gate.yaml +++ b/docker/catalog/services/migration-gate.yaml @@ -14,12 +14,23 @@ hardcoded_env: operational_defaults: POSTGRES_USER: postgres ports: null +# start_period_s must cover the real forward+intelligence migration window. +# The sentinel is a long-running keepalive (`while true; sleep 3600`) with a +# continuous healthcheck against db_metadata.migrations_complete; it is NOT a +# one-shot. On a cold prod volume, forward-migration + intelligence-migration +# take ~2 min to apply and stamp the flag (OMN-12973: prod gate started +# 09:35:22, intelligence-migration finished 09:37:18). With start_period_s=10 +# the gate exited `starting` after 10s and was reported UNHEALTHY for ~2 min +# until migrations completed. start_period_s=180 keeps the gate in +# `health: starting` for the full migration window so a still-applying gate is +# never reported UNHEALTHY. The start_period floor is enforced by +# validate_migration_gate_start_period (OMN-12973). healthcheck: test: sh /check_migrations_complete.sh interval_s: 10 timeout_s: 5 retries: 30 - start_period_s: 10 + start_period_s: 180 volumes: - ../scripts/check_migrations_complete.sh:/check_migrations_complete.sh:ro depends_on: diff --git a/docker/docker-compose.infisical-stability.yml b/docker/docker-compose.infisical-stability.yml new file mode 100644 index 0000000000..41b2fabef7 --- /dev/null +++ b/docker/docker-compose.infisical-stability.yml @@ -0,0 +1,68 @@ +# Additive Infisical for the stability-test lane (OMN-12966 / P1.2b-A). +# +# WHY THIS FILE EXISTS +# -------------------- +# The Infisical service declared in docker-compose.infra.yml is disabled in the +# stability-test and prod lane overlays via `profiles: !override [...-disabled]`, +# so no Infisical instance is reachable from the runtime/effect containers — the +# container environment becomes the de-facto secret store. P1.2b ("config +# authority end-state") requires a real Infisical value store reachable from the +# runtime boundary. This compose project stands one up ADDITIVELY: +# +# * Own compose project name — does NOT modify or restart any lane container. +# * Joins the EXISTING stability-test network as `external`, so runtime/effect +# containers reach it in-network at `omnibase-infra-infisical-stability:8080` +# and operators reach it at host port 8880. +# * Reuses the already-present stability-test postgres (`infisical_db`) and +# valkey — no new stateful services. +# +# NO SECRETS IN THIS FILE. All credentials are resolved from the operator +# environment (~/.omnibase/.env), matching the discipline in +# docker-compose.infra.yml. The required vars are the same ones the canonical +# infisical service consumes; the only difference is that the connection URIs +# must point at the stability-test-namespaced service hostnames +# (omnibase-infra-stability-test-postgres / -valkey), supplied via env. +# +# Usage (on .201): +# cd /docker +# INFISICAL_STABILITY_DB_CONNECTION_URI=... \ +# INFISICAL_STABILITY_REDIS_URL=... \ +# INFISICAL_ENCRYPTION_KEY=... INFISICAL_AUTH_SECRET=... \ +# docker compose -f docker-compose.infisical-stability.yml up -d +# +# Then provision identity + folders + secrets: +# uv run python scripts/provision-infisical.py --addr http://192.168.86.201:8880 \ +# --org omninode --project onex-platform +# uv run python scripts/seed-infisical.py --import-env --set-values \ +# --create-missing-keys --execute +name: omnibase-infra-infisical-stability +services: + infisical: + image: infisical/infisical:v0.146.0-postgres + container_name: omnibase-infra-infisical-stability + environment: + # Must point at the stability-test-namespaced postgres/valkey hostnames. + DB_CONNECTION_URI: ${INFISICAL_STABILITY_DB_CONNECTION_URI:?INFISICAL_STABILITY_DB_CONNECTION_URI must be set (postgresql://...@omnibase-infra-stability-test-postgres:5432/infisical_db)} + REDIS_URL: ${INFISICAL_STABILITY_REDIS_URL:?INFISICAL_STABILITY_REDIS_URL must be set (redis://:...@omnibase-infra-stability-test-valkey:6379)} + ENCRYPTION_KEY: ${INFISICAL_ENCRYPTION_KEY:?INFISICAL_ENCRYPTION_KEY must be set in ~/.omnibase/.env} + AUTH_SECRET: ${INFISICAL_AUTH_SECRET:?INFISICAL_AUTH_SECRET must be set in ~/.omnibase/.env} + SITE_URL: ${INFISICAL_SITE_URL:-http://192.168.86.201:8880} + TELEMETRY_ENABLED: "false" + ports: + - "${INFISICAL_EXTERNAL_PORT:-8880}:8080" + networks: + - stability + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://localhost:8080/api/status"] + interval: 10s + timeout: 5s + retries: 12 + start_period: 90s + restart: unless-stopped + labels: + - "com.omninode.service=infisical" + - "com.omninode.lane=stability-test-additive" +networks: + stability: + name: omnibase-infra-stability-test-network + external: true diff --git a/docker/docker-compose.infra.yml b/docker/docker-compose.infra.yml index f6c28349ed..424340dc83 100644 --- a/docker/docker-compose.infra.yml +++ b/docker/docker-compose.infra.yml @@ -145,7 +145,15 @@ x-runtime-env: &runtime-env ONEX_CONTRACTS_DIR: /app/contracts ONEX_STATE_ROOT: /app/data/.onex_state ONEX_STATE_DIR: /app/data/.onex_state - BIFROST_CONTRACT_PATH: ${BIFROST_CONTRACT_PATH:-/app/data/delegation/bifrost_delegation.yaml} + # OMN-12864: Hardcoded default — do NOT use ${BIFROST_CONTRACT_PATH:-...} here. + # The :-/empty fallback was a footgun: if a host operator accidentally exported + # BIFROST_CONTRACT_PATH="" in their shell, it propagated to ALL services including + # the runtime containers that need bifrost rendering, silently disabling it and + # causing a 9→4 backend drop on the next restart. Hardcoding the path removes that + # ambient-env override path. Services that legitimately disable rendering + # (projection-api, contract-resolver) explicitly set BIFROST_CONTRACT_PATH: "" + # in their own environment blocks — those overrides remain intentional. + BIFROST_CONTRACT_PATH: /app/data/delegation/bifrost_delegation.yaml BIFROST_SOURCE_CONTRACT_PATH: ${BIFROST_SOURCE_CONTRACT_PATH:-} BIFROST_VERIFY_ENDPOINTS: ${BIFROST_VERIFY_ENDPOINTS:?runtime policy contract must set BIFROST_VERIFY_ENDPOINTS} ONEX_SECRET_RESOLVER_CONFIG_PATH: ${DEV_RUNTIME_MAIN_SECRET_RESOLVER_CONFIG_PATH:?runtime policy contract must set DEV_RUNTIME_MAIN_SECRET_RESOLVER_CONFIG_PATH} @@ -229,10 +237,16 @@ x-runtime-env: &runtime-env LLM_CODER_FAST_URL: ${LLM_CODER_FAST_URL:?LLM_CODER_FAST_URL required for PluginLlm activation} LLM_EMBEDDING_URL: ${LLM_EMBEDDING_URL:?LLM_EMBEDDING_URL required for PluginLlm activation} LLM_DEEPSEEK_R1_URL: ${LLM_DEEPSEEK_R1_URL:?LLM_DEEPSEEK_R1_URL required for PluginLlm activation} - BIFROST_LOCAL_CODER_ENDPOINT_URL: ${BIFROST_LOCAL_CODER_ENDPOINT_URL:?BIFROST_LOCAL_CODER_ENDPOINT_URL required for Bifrost local coder} - BIFROST_LOCAL_REASONER_ENDPOINT_URL: ${BIFROST_LOCAL_REASONER_ENDPOINT_URL:?BIFROST_LOCAL_REASONER_ENDPOINT_URL required for Bifrost local reasoner} - BIFROST_LOCAL_EMBEDDING_ENDPOINT_URL: ${BIFROST_LOCAL_EMBEDDING_ENDPOINT_URL:?BIFROST_LOCAL_EMBEDDING_ENDPOINT_URL required for Bifrost local embedding} - BIFROST_LOCAL_DS_V4_FLASH_ENDPOINT_URL: ${BIFROST_LOCAL_DS_V4_FLASH_ENDPOINT_URL:?BIFROST_LOCAL_DS_V4_FLASH_ENDPOINT_URL required for Bifrost local DS V4 Flash} + # OMN-12864: Bifrost local endpoints — committed overlay is the authority. + # Values come from docker/lane-overlays/dev.bifrost.env (service-level env_file + # on the three runtime services that need local inference). Validation at the + # Python layer (ModelBifrostLaneOverlay + render_bifrost_delegation_contract) is + # the enforcement point; compose-level :? is removed so compose config renders + # cleanly in CI without the env vars pre-set. + BIFROST_LOCAL_CODER_ENDPOINT_URL: ${BIFROST_LOCAL_CODER_ENDPOINT_URL:-} + BIFROST_LOCAL_REASONER_ENDPOINT_URL: ${BIFROST_LOCAL_REASONER_ENDPOINT_URL:-} + BIFROST_LOCAL_EMBEDDING_ENDPOINT_URL: ${BIFROST_LOCAL_EMBEDDING_ENDPOINT_URL:-} + BIFROST_LOCAL_DS_V4_FLASH_ENDPOINT_URL: ${BIFROST_LOCAL_DS_V4_FLASH_ENDPOINT_URL:-} # Cloud-tier GLM route refs are optional at boot but must be present in the # container env when routing authority selects api_key_ref=LLM_GLM_API_KEY. LLM_GLM_URL: ${LLM_GLM_URL:?LLM_GLM_URL is required for GLM escalation routing} @@ -243,6 +257,21 @@ x-runtime-env: &runtime-env # maps that ref to GEMINI_API_KEY, so runtime containers must receive the key. GEMINI_API_KEY: ${GEMINI_API_KEY:?GEMINI_API_KEY is required for Gemini routing} GOOGLE_API_KEY: ${GOOGLE_API_KEY:?GOOGLE_API_KEY is required for Gemini/Google ADK routing} + # Vertex bearer path (OMN-12971): ADDITIVE next to the Gemini API-key path. The + # routing authority declares secret_ref=llm.vertex.access_token for + # cloud-vertex-gemini; the secret resolver maps it to the mounted token FILE + # (source_type=file, /run/secrets/vertex_access_token — see + # runtime_policy.contract.yaml). The effect boundary posts Authorization: Bearer + # , so that file holds a short-lived OAuth access token minted from ADC + # (`gcloud auth application-default print-access-token`), refreshed by the + # operator/bootstrap (tokens expire ~1h). The token VALUE is never in committed + # config. These vars fail fast when the Vertex route is configured so a partial + # project/region/endpoint setup cannot silently disable the bearer path. + # BIFROST_VERTEX_GEMINI_ENDPOINT_URL is the COMPLETE project+region Vertex + # OpenAI-compat chat-completions URL, supplied by the overlay (OMN-12815). + BIFROST_VERTEX_GEMINI_ENDPOINT_URL: ${BIFROST_VERTEX_GEMINI_ENDPOINT_URL:?BIFROST_VERTEX_GEMINI_ENDPOINT_URL is required for Vertex Gemini routing} + GOOGLE_CLOUD_PROJECT: ${GOOGLE_CLOUD_PROJECT:?GOOGLE_CLOUD_PROJECT is required for Vertex Gemini routing} + GOOGLE_CLOUD_LOCATION: ${GOOGLE_CLOUD_LOCATION:?GOOGLE_CLOUD_LOCATION is required for Vertex Gemini routing} LLM_ENDPOINT_CIDR_ALLOWLIST: ${LLM_ENDPOINT_CIDR_ALLOWLIST:?LLM_ENDPOINT_CIDR_ALLOWLIST required for local LLM HTTP egress} LLM_CLOUD_ENDPOINT_HOST_ALLOWLIST: ${LLM_CLOUD_ENDPOINT_HOST_ALLOWLIST:?runtime policy contract must set LLM_CLOUD_ENDPOINT_HOST_ALLOWLIST} LOCAL_LLM_SHARED_SECRET: ${LOCAL_LLM_SHARED_SECRET:?LOCAL_LLM_SHARED_SECRET required for local LLM HTTP request signing} @@ -579,7 +608,14 @@ services: # successfully before checking the sentinel flag. forward-migration: image: postgres:16-alpine - container_name: omnibase-forward-migration + # Project-namespaced to match the omnibase-infra-* convention used by every + # other base service (postgres, redpanda, valkey, ...). The bare + # "omnibase-forward-migration" name was the one base container_name NOT + # carrying the lane prefix, so it collided across compose projects on the + # same host (e.g. a leftover dev-lane container blocked a stability rebuild). + # Lane overlays append their lane segment, yielding the derivable form + # -forward-migration that deploy-runtime.sh waits on (OMN-12987). + container_name: omnibase-infra-forward-migration profiles: ["runtime", "full"] depends_on: postgres: @@ -644,11 +680,22 @@ services: networks: - omnibase-infra-network healthcheck: + # start_period must cover the real forward+intelligence migration window. + # This sentinel is a long-running keepalive (entrypoint above), NOT a + # one-shot; the healthcheck runs continuously against + # db_metadata.migrations_complete. On a cold prod volume the migrations + # take ~2 min to apply and stamp the flag (OMN-12973: prod gate started + # 09:35:22, intelligence-migration finished 09:37:18). With start_period + # 10s the gate left `starting` after 10s and was reported UNHEALTHY for + # ~2 min until migrations completed. start_period 180s keeps the gate in + # `health: starting` for the full migration window so a still-applying + # gate is never reported UNHEALTHY. Floor enforced by + # validate_migration_gate_start_period (OMN-12973). test: ["CMD-SHELL", "sh /check_migrations_complete.sh"] interval: 10s timeout: 5s retries: 30 - start_period: 10s + start_period: 180s logging: *logging-defaults restart: unless-stopped deploy: @@ -671,6 +718,13 @@ services: !!merge <<: *runtime-base container_name: omninode-runtime profiles: ["runtime", "full"] + # OMN-12864: Service-level env_file injects committed BIFROST_LOCAL_* overlay. + # The overlay YAML source is docker/lane-overlays/dev.bifrost.yaml; the sidecar + # is generated by: uv run python scripts/render_bifrost_lane_overlay_env.py + # Do NOT override these vars with shell exports — the committed file is the authority. + env_file: + - path: lane-overlays/dev.bifrost.env + required: true depends_on: migration-gate: condition: service_healthy @@ -707,6 +761,16 @@ services: - runtime_data:/app/data - ../contracts:/app/contracts:ro - ${OMNICLAUDE_SKILLS_DIR:-./skills}:/app/skills:ro + # OMN-12971: Vertex bearer-token delivery. Bind the operator-supplied token + # file read-only at the path the secret resolver reads + # (llm.vertex.access_token → /run/secrets/vertex_access_token). The file + # holds a short-lived OAuth access token minted from ADC and refreshed by + # the operator/bootstrap; the effect boundary posts it as Authorization: + # Bearer . The token VALUE never enters committed config — only the + # host path is operator-supplied. Defaults to /dev/null so lanes without + # Vertex configured still start (resolver returns empty → router skips + # Vertex, Gemini key path unaffected). + - ${VERTEX_ACCESS_TOKEN_HOST_FILE:-/dev/null}:/run/secrets/vertex_access_token:ro tmpfs: - /run/onex-runtime:uid=1000,gid=1000,mode=0770 healthcheck: @@ -735,6 +799,10 @@ services: !!merge <<: *runtime-base container_name: omninode-runtime-effects profiles: ["runtime", "full"] + # OMN-12864: Service-level env_file injects committed BIFROST_LOCAL_* overlay. + env_file: + - path: lane-overlays/dev.bifrost.env + required: true depends_on: omninode-runtime: condition: service_healthy @@ -766,6 +834,10 @@ services: - effects_data:/app/data - ../contracts:/app/contracts:ro - ${OMNICLAUDE_SKILLS_DIR:-./skills}:/app/skills:ro + # OMN-12971: Vertex bearer-token delivery to the effects container (the + # runtime that makes the outbound provider call). Operator-supplied host + # token file bound read-only; token VALUE never in committed config. + - ${VERTEX_ACCESS_TOKEN_HOST_FILE:-/dev/null}:/run/secrets/vertex_access_token:ro healthcheck: test: ["CMD", "curl", "-sf", "http://localhost:8085/health"] interval: 30s @@ -828,6 +900,10 @@ services: runtime-worker: !!merge <<: *runtime-base profiles: ["runtime", "full"] + # OMN-12864: Service-level env_file injects committed BIFROST_LOCAL_* overlay. + env_file: + - path: lane-overlays/dev.bifrost.env + required: true depends_on: omninode-runtime: condition: service_healthy diff --git a/docker/docker-compose.judge.yml b/docker/docker-compose.judge.yml index 3c1700f1b9..39706562f5 100644 --- a/docker/docker-compose.judge.yml +++ b/docker/docker-compose.judge.yml @@ -14,7 +14,6 @@ # Running containers or mutating a remote host is outside this file's proof # boundary unless a cold-clone/cold-compose run is explicitly approved. name: omnibase-infra-judge - x-omninode-contract-overlay: judge-redpanda: advertised_kafka_addr: &judge_redpanda_advertised_kafka_addr "internal://redpanda:9092,external://localhost:49092" @@ -26,19 +25,16 @@ x-omninode-contract-overlay: optional_secret_refs: - OPEN_ROUTER_API_KEY - LLM_GLM_API_KEY - x-healthcheck-defaults: &healthcheck-defaults interval: 30s timeout: 10s retries: 3 start_period: 10s - x-logging-defaults: &logging-defaults driver: json-file options: max-size: "10m" max-file: "3" - x-runtime-base: &runtime-base build: context: .. @@ -59,7 +55,6 @@ x-runtime-base: &runtime-base reservations: cpus: "0.25" memory: 256M - x-judge-runtime-env: &judge-runtime-env POSTGRES_USER: ${POSTGRES_USER:-postgres} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD required} @@ -128,7 +123,6 @@ x-judge-runtime-env: &judge-runtime-env QDRANT_PORT: "6333" QDRANT_API_KEY: "" KAFKA_MAX_POLL_INTERVAL_MS: "1800000" - services: postgres: image: postgres:16-alpine @@ -146,11 +140,10 @@ services: networks: - omnibase-infra-judge-network healthcheck: - <<: *healthcheck-defaults + !!merge <<: *healthcheck-defaults test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres} -d omnibase_infra"] logging: *logging-defaults restart: unless-stopped - redpanda: image: redpandadata/redpanda:v24.2.7 container_name: omnibase-infra-judge-redpanda @@ -193,12 +186,11 @@ services: networks: - omnibase-infra-judge-network healthcheck: - <<: *healthcheck-defaults + !!merge <<: *healthcheck-defaults test: ["CMD-SHELL", "rpk cluster health | grep -q 'Healthy:.*true' || exit 1"] start_period: 30s logging: *logging-defaults restart: unless-stopped - redpanda-partition-cap: image: redpandadata/redpanda:v24.2.7 container_name: omnibase-infra-judge-redpanda-partition-cap @@ -215,7 +207,6 @@ services: networks: - omnibase-infra-judge-network restart: "no" - valkey: image: valkey/valkey:8.0-alpine container_name: omnibase-infra-judge-valkey @@ -231,11 +222,10 @@ services: networks: - omnibase-infra-judge-network healthcheck: - <<: *healthcheck-defaults + !!merge <<: *healthcheck-defaults test: ["CMD-SHELL", "valkey-cli -a \"$$VALKEY_PASSWORD\" --no-auth-warning ping | grep -q PONG"] logging: *logging-defaults restart: unless-stopped - forward-migration: image: postgres:16-alpine container_name: omnibase-infra-judge-forward-migration @@ -259,7 +249,6 @@ services: - omnibase-infra-judge-network restart: "no" logging: *logging-defaults - migration-gate: image: postgres:16-alpine container_name: omnibase-infra-judge-migration-gate @@ -290,7 +279,6 @@ services: start_period: 10s logging: *logging-defaults restart: unless-stopped - intelligence-migration: image: postgres:16-alpine container_name: omnibase-infra-judge-intelligence-migration @@ -312,9 +300,8 @@ services: - omnibase-infra-judge-network restart: "no" logging: *logging-defaults - omninode-runtime: - <<: *runtime-base + !!merge <<: *runtime-base container_name: omninode-judge-runtime depends_on: migration-gate: @@ -328,7 +315,7 @@ services: valkey: condition: service_healthy environment: - <<: *judge-runtime-env + !!merge <<: *judge-runtime-env RUNTIME_PROFILE: main OTEL_SERVICE_NAME: omninode-judge-runtime-main ONEX_RUNTIME_ID: judge-main @@ -361,9 +348,8 @@ services: - "com.omninode.ticket=OMN-12924" - "com.omninode.runtime.address=runtime://judge-local/judge/main" - "com.omninode.runtime.id=judge-main" - runtime-effects: - <<: *runtime-base + !!merge <<: *runtime-base container_name: omninode-judge-runtime-effects depends_on: omninode-runtime: @@ -371,7 +357,7 @@ services: redpanda: condition: service_healthy environment: - <<: *judge-runtime-env + !!merge <<: *judge-runtime-env RUNTIME_PROFILE: effects OTEL_SERVICE_NAME: omninode-judge-runtime-effects ONEX_RUNTIME_ID: judge-effects @@ -404,9 +390,8 @@ services: - "com.omninode.ticket=OMN-12924" - "com.omninode.runtime.address=runtime://judge-local/judge/effects" - "com.omninode.runtime.id=judge-effects" - projection-api: - <<: *runtime-base + !!merge <<: *runtime-base container_name: omnimarket-judge-projection-api command: ["omnimarket-projection-api"] depends_on: @@ -417,7 +402,7 @@ services: postgres: condition: service_healthy environment: - <<: *judge-runtime-env + !!merge <<: *judge-runtime-env RUNTIME_PROFILE: projection-api OTEL_SERVICE_NAME: omnimarket-judge-projection-api BIFROST_CONTRACT_PATH: "" @@ -436,12 +421,10 @@ services: - "com.omninode.lane=judge" - "com.omninode.ticket=OMN-12924" - "com.omninode.service=projection-api" - networks: omnibase-infra-judge-network: name: omnibase-infra-judge-network driver: bridge - volumes: judge_postgres_data: name: omnibase-infra-judge-postgres-data diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 5d1478299b..e431d125ed 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -216,6 +216,13 @@ services: - "com.omninode.layer=runtime" - "com.omninode.runtime.address=runtime://omninode-pc/prod/worker" - "com.omninode.runtime.id=prod-worker" + deploy: + # OMN-12990: fail-fast on the ledgered config value. Without this override + # the prod worker inherited the base ${WORKER_REPLICAS:-0} default and a + # plain recreate would silently run zero workers. Sourcing + # docker/runtime-policy.env supplies PROD_WORKER_REPLICAS=1; a recreate + # without it now fails loudly instead of dropping the worker with no signal. + replicas: ${PROD_WORKER_REPLICAS:?runtime policy contract must set PROD_WORKER_REPLICAS (source docker/runtime-policy.env)} agent-actions-consumer: profiles: !override ["prod-disabled"] skill-lifecycle-consumer: diff --git a/docker/docker-compose.stability-test.yml b/docker/docker-compose.stability-test.yml index b51820090e..bc71f7ef34 100644 --- a/docker/docker-compose.stability-test.yml +++ b/docker/docker-compose.stability-test.yml @@ -234,7 +234,13 @@ services: - "com.omninode.runtime.address=runtime://omninode-pc/stability-test/worker" - "com.omninode.runtime.id=stability-test-worker" deploy: - replicas: ${STABILITY_TEST_WORKER_REPLICAS:-1} + # OMN-12990: fail-fast on the ledgered config value, NOT a soft :-1 + # default. The base compose sets ${WORKER_REPLICAS:-0}; a plain recreate + # that omits the policy env would silently drop the worker. Sourcing + # docker/runtime-policy.env (rendered from runtime_policy.contract.yaml) + # supplies STABILITY_TEST_WORKER_REPLICAS=1; a recreate without it now + # fails loudly here instead of scaling the worker to zero with no signal. + replicas: ${STABILITY_TEST_WORKER_REPLICAS:?runtime policy contract must set STABILITY_TEST_WORKER_REPLICAS (source docker/runtime-policy.env)} agent-actions-consumer: profiles: !override ["stability-test-disabled"] skill-lifecycle-consumer: diff --git a/docker/entrypoint-runtime.sh b/docker/entrypoint-runtime.sh index 43b90eb18c..7fa1467626 100755 --- a/docker/entrypoint-runtime.sh +++ b/docker/entrypoint-runtime.sh @@ -131,8 +131,14 @@ else fi if [ -n "${BIFROST_CONTRACT_PATH:-}" ]; then - echo "[entrypoint] Rendering Bifrost delegation contract..." - python -m omnibase_infra.runtime.render_bifrost_delegation_contract + # OMN-12945: Force re-seed from packaged source on every container restart. + # BIFROST_FORCE_RESEED=1 bypasses the stale-volume early-return path so the + # named-volume copy is always rebuilt from the committed packaged source + # merged with the lane-overlay BIFROST_LOCAL_* endpoint vars. This eliminates + # the volume-drift defect where a stale /app/data/delegation/bifrost_delegation.yaml + # silently survived image rebuilds and exposed wrong/missing backends. + echo "[entrypoint] Rendering Bifrost delegation contract (force-reseed)..." + BIFROST_FORCE_RESEED=1 python -m omnibase_infra.runtime.render_bifrost_delegation_contract fi if [ -n "${ONEX_SECRET_RESOLVER_CONFIG_PATH:-}" ]; then diff --git a/docker/lane-overlays/dev.bifrost.env b/docker/lane-overlays/dev.bifrost.env new file mode 100644 index 0000000000..31f4241989 --- /dev/null +++ b/docker/lane-overlays/dev.bifrost.env @@ -0,0 +1,10 @@ +# Generated from docker/lane-overlays/dev.bifrost.yaml. +# Do not edit directly — edit the YAML source and re-render: +# uv run python scripts/render_bifrost_lane_overlay_env.py +# +# OMN-12864: committed endpoint authority for BIFROST_LOCAL_* vars. +# OMN-12815: every URL is the COMPLETE final chat-completions URL. +BIFROST_LOCAL_CODER_ENDPOINT_URL=http://192.168.86.201:8000/v1/chat/completions +BIFROST_LOCAL_DS_V4_FLASH_ENDPOINT_URL=http://192.168.86.200:8101/v1/chat/completions +BIFROST_LOCAL_EMBEDDING_ENDPOINT_URL=http://192.168.86.201:8100/v1/chat/completions +BIFROST_LOCAL_REASONER_ENDPOINT_URL=http://192.168.86.201:8001/v1/chat/completions diff --git a/docker/lane-overlays/dev.bifrost.yaml b/docker/lane-overlays/dev.bifrost.yaml new file mode 100644 index 0000000000..09c8f5d0a6 --- /dev/null +++ b/docker/lane-overlays/dev.bifrost.yaml @@ -0,0 +1,38 @@ +# docker/lane-overlays/dev.bifrost.yaml +# +# OMN-12864: Committed lane overlay — Bifrost local backend endpoint URLs. +# +# These four values were previously only available via ephemeral shell exports +# on .201. Moving them into this committed YAML file makes the deployment +# authority auditable, diff-able, and CI-checked. +# +# AUTHORITY: This file is the single source of truth for the dev lane's +# BIFROST_LOCAL_* endpoint URL bindings. Values here are TYPED and validated +# by ModelBifrostLaneOverlay (OMN-12815: every URL must be the COMPLETE final +# chat-completions URL, not a bare base). +# +# RENDERED FORM: docker/lane-overlays/dev.bifrost.env is generated from this +# file via `uv run python scripts/render_bifrost_lane_overlay_env.py`. +# The compose stack reads the .env sidecar via an `env_file:` reference. +# Never edit the .env sidecar directly — edit this YAML and re-render. +# +# RE-SEED: On each deploy restart, entrypoint-runtime.sh re-renders the +# Bifrost delegation contract FROM the packaged source (OMN-12945). The volume +# copy is never used as an authority on a fresh restart — it is always rebuilt +# from the packaged bifrost_delegation.yaml merged with the env-var endpoints +# supplied by this overlay. +# +# INTERNAL IPs: the .201 and .200 addresses below are local-lane runtime +# endpoints. They are intentionally committed here — this overlay is the +# canonical config authority, not ephemeral shell state. +# onex-allow-internal-ip OMN-12864 reason="committed lane overlay — dev lane local LLM endpoint bindings" +lane: dev +# Coder: Qwen3.6-35B-A3B on .201 port 8000 (RTX 5090) +coder_endpoint_url: "http://192.168.86.201:8000/v1/chat/completions" +# Reasoner: Qwen3.6-27B-MTP-IQ4_XS on .201 port 8001 (RTX 4090) +reasoner_endpoint_url: "http://192.168.86.201:8001/v1/chat/completions" +# Embedding: local embedding server on .201 port 8100 (M2 Ultra) +embedding_endpoint_url: "http://192.168.86.201:8100/v1/chat/completions" +# DS-V4-Flash: DeepSeek V4 Flash 284B MoE on .200 port 8101 (OMN-12492) +# onex-allow-internal-ip OMN-12492 reason="DS-V4-Flash (.200) endpoint in committed lane overlay" +ds4_flash_endpoint_url: "http://192.168.86.200:8101/v1/chat/completions" diff --git a/docker/migrations/forward/064_create_pattern_learning_artifacts.sql b/docker/migrations/forward/064_create_pattern_learning_artifacts.sql deleted file mode 100644 index ea2a97cd2b..0000000000 --- a/docker/migrations/forward/064_create_pattern_learning_artifacts.sql +++ /dev/null @@ -1,92 +0,0 @@ --- Migration: 064_create_pattern_learning_artifacts --- Description: Create pattern_learning_artifacts projection table (OMN-8540) --- Created: 2026-04-12 --- --- Purpose: Projection table for onex.evt.omniintelligence.pattern-stored.v1 events. --- Consumed by omniintelligence projection consumers. Closes the golden chain --- pattern_learning chain: the tail table was referenced but never migrated into --- the omnibase_infra Postgres schema, causing 0/5 golden chain sweep results. --- --- Schema derived from omnidash migrations 0001_omnidash_analytics_read_model.sql, --- 0038_pattern_learning_unique_pattern_id.sql, and 0040_add_evidence_tier_to_patlearn.sql. --- --- Idempotency: All statements use IF NOT EXISTS; safe to re-apply. --- Rollback: See rollback/rollback_064_pattern_learning_artifacts.sql - --- ============================================================================ --- TABLE: pattern_learning_artifacts --- ============================================================================ --- Projects pattern-stored events from omniintelligence. --- Lookup key for golden chain: pattern_name (not correlation_id). - -CREATE TABLE IF NOT EXISTS pattern_learning_artifacts ( - id UUID NOT NULL DEFAULT gen_random_uuid(), - pattern_id UUID NOT NULL, - pattern_name VARCHAR(255) NOT NULL, - pattern_type VARCHAR(100) NOT NULL, - language VARCHAR(50), - lifecycle_state TEXT NOT NULL DEFAULT 'candidate', - state_changed_at TIMESTAMPTZ, - composite_score NUMERIC(10, 6) NOT NULL, - scoring_evidence JSONB NOT NULL DEFAULT '{}', - signature JSONB NOT NULL DEFAULT '{}', - metrics JSONB DEFAULT '{}', - metadata JSONB DEFAULT '{}', - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - projected_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - - CONSTRAINT pk_pattern_learning_artifacts PRIMARY KEY (id), - CONSTRAINT uq_pattern_learning_pattern_id UNIQUE (pattern_id) -); - --- ============================================================================ --- INDEXES --- ============================================================================ - -CREATE INDEX IF NOT EXISTS idx_patlearn_lifecycle_state - ON pattern_learning_artifacts (lifecycle_state); - -CREATE INDEX IF NOT EXISTS idx_patlearn_composite_score - ON pattern_learning_artifacts (composite_score DESC); - -CREATE INDEX IF NOT EXISTS idx_patlearn_state_changed_at - ON pattern_learning_artifacts (state_changed_at DESC); - -CREATE INDEX IF NOT EXISTS idx_patlearn_created_at - ON pattern_learning_artifacts (created_at DESC); - -CREATE INDEX IF NOT EXISTS idx_patlearn_updated_at - ON pattern_learning_artifacts (updated_at DESC); - -CREATE INDEX IF NOT EXISTS idx_patlearn_lifecycle_score - ON pattern_learning_artifacts (lifecycle_state, composite_score DESC); - -CREATE INDEX IF NOT EXISTS idx_patlearn_pattern_name - ON pattern_learning_artifacts (pattern_name); - --- ============================================================================ --- COMMENTS --- ============================================================================ - -COMMENT ON TABLE pattern_learning_artifacts IS - 'Pattern learning projection from onex.evt.omniintelligence.pattern-stored.v1. ' - 'UPSERT key: pattern_id. Golden chain pattern_learning tail table (OMN-8540).'; - -COMMENT ON COLUMN pattern_learning_artifacts.pattern_id IS - 'Unique pattern identifier. Used as UPSERT conflict target.'; - -COMMENT ON COLUMN pattern_learning_artifacts.pattern_name IS - 'Human-readable pattern name. Used as alternate golden chain lookup key.'; - -COMMENT ON COLUMN pattern_learning_artifacts.lifecycle_state IS - 'Pattern lifecycle state: candidate, promoted, deprecated, archived.'; - -COMMENT ON COLUMN pattern_learning_artifacts.composite_score IS - 'Composite quality score (0-1) used for pattern promotion decisions.'; - -COMMENT ON COLUMN pattern_learning_artifacts.scoring_evidence IS - 'JSON evidence used to compute composite_score (signal breakdown).'; - -COMMENT ON COLUMN pattern_learning_artifacts.signature IS - 'Structural signature of the pattern for similarity matching.'; diff --git a/docker/migrations/forward/085_add_runner_completed_at_to_db_metadata.sql b/docker/migrations/forward/085_add_runner_completed_at_to_db_metadata.sql new file mode 100644 index 0000000000..0e3cc78ec2 --- /dev/null +++ b/docker/migrations/forward/085_add_runner_completed_at_to_db_metadata.sql @@ -0,0 +1,38 @@ +-- ============================================================================= +-- MIGRATION: Add runner_completed_at column to db_metadata (OMN-13062) +-- ============================================================================= +-- Ticket: OMN-13062 (migration-gate vacuity fix — retro A-10) +-- Recurrences: OMN-12885, OMN-12934 +-- Version: 1.0.0 +-- +-- PURPOSE: +-- Extends db_metadata with runner_completed_at TIMESTAMPTZ. The forward +-- migration runner (run-forward-migrations.sh) stamps this column as its +-- FINAL act after every migration in the infra and node sets succeeds. +-- +-- This timestamp is the durable evidence that the runner reached +-- successful completion. Combined with the migrations_complete=TRUE +-- sentinel (OMN-3737), the migration-gate healthcheck now has two +-- independent signals: +-- 1. migrations_complete=TRUE — the flag was set without error +-- 2. runner_completed_at IS NOT NULL — the runner reached its final act +-- +-- The runner clears migrations_complete to FALSE at the start of every run +-- and sets it (together with runner_completed_at) only after all migrations +-- succeed. Any nonzero exit from any migration leaves the gate UNHEALTHY. +-- +-- IDEMPOTENCY: +-- ALTER TABLE ... ADD COLUMN IF NOT EXISTS is safe to re-run. +-- +-- ROLLBACK: +-- See rollback/rollback_085_add_runner_completed_at_to_db_metadata.sql +-- ============================================================================= + +ALTER TABLE public.db_metadata + ADD COLUMN IF NOT EXISTS runner_completed_at TIMESTAMPTZ; + +COMMENT ON COLUMN public.db_metadata.runner_completed_at IS + 'Stamped by run-forward-migrations.sh as its final act after all ' + 'infra and node migrations succeed. NULL means the runner never ' + 'completed successfully (partial run, first boot, or mid-run failure). ' + 'OMN-13062 (retro A-10).'; diff --git a/docker/migrations/forward/nodes/node_canary_score_reducer/0001_create_capability_scores.sql b/docker/migrations/forward/nodes/node_canary_score_reducer/0001_create_capability_scores.sql new file mode 100644 index 0000000000..f02a406dc8 --- /dev/null +++ b/docker/migrations/forward/nodes/node_canary_score_reducer/0001_create_capability_scores.sql @@ -0,0 +1,53 @@ +-- OMN-12970: node-owned projection migration for capability_scores. +-- +-- WHY THIS EXISTS +-- node_canary_score_reducer declares projection_api over capability_scores +-- (topic onex.snapshot.projection.capability-scores.v1). The table was +-- historically created ONLY by omnibase_infra forward migration +-- 060_create_routing_outcomes.sql in the omnibase_infra DB, never in the +-- dashboard projection DB (omnidash_analytics) the projection API binds to. +-- Result: the capability-scores topic was DEGRADED at startup +-- ("table 'public.capability_scores' not found"). +-- +-- Vendored by omnibase_infra/scripts/sync-node-migrations.sh into +-- docker/migrations/forward/nodes/node_canary_score_reducer/ and applied to +-- NODE_POSTGRES_DB (omnidash_analytics) by run-forward-migrations.sh under the +-- namespaced migration id node:node_canary_score_reducer:. +-- +-- SCHEMA SOURCE OF TRUTH +-- Mirrors the capability_scores table in omnibase_infra forward migration +-- 060_create_routing_outcomes.sql (table + indexes). routing_outcomes is NOT +-- created here: no projection contract declares it as a projection-API read +-- model, so it stays owned by the omnibase_infra DB. +-- +-- Idempotency: CREATE TABLE / INDEX guarded by IF NOT EXISTS so the migration is +-- safe on a DB where the table already exists and on a fresh omnidash_analytics. + +-- ============================================================================ +-- CAPABILITY_SCORES TABLE +-- ============================================================================ +-- Persisted reducer state: rolling capability metrics per (model_key, task_type). +CREATE TABLE IF NOT EXISTS public.capability_scores ( + id BIGSERIAL PRIMARY KEY, + model_key TEXT NOT NULL, + task_type TEXT NOT NULL, + success_count INT NOT NULL DEFAULT 0, + failure_count INT NOT NULL DEFAULT 0, + total_count INT NOT NULL DEFAULT 0, + success_rate DOUBLE PRECISION NOT NULL DEFAULT 0.0, + avg_latency_ms INT NOT NULL DEFAULT 0, + avg_tokens_per_sec DOUBLE PRECISION NOT NULL DEFAULT 0.0, + total_cost DOUBLE PRECISION NOT NULL DEFAULT 0.0, + graduated BOOLEAN NOT NULL DEFAULT FALSE, + last_updated TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (model_key, task_type) +); + +-- Lookup by model for router input +CREATE INDEX IF NOT EXISTS idx_capability_scores_model + ON capability_scores (model_key); + +-- Graduated models for fast filtering +CREATE INDEX IF NOT EXISTS idx_capability_scores_graduated + ON capability_scores (graduated) + WHERE graduated = TRUE; diff --git a/docker/migrations/forward/nodes/node_omnigate_projection/0000_create_gate_projection_tables.sql b/docker/migrations/forward/nodes/node_omnigate_projection/0000_create_gate_projection_tables.sql new file mode 100644 index 0000000000..4009439cc6 --- /dev/null +++ b/docker/migrations/forward/nodes/node_omnigate_projection/0000_create_gate_projection_tables.sql @@ -0,0 +1,52 @@ +-- Migration: 0000_create_gate_projection_tables.sql +-- Node: node_omnigate_projection +-- Ticket: OMN-13067 +-- Creates gate_activity and gate_metrics tables for the OmniGate projection API. +-- These tables back the projection API endpoints for +-- onex.snapshot.projection.gate.activity.v1 and +-- onex.snapshot.projection.gate.metrics.v1. + +CREATE TABLE IF NOT EXISTS gate_activity ( + id BIGSERIAL PRIMARY KEY, + repository_id TEXT NOT NULL, + project_name TEXT NOT NULL DEFAULT '', + branch TEXT NOT NULL DEFAULT '', + base_sha TEXT NOT NULL DEFAULT '', + head_sha TEXT NOT NULL DEFAULT '', + diff_hash TEXT, + config_hash TEXT, + status TEXT NOT NULL, + action TEXT, + reason TEXT NOT NULL DEFAULT '', + total_checks INTEGER NOT NULL DEFAULT 0, + failed_checks INTEGER NOT NULL DEFAULT 0, + advisory_checks INTEGER NOT NULL DEFAULT 0, + pending_checks INTEGER NOT NULL DEFAULT 0, + observed_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX IF NOT EXISTS gate_activity_observed_at_idx + ON gate_activity (observed_at DESC); + +CREATE INDEX IF NOT EXISTS gate_activity_repository_id_idx + ON gate_activity (repository_id); + +CREATE INDEX IF NOT EXISTS gate_activity_status_idx + ON gate_activity (status); + +-- Aggregate metrics snapshot — single row upserted on each event. +-- id=1 is the canonical singleton row. +CREATE TABLE IF NOT EXISTS gate_metrics ( + id INTEGER PRIMARY KEY DEFAULT 1, + total_events INTEGER NOT NULL DEFAULT 0, + passed INTEGER NOT NULL DEFAULT 0, + failed INTEGER NOT NULL DEFAULT 0, + advisory INTEGER NOT NULL DEFAULT 0, + pending INTEGER NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Seed the singleton metrics row so the projection API returns 1 row immediately. +INSERT INTO gate_metrics (id, total_events, passed, failed, advisory, pending, updated_at) +VALUES (1, 0, 0, 0, 0, 0, NOW()) +ON CONFLICT (id) DO NOTHING; diff --git a/docker/migrations/forward/nodes/node_projection_context_roi/001_create_context_roi_scores.sql b/docker/migrations/forward/nodes/node_projection_context_roi/001_create_context_roi_scores.sql new file mode 100644 index 0000000000..44ebf3dc0c --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_context_roi/001_create_context_roi_scores.sql @@ -0,0 +1,66 @@ +-- OMN-12955: Context-ROI experiment scores projection table. +-- +-- Backs the projection topic +-- onex.snapshot.projection.context.experiment-scores.v1 +-- consumed by the omnidash /experiments ContextExperimentHero and +-- ContextEffectivenessHeatmap panels. +-- +-- Source: per-(task x arm x trial) ModelAttemptReductionRow instances carried +-- on the runner terminal event onex.evt.omnimarket.context-roi-run-completed.v1. +-- One projection row per (run_id, task_id, context_factor_subset, correlation_id) +-- cell. Identity is the runner correlation_id which is unique per cell. + +CREATE EXTENSION IF NOT EXISTS pgcrypto; + +CREATE TABLE IF NOT EXISTS context_roi_scores ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + -- Identity / correlation + run_id TEXT NOT NULL, + correlation_id TEXT NOT NULL, + task_id TEXT NOT NULL, + run_order INTEGER NOT NULL DEFAULT 0 CHECK (run_order >= 0), + -- Arm / context pack (the heatmap "segment") + context_factor_subset TEXT NOT NULL DEFAULT 'off', + context_pack_hash TEXT NOT NULL DEFAULT '', + -- Generation telemetry + attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0), + first_pass_success BOOLEAN NOT NULL DEFAULT FALSE, + final_success BOOLEAN NOT NULL DEFAULT FALSE, + failure_stage TEXT NOT NULL DEFAULT 'none', + -- Token accounting + prompt_tokens INTEGER NOT NULL DEFAULT 0 CHECK (prompt_tokens >= 0), + completion_tokens INTEGER NOT NULL DEFAULT 0 CHECK (completion_tokens >= 0), + tokens_used INTEGER NOT NULL DEFAULT 0 CHECK (tokens_used >= 0), + estimated_cost NUMERIC(18, 6) NOT NULL DEFAULT 0 CHECK (estimated_cost >= 0), + -- Model / routing identity + model_id TEXT NOT NULL DEFAULT '', + provider TEXT NOT NULL DEFAULT '', + endpoint_ref TEXT NOT NULL DEFAULT '', + -- Evidence classification + proof_class TEXT NOT NULL DEFAULT 'runtime-observed-only', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE UNIQUE INDEX IF NOT EXISTS ux_context_roi_scores_identity + ON context_roi_scores (correlation_id); + +CREATE INDEX IF NOT EXISTS ix_context_roi_scores_run + ON context_roi_scores (run_id); + +CREATE INDEX IF NOT EXISTS ix_context_roi_scores_segment_model + ON context_roi_scores (context_factor_subset, model_id); + +CREATE OR REPLACE FUNCTION refresh_context_roi_scores_updated_at() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_context_roi_scores_updated_at ON context_roi_scores; +CREATE TRIGGER trg_context_roi_scores_updated_at + BEFORE UPDATE ON context_roi_scores + FOR EACH ROW + EXECUTE FUNCTION refresh_context_roi_scores_updated_at(); diff --git a/docker/migrations/forward/nodes/node_projection_cost_summary/0001_create_llm_cost_aggregates.sql b/docker/migrations/forward/nodes/node_projection_cost_summary/0001_create_llm_cost_aggregates.sql new file mode 100644 index 0000000000..f0a40b98a3 --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_cost_summary/0001_create_llm_cost_aggregates.sql @@ -0,0 +1,87 @@ +-- OMN-12970: node-owned projection migration for llm_cost_aggregates. +-- +-- WHY THIS EXISTS +-- node_projection_cost_summary declares projection_api over llm_cost_aggregates +-- (topic onex.snapshot.projection.cost.summary.v1). Like llm_call_metrics, this +-- table was historically created ONLY by omnibase_infra forward migration 031 in +-- the omnibase_infra DB, never in the dashboard projection DB +-- (omnidash_analytics) the projection API binds to. Result: the cost-summary +-- topic was DEGRADED at startup ("table 'public.llm_cost_aggregates' not found"). +-- +-- Vendored by omnibase_infra/scripts/sync-node-migrations.sh into +-- docker/migrations/forward/nodes/node_projection_cost_summary/ and applied to +-- NODE_POSTGRES_DB (omnidash_analytics) by run-forward-migrations.sh under the +-- namespaced migration id node:node_projection_cost_summary:. +-- +-- SCHEMA SOURCE OF TRUTH +-- Mirrors omnibase_infra forward migration +-- 031_create_llm_call_metrics_and_cost_aggregates.sql (cost_aggregation_window +-- enum + llm_cost_aggregates table + indexes + updated_at trigger). +-- +-- Idempotency: CREATE TYPE / TABLE / INDEX / TRIGGER guarded so the migration is +-- safe on a DB where the table already exists and on a fresh omnidash_analytics. + +-- ============================================================================ +-- ENUM: cost_aggregation_window +-- ============================================================================ +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'cost_aggregation_window') THEN + CREATE TYPE cost_aggregation_window AS ENUM ('24h', '7d', '30d'); + END IF; +END$$; + +-- ============================================================================ +-- LLM_COST_AGGREGATES TABLE +-- ============================================================================ +CREATE TABLE IF NOT EXISTS llm_cost_aggregates ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + aggregation_key VARCHAR(512) NOT NULL, + "window" cost_aggregation_window NOT NULL, + + total_cost_usd NUMERIC(14, 6) NOT NULL DEFAULT 0, + total_tokens BIGINT NOT NULL DEFAULT 0, + call_count INTEGER NOT NULL DEFAULT 0, + + estimated_coverage_pct NUMERIC(5, 2), + + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT unique_aggregation_key_window UNIQUE (aggregation_key, "window"), + + CONSTRAINT non_negative_total_cost_usd CHECK (total_cost_usd >= 0), + CONSTRAINT non_negative_agg_total_tokens CHECK (total_tokens >= 0), + CONSTRAINT non_negative_call_count CHECK (call_count >= 0), + CONSTRAINT valid_estimated_coverage_pct CHECK ( + estimated_coverage_pct IS NULL + OR (estimated_coverage_pct >= 0.00 AND estimated_coverage_pct <= 100.00) + ) +); + +CREATE INDEX IF NOT EXISTS idx_llm_cost_aggregates_aggregation_key + ON llm_cost_aggregates (aggregation_key); + +CREATE INDEX IF NOT EXISTS idx_llm_cost_aggregates_window + ON llm_cost_aggregates ("window"); + +CREATE INDEX IF NOT EXISTS idx_llm_cost_aggregates_updated_at + ON llm_cost_aggregates (updated_at DESC); + +-- ============================================================================ +-- TRIGGER: auto-update updated_at +-- ============================================================================ +CREATE OR REPLACE FUNCTION update_llm_cost_aggregates_updated_at() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trigger_llm_cost_aggregates_updated_at ON llm_cost_aggregates; +CREATE TRIGGER trigger_llm_cost_aggregates_updated_at + BEFORE UPDATE ON llm_cost_aggregates + FOR EACH ROW + EXECUTE FUNCTION update_llm_cost_aggregates_updated_at(); diff --git a/docker/migrations/forward/nodes/node_projection_llm_cost/0001_create_llm_call_metrics.sql b/docker/migrations/forward/nodes/node_projection_llm_cost/0001_create_llm_call_metrics.sql new file mode 100644 index 0000000000..8a5671f4c0 --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_llm_cost/0001_create_llm_call_metrics.sql @@ -0,0 +1,150 @@ +-- OMN-12970: node-owned projection migration for llm_call_metrics. +-- +-- WHY THIS EXISTS +-- The projection API (omnimarket-*-projection-api) binds to the dashboard +-- projection database (omnidash_analytics), per +-- ModelProjectionDatabaseBinding precedence in +-- omnimarket/projection/api_server.py. Three projection contracts declare +-- llm_call_metrics as their read model: +-- - node_ab_compare_reducer -> onex.snapshot.projection.ab-compare.v1 +-- - node_projection_cost_token_usage -> onex.snapshot.projection.cost.token_usage.v1 +-- - node_projection_llm_cost (writer/owner of this table) +-- The table was historically created ONLY by omnibase_infra forward +-- migration 031 in the omnibase_infra DB. It was never created in +-- omnidash_analytics, so the projection API marked the ab-compare topic +-- DEGRADED ("table 'public.llm_call_metrics' not found at startup") and the +-- dashboard panel rendered its empty state. +-- +-- This node-owned migration is vendored by +-- omnibase_infra/scripts/sync-node-migrations.sh into +-- docker/migrations/forward/nodes/node_projection_llm_cost/ and applied to +-- NODE_POSTGRES_DB (omnidash_analytics) by run-forward-migrations.sh — the +-- same deploy-time materialization path every other marketplace projection +-- node uses. No renumber against the flat infra sequence is required: node +-- migrations are tracked under the namespaced id node::. +-- +-- SCHEMA SOURCE OF TRUTH +-- Mirrors omnibase_infra forward migration +-- 031_create_llm_call_metrics_and_cost_aggregates.sql (table + enum + indexes) +-- so a row written in omnibase_infra is readable verbatim in omnidash_analytics +-- for the projection API. llm_cost_aggregates is intentionally NOT created +-- here: no projection contract declares it as a projection-API read model, so +-- it stays owned by the omnibase_infra DB. +-- +-- Idempotency: CREATE TYPE / TABLE / INDEX guarded by IF NOT EXISTS so the +-- migration is safe on a DB where the table already exists (e.g. omnibase_infra) +-- and on a fresh omnidash_analytics. + +-- ============================================================================ +-- ENUM: usage_source_type +-- ============================================================================ +-- Provenance of token usage data. Mirrors omnibase_infra migration 031. +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'usage_source_type') THEN + CREATE TYPE usage_source_type AS ENUM ('API', 'ESTIMATED', 'MISSING'); + END IF; +END$$; + +-- ============================================================================ +-- LLM_CALL_METRICS TABLE +-- ============================================================================ +-- Per-LLM-call token usage and cost data. Append-only. Column set matches the +-- projection_api columns declared by node_ab_compare_reducer and +-- node_projection_cost_token_usage contracts, plus the provenance/idempotency +-- columns the node_projection_llm_cost consumer writes. +CREATE TABLE IF NOT EXISTS llm_call_metrics ( + -- Identity + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + correlation_id UUID, + + -- Session/run context (logical references, no FK — async event arrival) + session_id VARCHAR(255), + run_id VARCHAR(255), + + -- Model identification + model_id VARCHAR(255) NOT NULL, + + -- Token usage (nullable: not all providers report all fields) + prompt_tokens INTEGER, + completion_tokens INTEGER, + total_tokens INTEGER, + + -- Cost (NULL for unknown models, NOT 0) + estimated_cost_usd NUMERIC(12, 6), + + -- Performance (nullable: failed LLM calls may not have latency data) + latency_ms NUMERIC(10, 2), + + -- Usage provenance + usage_source usage_source_type NOT NULL DEFAULT 'MISSING', + usage_is_estimated BOOLEAN NOT NULL DEFAULT FALSE, + + -- Raw provider response (redacted, capped at 64KB) + usage_raw JSONB, + + -- Provenance / idempotency columns + input_hash VARCHAR(71), + code_version VARCHAR(64), + contract_version VARCHAR(64), + source VARCHAR(255), + + -- Audit + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + -- Constraints + CONSTRAINT non_negative_prompt_tokens CHECK ( + prompt_tokens IS NULL OR prompt_tokens >= 0 + ), + CONSTRAINT non_negative_completion_tokens CHECK ( + completion_tokens IS NULL OR completion_tokens >= 0 + ), + CONSTRAINT non_negative_total_tokens CHECK ( + total_tokens IS NULL OR total_tokens >= 0 + ), + CONSTRAINT non_negative_estimated_cost_usd CHECK ( + estimated_cost_usd IS NULL OR estimated_cost_usd >= 0 + ), + CONSTRAINT non_negative_latency_ms CHECK (latency_ms IS NULL OR latency_ms >= 0), + CONSTRAINT usage_raw_size_limit CHECK ( + usage_raw IS NULL OR octet_length(usage_raw::text) <= 65536 + ) +); + +-- ============================================================================ +-- IDEMPOTENCY: input_hash unique (mirrors omnibase_infra migration 071) +-- ============================================================================ +-- The node_projection_llm_cost consumer inserts with +-- ON CONFLICT (input_hash) DO NOTHING, which requires a unique constraint on +-- the (non-NULL) input_hash. A partial unique index keeps NULL input_hash rows +-- (legacy/unattributed writes) unconstrained. +CREATE UNIQUE INDEX IF NOT EXISTS ux_llm_call_metrics_input_hash + ON llm_call_metrics (input_hash) + WHERE input_hash IS NOT NULL; + +-- ============================================================================ +-- INDEXES +-- ============================================================================ +CREATE INDEX IF NOT EXISTS idx_llm_call_metrics_session_id + ON llm_call_metrics (session_id) + WHERE session_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_llm_call_metrics_run_id + ON llm_call_metrics (run_id) + WHERE run_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_llm_call_metrics_model_id + ON llm_call_metrics (model_id); + +CREATE INDEX IF NOT EXISTS idx_llm_call_metrics_correlation_id + ON llm_call_metrics (correlation_id) + WHERE correlation_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_llm_call_metrics_created_at + ON llm_call_metrics (created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_llm_call_metrics_model_created + ON llm_call_metrics (model_id, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_llm_call_metrics_usage_source + ON llm_call_metrics (usage_source); diff --git a/docker/migrations/forward/nodes/node_projection_llm_routing/0000_create_llm_routing_decisions.sql b/docker/migrations/forward/nodes/node_projection_llm_routing/0000_create_llm_routing_decisions.sql new file mode 100644 index 0000000000..bb593e04b2 --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_llm_routing/0000_create_llm_routing_decisions.sql @@ -0,0 +1,72 @@ +-- OMN-12942: base table for the routing-decision dashboard projection view. +-- +-- node_projection_llm_routing/0001_create_routing_dashboard_projection_view.sql +-- composes the `projection_routing_decision` view over `llm_routing_decisions`. +-- That base table is also created by the flat infra forward set +-- (omnibase_infra docker/migrations/forward/065_create_llm_routing_decisions.sql), +-- which the runner applies in its first phase. But this node-owned migration set +-- must be SELF-CONTAINED: the node forward-migration runner applies node dirs in +-- their own pass, and a clean apply of the node set alone (the path used by the +-- node-migration vendoring/sync flow) hard-fails the view migration when the flat +-- infra table is absent — exactly what happened during the 2026-06-11 04:24Z +-- stability hot-patch, which forced the view migration to be pruned. Owning the +-- base table inside the node removes that ordering dependency. +-- +-- Schema is identical to the flat infra 065 table. All statements are +-- IF NOT EXISTS, so re-applying after 065 (or vice-versa) is a no-op — the two +-- migrations converge on the same table and are safe in either order. + +CREATE EXTENSION IF NOT EXISTS pgcrypto; + +CREATE TABLE IF NOT EXISTS llm_routing_decisions ( + id UUID NOT NULL DEFAULT gen_random_uuid(), + correlation_id UUID NOT NULL, + session_id TEXT, + + -- Routing decision + llm_agent TEXT NOT NULL, + fuzzy_agent TEXT, + agreement BOOLEAN NOT NULL DEFAULT FALSE, + + -- Confidence scores (0-1, NULL when not provided) + llm_confidence NUMERIC(5, 4) CHECK (llm_confidence IS NULL OR (llm_confidence >= 0 AND llm_confidence <= 1)), + fuzzy_confidence NUMERIC(5, 4) CHECK (fuzzy_confidence IS NULL OR (fuzzy_confidence >= 0 AND fuzzy_confidence <= 1)), + + -- Latency + llm_latency_ms INTEGER NOT NULL DEFAULT 0, + fuzzy_latency_ms INTEGER NOT NULL DEFAULT 0, + + -- Routing metadata + used_fallback BOOLEAN NOT NULL DEFAULT FALSE, + routing_prompt_version TEXT NOT NULL DEFAULT 'unknown', + intent TEXT, + model TEXT, + cost_usd NUMERIC(12, 8) CHECK (cost_usd IS NULL OR cost_usd >= 0), + + -- Audit + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + projected_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT pk_llm_routing_decisions PRIMARY KEY (id), + CONSTRAINT uq_llm_routing_decisions_correlation UNIQUE (correlation_id) +); + +CREATE INDEX IF NOT EXISTS idx_lrd_created_at + ON llm_routing_decisions (created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_lrd_agreement + ON llm_routing_decisions (agreement, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_lrd_used_fallback + ON llm_routing_decisions (used_fallback, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_lrd_prompt_version + ON llm_routing_decisions (routing_prompt_version, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_lrd_agent_pair + ON llm_routing_decisions (llm_agent, fuzzy_agent, created_at DESC) + WHERE agreement = FALSE; + +COMMENT ON TABLE llm_routing_decisions IS + 'LLM routing decision projection from onex.evt.omniclaude.llm-routing-decision.v1. ' + 'UPSERT key: correlation_id (UUID). Base table for projection_routing_decision view (OMN-12942).'; diff --git a/docker/migrations/forward/nodes/node_projection_llm_routing/0001_create_routing_dashboard_projection_view.sql b/docker/migrations/forward/nodes/node_projection_llm_routing/0001_create_routing_dashboard_projection_view.sql new file mode 100644 index 0000000000..39d7d99d34 --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_llm_routing/0001_create_routing_dashboard_projection_view.sql @@ -0,0 +1,135 @@ +-- OMN-12839: dashboard-ready routing snapshot over llm_routing_decisions. +-- +-- The raw projection table stores one row per routing decision. The dashboard +-- routing-decision widget consumes a single composed snapshot row with model, +-- intent, task preset, and rule arrays. Keep that composition in Postgres so +-- the projection API remains a typed read surface instead of a client-side +-- state owner. + +CREATE OR REPLACE VIEW projection_routing_decision AS +WITH totals AS ( + SELECT + COUNT(*)::int AS total_count, + MAX(projected_at) AS latest_projection_updated_at + FROM llm_routing_decisions +), +model_candidates AS ( + SELECT + llm_agent AS model_id, + llm_agent AS model_name, + COALESCE(cost_usd, 0)::float AS cost_usd, + 1 AS sort_order + FROM llm_routing_decisions + WHERE llm_agent IS NOT NULL AND llm_agent <> '' + UNION ALL + SELECT + fuzzy_agent AS model_id, + fuzzy_agent AS model_name, + 0::float AS cost_usd, + 2 AS sort_order + FROM llm_routing_decisions + WHERE fuzzy_agent IS NOT NULL AND fuzzy_agent <> '' +), +model_summaries AS ( + SELECT + model_id, + model_name, + MAX(cost_usd)::float AS cost_usd, + MIN(sort_order) AS sort_order + FROM model_candidates + GROUP BY model_id, model_name +), +models AS ( + SELECT COALESCE( + jsonb_agg( + jsonb_build_object( + 'id', model_id, + 'name', model_name, + 'tier', CASE WHEN cost_usd = 0 THEN 'local' ELSE 'cloud' END, + 'cost', cost_usd, + 'host', 'unknown' + ) + ORDER BY sort_order, model_name + ), + '[]'::jsonb + ) AS rows + FROM model_summaries +), +intent_rows AS ( + SELECT COALESCE( + jsonb_agg( + jsonb_build_object( + 'id', intent_id, + 'label', intent_label, + 'color', 'var(--accent)' + ) + ORDER BY intent_label + ), + '[]'::jsonb + ) AS rows + FROM ( + SELECT DISTINCT + COALESCE(NULLIF(intent, ''), 'unknown') AS intent_id, + COALESCE(NULLIF(intent, ''), 'Unknown') AS intent_label + FROM llm_routing_decisions + ) intents +), +task_presets AS ( + SELECT COALESCE( + jsonb_agg( + jsonb_build_object( + 'id', correlation_id::text, + 'label', COALESCE(NULLIF(intent, ''), correlation_id::text), + 'intent', COALESCE(NULLIF(intent, ''), 'unknown'), + 'chosen', llm_agent + ) + ORDER BY created_at DESC + ), + '[]'::jsonb + ) AS rows + FROM ( + SELECT correlation_id, intent, llm_agent, created_at + FROM llm_routing_decisions + ORDER BY created_at DESC + LIMIT 20 + ) recent +), +routing_rules AS ( + SELECT COALESCE( + jsonb_agg( + jsonb_build_object( + 'type', intent_label, + 'model', llm_agent, + 'cost', cost_usd, + 'intentId', intent_id + ) + ORDER BY intent_label, llm_agent + ), + '[]'::jsonb + ) AS rows + FROM ( + SELECT + COALESCE(NULLIF(intent, ''), 'Unknown') AS intent_label, + COALESCE(NULLIF(intent, ''), 'unknown') AS intent_id, + llm_agent, + MIN(COALESCE(cost_usd, 0))::float AS cost_usd + FROM llm_routing_decisions + WHERE llm_agent IS NOT NULL AND llm_agent <> '' + GROUP BY COALESCE(NULLIF(intent, ''), 'Unknown'), + COALESCE(NULLIF(intent, ''), 'unknown'), + llm_agent + ) rules +) +SELECT + COALESCE(models.rows, '[]'::jsonb) AS models, + COALESCE(intent_rows.rows, '[]'::jsonb) AS intents, + COALESCE(task_presets.rows, '[]'::jsonb) AS task_presets, + COALESCE(routing_rules.rows, '[]'::jsonb) AS routing_rules, + NOW() AS captured_at, + (totals.total_count > 0) AS provisioned, + totals.latest_projection_updated_at +FROM totals +CROSS JOIN models +CROSS JOIN intent_rows +CROSS JOIN task_presets +CROSS JOIN routing_rules; diff --git a/docker/migrations/forward/nodes/node_projection_overnight/0000_create_overnight_sessions_tables.sql b/docker/migrations/forward/nodes/node_projection_overnight/0000_create_overnight_sessions_tables.sql new file mode 100644 index 0000000000..c0d3f628f0 --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_overnight/0000_create_overnight_sessions_tables.sql @@ -0,0 +1,76 @@ +-- OMN-12936: base tables for the overnight readiness projection. +-- +-- The 0001 view (projection_overnight_readiness) reads from overnight_sessions. +-- Those base tables were previously created only through the omnibase_infra +-- schema path (schema_overnight_sessions.sql), which the omnimarket projection +-- migration runner does not apply. As a result the projection-api startup found +-- no projection_overnight_readiness relation and served HTTP 503 +-- "table 'public.projection_overnight_readiness' not found at startup". +-- +-- Shipping the base-table DDL as a node-owned migration ordered before the view +-- makes the omnimarket migration runner self-sufficient: it creates the base +-- tables, then the 0001 view materializes against them. The DDL is identical to +-- omnibase_infra schema_overnight_sessions.sql and fully idempotent +-- (IF NOT EXISTS throughout), so re-applying over an infra-seeded database is a +-- no-op. +-- +-- Related: +-- - OMN-8455: W2.8 overnight_sessions migration + node_projection_overnight +-- - node_projection_overnight/migrations/0001_create_overnight_readiness_projection_view.sql + +CREATE TABLE IF NOT EXISTS overnight_sessions ( + session_id TEXT PRIMARY KEY, -- correlation_id from executor + session_start_ts TIMESTAMPTZ NOT NULL, + contract_path TEXT, + dry_run BOOLEAN NOT NULL DEFAULT FALSE, + + -- Aggregate metrics (updated on session-completed) + phases_run TEXT[] NOT NULL DEFAULT '{}', + phases_failed TEXT[] NOT NULL DEFAULT '{}', + phases_skipped TEXT[] NOT NULL DEFAULT '{}', + dispatch_count INT NOT NULL DEFAULT 0, + + -- Halt tracking + halt_reason TEXT, + + -- Terminal state + session_status TEXT NOT NULL DEFAULT 'in_progress', + session_end_ts TIMESTAMPTZ, + + -- Lifecycle + accumulated_cost_usd NUMERIC(10,4) NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT valid_session_status CHECK ( + session_status IN ('in_progress', 'completed', 'partial', 'failed') + ) +); + +-- Normalized phase results table — avoids JSONB overhead, enables per-phase indexing +CREATE TABLE IF NOT EXISTS overnight_session_phases ( + id BIGSERIAL PRIMARY KEY, + session_id TEXT NOT NULL REFERENCES overnight_sessions(session_id) ON DELETE CASCADE, + phase_name TEXT NOT NULL, + phase_status TEXT NOT NULL, -- success | failed | skipped + duration_ms INT NOT NULL DEFAULT 0, + side_effect_summary TEXT NOT NULL DEFAULT '', + error_message TEXT, + sequence_number INT NOT NULL DEFAULT 0, + recorded_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT valid_phase_status CHECK ( + phase_status IN ('success', 'failed', 'skipped') + ) +); + +CREATE UNIQUE INDEX IF NOT EXISTS ix_session_phases_unique + ON overnight_session_phases(session_id, phase_name, sequence_number); + +CREATE INDEX IF NOT EXISTS ix_overnight_sessions_status + ON overnight_sessions(session_status); + +CREATE INDEX IF NOT EXISTS ix_overnight_sessions_start + ON overnight_sessions(session_start_ts DESC); + +CREATE INDEX IF NOT EXISTS ix_overnight_session_phases_session + ON overnight_session_phases(session_id); diff --git a/docker/migrations/forward/nodes/node_projection_overnight/0001_create_overnight_readiness_projection_view.sql b/docker/migrations/forward/nodes/node_projection_overnight/0001_create_overnight_readiness_projection_view.sql new file mode 100644 index 0000000000..113adcd495 --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_overnight/0001_create_overnight_readiness_projection_view.sql @@ -0,0 +1,74 @@ +-- OMN-12839: dashboard readiness snapshot over overnight_sessions. +-- +-- The readiness gate widget consumes a compact status summary rather than raw +-- overnight session rows. This view keeps the rollup server-side and exposes a +-- contract-owned projection topic through projection-api. + +CREATE OR REPLACE VIEW projection_overnight_readiness AS +WITH totals AS ( + SELECT + COUNT(*)::int AS session_count, + COALESCE(SUM(CASE WHEN session_status = 'completed' THEN 1 ELSE 0 END), 0)::int + AS completed_count, + COALESCE(SUM(CASE WHEN session_status = 'partial' THEN 1 ELSE 0 END), 0)::int + AS partial_count, + COALESCE(SUM(CASE WHEN session_status = 'failed' THEN 1 ELSE 0 END), 0)::int + AS failed_count, + COALESCE(SUM(CASE WHEN session_status = 'in_progress' THEN 1 ELSE 0 END), 0)::int + AS in_progress_count, + COALESCE(SUM(dispatch_count), 0)::int AS dispatch_count, + COALESCE(SUM(cardinality(phases_failed)), 0)::int AS failed_phase_count, + MAX(updated_at) AS latest_projection_updated_at + FROM overnight_sessions +), +status AS ( + SELECT + CASE + WHEN totals.session_count = 0 THEN 'WARN' + WHEN totals.failed_count > 0 OR totals.failed_phase_count > 0 THEN 'FAIL' + WHEN totals.partial_count > 0 OR totals.in_progress_count > 0 THEN 'WARN' + ELSE 'PASS' + END AS overall_status + FROM totals +), +dimensions AS ( + SELECT jsonb_build_array( + jsonb_build_object( + 'name', 'Sessions completed', + 'status', CASE + WHEN totals.session_count = 0 THEN 'WARN' + WHEN totals.completed_count > 0 THEN 'PASS' + ELSE 'WARN' + END, + 'detail', totals.completed_count::text || ' completed of ' + || totals.session_count::text || ' tracked sessions' + ), + jsonb_build_object( + 'name', 'Failed phases', + 'status', CASE + WHEN totals.failed_count > 0 OR totals.failed_phase_count > 0 THEN 'FAIL' + ELSE 'PASS' + END, + 'detail', totals.failed_phase_count::text || ' failed phases across ' + || totals.failed_count::text || ' failed sessions' + ), + jsonb_build_object( + 'name', 'Dispatch activity', + 'status', CASE + WHEN totals.session_count = 0 THEN 'WARN' + WHEN totals.dispatch_count > 0 THEN 'PASS' + ELSE 'WARN' + END, + 'detail', totals.dispatch_count::text || ' dispatches recorded' + ) + ) AS rows + FROM totals +) +SELECT + dimensions.rows AS dimensions, + status.overall_status AS "overallStatus", + COALESCE(totals.latest_projection_updated_at, NOW()) AS "lastCheckedAt", + totals.latest_projection_updated_at +FROM totals +CROSS JOIN status +CROSS JOIN dimensions; diff --git a/docker/migrations/forward/nodes/node_projection_pattern_learning/0000_create_pattern_learning_artifacts.sql b/docker/migrations/forward/nodes/node_projection_pattern_learning/0000_create_pattern_learning_artifacts.sql new file mode 100644 index 0000000000..976b9d0b17 --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_pattern_learning/0000_create_pattern_learning_artifacts.sql @@ -0,0 +1,68 @@ +-- Migration: 0000_create_pattern_learning_artifacts +-- Node: node_projection_pattern_learning +-- Target DB: omnidash_analytics (omnibase_infra postgres on .201:5436) +-- +-- Purpose: Projection table for onex.evt.omniintelligence.pattern-stored.v1. +-- Consumed by node_projection_pattern_learning. Closes the golden chain +-- pattern_learning chain (OMN-13124, clears OMN-13102): the tail table had no +-- consumer for pattern-stored.v1, so the sweep timed out (0/N golden chains). +-- +-- UPSERT key: pattern_id (latest-state-wins). Idempotent: IF NOT EXISTS. +-- Schema derived from omnibase_infra migration 064 + omnidash read-model, with +-- correlation_id added (the golden-chain expected field) and NOT-NULL defaults +-- relaxed so sparse pattern-stored events project without violating constraints. + +CREATE TABLE IF NOT EXISTS pattern_learning_artifacts ( + id UUID NOT NULL DEFAULT gen_random_uuid(), + pattern_id UUID NOT NULL, + pattern_name VARCHAR(255) NOT NULL DEFAULT '', + pattern_type VARCHAR(100) NOT NULL DEFAULT '', + language VARCHAR(50), + lifecycle_state TEXT NOT NULL DEFAULT 'candidate', + state_changed_at TIMESTAMPTZ, + composite_score NUMERIC(10, 6) NOT NULL DEFAULT 0, + scoring_evidence JSONB NOT NULL DEFAULT '{}', + signature JSONB NOT NULL DEFAULT '{}', + metrics JSONB DEFAULT '{}', + metadata JSONB DEFAULT '{}', + correlation_id TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + projected_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT pk_pattern_learning_artifacts PRIMARY KEY (id), + CONSTRAINT uq_pattern_learning_pattern_id UNIQUE (pattern_id) +); + +-- Backfill correlation_id on pre-existing deployments of this table that were +-- created by the legacy omnibase_infra 064 migration (which lacked the column). +ALTER TABLE pattern_learning_artifacts + ADD COLUMN IF NOT EXISTS correlation_id TEXT; + +CREATE INDEX IF NOT EXISTS idx_patlearn_lifecycle_state + ON pattern_learning_artifacts (lifecycle_state); + +CREATE INDEX IF NOT EXISTS idx_patlearn_composite_score + ON pattern_learning_artifacts (composite_score DESC); + +CREATE INDEX IF NOT EXISTS idx_patlearn_state_changed_at + ON pattern_learning_artifacts (state_changed_at DESC); + +CREATE INDEX IF NOT EXISTS idx_patlearn_created_at + ON pattern_learning_artifacts (created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_patlearn_updated_at + ON pattern_learning_artifacts (updated_at DESC); + +CREATE INDEX IF NOT EXISTS idx_patlearn_pattern_name + ON pattern_learning_artifacts (pattern_name); + +COMMENT ON TABLE pattern_learning_artifacts IS + 'Pattern learning projection from onex.evt.omniintelligence.pattern-stored.v1. ' + 'UPSERT key: pattern_id. Golden chain pattern_learning tail table (OMN-13124).'; + +COMMENT ON COLUMN pattern_learning_artifacts.pattern_id IS + 'Unique pattern identifier. Used as UPSERT conflict target.'; + +COMMENT ON COLUMN pattern_learning_artifacts.correlation_id IS + 'Correlation ID for distributed tracing. Golden-chain expected field.'; diff --git a/docker/migrations/forward/nodes/node_projection_routing_decision/0021_create_agent_routing_decisions.sql b/docker/migrations/forward/nodes/node_projection_routing_decision/0021_create_agent_routing_decisions.sql new file mode 100644 index 0000000000..fe97833b59 --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_routing_decision/0021_create_agent_routing_decisions.sql @@ -0,0 +1,58 @@ +-- OMN-13150 (FIX-1): agent_routing_decisions projection table in omnidash_analytics. +-- Target DB: omnidash_analytics (omnibase_infra postgres on .201:5436) +-- Node: omnimarket.nodes.node_projection_routing_decision (DDL owner for the +-- omnidash_analytics copy of agent_routing_decisions). +-- Source topic: onex.evt.omniclaude.routing-decision.v1 +-- Idempotency: INSERT ... ON CONFLICT (id) DO NOTHING (append-only observability). +-- +-- WHY THIS MIGRATION EXISTS +-- The projection node's contract declares db_io.db_tables[0].database: omnidash_analytics, +-- and the runtime DB resolution (ModelProjectionRuntimeBinding.from_legacy_settings and the +-- projection-api _projection_database_binding_from_settings) both resolve +-- omnidash_analytics_db_url FIRST. The contract-declared AND runtime-resolved target is +-- therefore omnidash_analytics. But the base DDL +-- (omnibase_infra docker/migrations/forward/021_create_agent_routing_decisions_table.sql) +-- is applied by the flat infra forward set to the omnibase_infra DB only, so +-- agent_routing_decisions never landed in omnidash_analytics. All sibling projection tables +-- (session_outcomes, llm_call_metrics, pattern_learning_artifacts, llm_routing_decisions, +-- node_service_registry, savings_estimates, ...) live in omnidash_analytics via per-node +-- migrations under /migrations/, vendored into +-- omnibase_infra/docker/migrations/forward/nodes// and applied to $NODE_PGDB +-- (omnidash_analytics) by run-forward-migrations.sh. node_projection_routing_decision was +-- the only projection node missing its per-node migration; this file closes the gap. +-- +-- This node-owned migration is SELF-CONTAINED and idempotent: every statement is +-- IF NOT EXISTS, so applying it to omnidash_analytics is independent of the flat infra 021 +-- (which targets a different DB). Schema is identical to the flat infra 021 table +-- (id PK + the OMN-2057 project-context columns absorbed from omniclaude). + +CREATE TABLE IF NOT EXISTS agent_routing_decisions ( + -- Identity + id UUID PRIMARY KEY, + correlation_id UUID, + + -- Routing decision + selected_agent VARCHAR(255), + confidence_score DECIMAL(5, 4), + + -- Audit (TTL keys off created_at) + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + -- Request context + request_type VARCHAR(100), + alternatives JSONB, + routing_reason TEXT, + domain VARCHAR(255), + metadata JSONB, + + -- Project context (absorbed from omniclaude - OMN-2057) + project_path TEXT, + project_name VARCHAR(255), + claude_session_id VARCHAR(255) +); + +-- Minimal indexing for write-heavy workload - only TTL cleanup index. +CREATE INDEX IF NOT EXISTS idx_agent_routing_decisions_created_at + ON agent_routing_decisions (created_at); + +COMMENT ON TABLE agent_routing_decisions IS 'Agent routing decisions from polymorphic router (OMN-1743). Append-only observability projection (node_projection_routing_decision, omnidash_analytics).'; diff --git a/docker/migrations/forward/nodes/node_projection_savings/077_create_cost_savings_overview_projection_view.sql b/docker/migrations/forward/nodes/node_projection_savings/077_create_cost_savings_overview_projection_view.sql new file mode 100644 index 0000000000..dcaa1c4873 --- /dev/null +++ b/docker/migrations/forward/nodes/node_projection_savings/077_create_cost_savings_overview_projection_view.sql @@ -0,0 +1,121 @@ +-- OMN-12839: dashboard cost-savings overview snapshot over savings_estimates. +-- +-- The dashboard overview widget consumes one composed row with KPI totals, +-- per-model savings rows, recent runs, and warning metadata. Compose that view +-- next to the savings reducer so the projection API can serve the widget from a +-- contract-declared read model instead of fixture-only data. + +CREATE OR REPLACE VIEW projection_cost_savings_overview AS +WITH source_rows AS ( + SELECT + session_id, + COALESCE(NULLIF(model_local, ''), 'local') AS model_id, + COALESCE(NULLIF(model_local, ''), 'Local model') AS display_name, + COALESCE(NULLIF(model_cloud_baseline, ''), 'cloud-baseline') + AS baseline_model, + local_cost_usd::float AS local_cost_usd, + cloud_cost_usd::float AS cloud_cost_usd, + savings_usd::float AS savings_usd, + COALESCE(updated_at, created_at, event_timestamp) AS projected_at + FROM savings_estimates +), +totals AS ( + SELECT + COALESCE(SUM(local_cost_usd), 0)::float AS total_cost_usd, + COALESCE(SUM(cloud_cost_usd), 0)::float AS total_baseline_cost_usd, + COALESCE(SUM(savings_usd), 0)::float AS total_savings_usd, + COUNT(*)::int AS run_count, + MAX(projected_at) AS latest_projection_updated_at + FROM source_rows +), +model_rows AS ( + SELECT COALESCE( + jsonb_agg( + jsonb_build_object( + 'model_id', model_id, + 'display_name', display_name, + 'execution_mode', 'delegated', + 'task_count', task_count, + 'tokens_total', 0, + 'cost_usd', cost_usd, + 'baseline_cost_usd', baseline_cost_usd, + 'savings_usd', savings_usd, + 'savings_pct', CASE WHEN baseline_cost_usd > 0 + THEN savings_usd / baseline_cost_usd ELSE 0 END, + 'runtime_address', NULL, + 'evidence_ref', NULL + ) + ORDER BY savings_usd DESC, display_name + ), + '[]'::jsonb + ) AS rows + FROM ( + SELECT + model_id, + display_name, + COUNT(*)::int AS task_count, + COALESCE(SUM(local_cost_usd), 0)::float AS cost_usd, + COALESCE(SUM(cloud_cost_usd), 0)::float AS baseline_cost_usd, + COALESCE(SUM(savings_usd), 0)::float AS savings_usd + FROM source_rows + GROUP BY model_id, display_name + ) grouped_models +), +recent_runs AS ( + SELECT COALESCE( + jsonb_agg( + jsonb_build_object( + 'session_id', session_id, + 'task_type', 'savings-estimated', + 'model_name', display_name, + 'prompt_tokens', 0, + 'completion_tokens', 0, + 'total_tokens', 0, + 'savings_usd', savings_usd, + 'latency_ms', NULL, + 'created_at', projected_at, + 'token_provenance', 'unknown' + ) + ORDER BY projected_at DESC + ), + '[]'::jsonb + ) AS rows + FROM ( + SELECT * + FROM source_rows + ORDER BY projected_at DESC + LIMIT 20 + ) limited_runs +), +warnings AS ( + SELECT CASE WHEN totals.run_count > 0 THEN + jsonb_build_array( + 'Token counts unavailable in savings_estimates; token KPIs remain zero until measured token provenance is projected.' + ) + ELSE '[]'::jsonb END AS rows + FROM totals +) +SELECT + 'all'::text AS "window", + totals.total_cost_usd, + totals.total_baseline_cost_usd, + totals.total_savings_usd, + CASE WHEN totals.total_baseline_cost_usd > 0 + THEN totals.total_savings_usd / totals.total_baseline_cost_usd + ELSE 0 + END AS savings_rate, + 0::int AS tokens_total, + 0::int AS tokens_to_compliance, + 0::float AS local_token_pct, + COALESCE(totals.latest_projection_updated_at, NOW()) AS captured_at, + model_rows.rows, + recent_runs.rows AS recent_runs, + 0::int AS measured_run_count, + totals.run_count AS zero_token_run_count, + warnings.rows AS warnings, + (totals.run_count > 0) AS provisioned, + totals.latest_projection_updated_at +FROM totals +CROSS JOIN model_rows +CROSS JOIN recent_runs +CROSS JOIN warnings; diff --git a/docker/migrations/rollback/rollback_064_pattern_learning_artifacts.sql b/docker/migrations/rollback/rollback_064_pattern_learning_artifacts.sql deleted file mode 100644 index 9550b62c60..0000000000 --- a/docker/migrations/rollback/rollback_064_pattern_learning_artifacts.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Rollback: 064_create_pattern_learning_artifacts (OMN-8540) -DROP TABLE IF EXISTS pattern_learning_artifacts; diff --git a/docker/migrations/rollback/rollback_085_add_runner_completed_at_to_db_metadata.sql b/docker/migrations/rollback/rollback_085_add_runner_completed_at_to_db_metadata.sql new file mode 100644 index 0000000000..5608069cae --- /dev/null +++ b/docker/migrations/rollback/rollback_085_add_runner_completed_at_to_db_metadata.sql @@ -0,0 +1,9 @@ +-- ============================================================================= +-- ROLLBACK: Remove runner_completed_at column from db_metadata (OMN-13062) +-- ============================================================================= +-- Ticket: OMN-13062 (migration-gate vacuity fix — retro A-10) +-- Version: 1.0.0 +-- ============================================================================= + +ALTER TABLE public.db_metadata + DROP COLUMN IF EXISTS runner_completed_at; diff --git a/docker/migrations/schema_fingerprint.sha256 b/docker/migrations/schema_fingerprint.sha256 index 20a5206436..be3554a364 100644 --- a/docker/migrations/schema_fingerprint.sha256 +++ b/docker/migrations/schema_fingerprint.sha256 @@ -1,6 +1,6 @@ # Schema migration fingerprint for omnibase_infra (auto-generated) # Regenerate: python scripts/check_schema_fingerprint.py stamp # Verify: python scripts/check_schema_fingerprint.py verify -sha256:dc4b265ba91d2d28ca02b75546cf73f4c05be55e8a3ecd27585653959a19c734 -generated_at: 2026-06-09T17:45:31Z +sha256:8fb160a6ef81d8b2d88e2681c3312ddcc04dbdb6fe8199c1751bcdd97bb78fc7 +generated_at: 2026-06-14T06:12:29Z migration_file_count: 70 diff --git a/docker/migrations/skip-manifest.yaml b/docker/migrations/skip-manifest.yaml new file mode 100644 index 0000000000..cccfbad11a --- /dev/null +++ b/docker/migrations/skip-manifest.yaml @@ -0,0 +1,34 @@ +# skip-manifest.yaml — Intentionally-skipped migration IDs +# +# Ticket: OMN-13062 (migration-gate vacuity fix — retro A-10) +# +# PURPOSE +# The forward migration runner (run-forward-migrations.sh) checks this +# file before applying each migration. A migration listed here is treated +# as already-applied (a skip record is written to schema_migrations with +# checksum "skip-manifest") without executing the SQL. +# +# WHEN TO USE +# - A migration was discovered to be a no-op on warm volumes (already +# applied via docker-entrypoint-initdb.d on first boot). +# - A migration contains a known-safe but environment-specific statement +# that must not re-run on warm volumes. +# - Never use this to hide a broken migration — fix the SQL instead. +# +# HOW TO ADD AN ENTRY +# 1. Confirm the migration is safe to skip (e.g., idempotent but already +# applied, or intentionally empty). +# 2. Add the entry to skipped_migrations below. +# 3. Commit the manifest change in the SAME PR that deems the migration +# unrunnable. Operator env cannot inject skips. +# +# ENTRY FORMAT +# id: "docker/" — migration_id as recorded in schema_migrations +# reason: "human-readable reason" +# ticket: "OMN-XXXX" — Linear ticket authorising the skip +# +# EXIT CODES for run-forward-migrations.sh +# 0 — all migrations applied (or skipped by this manifest) +# 1 — migration failure (nonzero psql exit) + +skipped_migrations: [] diff --git a/docker/runners/runner-image.lock.json b/docker/runners/runner-image.lock.json index eb0d43a063..c2b1eb0279 100644 --- a/docker/runners/runner-image.lock.json +++ b/docker/runners/runner-image.lock.json @@ -1,12 +1,12 @@ { "base_image_digest": "sha256:3ba65aa20f86a0fad9df2b2c259c613df006b2e6d0bfcc8a146afb8c525a9751", "gh_version": "2.67.0", - "identity_digest": "7f4a28050457a278a6712b05de154cb8", + "identity_digest": "efc067283a47eb66bfae03322e966b0c", "image_version": 5, "kubectl_version": "1.32.1", "python_version": "3.12", "runner_version": "2.334.0", - "shared_env_digest": "83c23e55321a8c8cb95d48c1", + "shared_env_digest": "215aa5ffa92e228bf703615d", "shared_env_install_args": "--frozen --all-extras --all-groups --no-install-project", "uv_version": "0.6.14" } diff --git a/docker/runtime-policy.env b/docker/runtime-policy.env index e8b54fbb26..aa2a91fe52 100644 --- a/docker/runtime-policy.env +++ b/docker/runtime-policy.env @@ -2,6 +2,7 @@ # Do not edit policy values here; edit the contract and re-render. AUXILIARY_SERVICES_OMNIMEMORY_ENABLED=false BIFROST_VERIFY_ENDPOINTS=1 +BIFROST_VERTEX_GEMINI_ENDPOINT_URL=https://us-central1-aiplatform.googleapis.com/v1beta1/projects/gen-lang-client-0084338881/locations/us-central1/endpoints/openapi/chat/completions DEV_COMPOSE_PROJECT=omnibase-infra DEV_RUNTIME_EFFECTS_BIFROST_VERIFY_ENDPOINTS=1 DEV_RUNTIME_EFFECTS_CAPABILITIES=effects.consumer,market.skill-proof,runtime.effects @@ -9,7 +10,7 @@ DEV_RUNTIME_EFFECTS_OMNIMEMORY_ENABLED=false DEV_RUNTIME_EFFECTS_OMNIMEMORY_MEMGRAPH_HOST= DEV_RUNTIME_EFFECTS_PORT=8086 DEV_RUNTIME_EFFECTS_PUBLISH_INTROSPECTION=false -DEV_RUNTIME_EFFECTS_SECRET_RESOLVER_CONFIG_JSON='{"enable_convention_fallback":false,"mappings":[{"logical_name":"llm.openrouter.api_key","source":{"source_path":"OPEN_ROUTER_API_KEY","source_type":"env"}},{"logical_name":"llm.glm.api_key","source":{"source_path":"LLM_GLM_API_KEY","source_type":"env"}},{"logical_name":"llm.gemini.api_key","source":{"source_path":"GEMINI_API_KEY","source_type":"env"}}]}' +DEV_RUNTIME_EFFECTS_SECRET_RESOLVER_CONFIG_JSON='{"enable_convention_fallback":false,"mappings":[{"logical_name":"llm.openrouter.api_key","source":{"source_path":"OPEN_ROUTER_API_KEY","source_type":"env"}},{"logical_name":"llm.glm.api_key","source":{"source_path":"LLM_GLM_API_KEY","source_type":"env"}},{"logical_name":"llm.gemini.api_key","source":{"source_path":"GEMINI_API_KEY","source_type":"env"}},{"logical_name":"llm.vertex.access_token","source":{"source_path":"/run/secrets/vertex_access_token","source_type":"file"}}]}' DEV_RUNTIME_EFFECTS_SECRET_RESOLVER_CONFIG_PATH=/app/data/delegation/secret_resolver.yaml DEV_RUNTIME_MAIN_BIFROST_VERIFY_ENDPOINTS=1 DEV_RUNTIME_MAIN_CAPABILITIES=market.skill-proof,workflow.orchestration,runtime.main @@ -17,16 +18,19 @@ DEV_RUNTIME_MAIN_OMNIMEMORY_ENABLED=false DEV_RUNTIME_MAIN_OMNIMEMORY_MEMGRAPH_HOST= DEV_RUNTIME_MAIN_PORT=8085 DEV_RUNTIME_MAIN_PUBLISH_INTROSPECTION=true -DEV_RUNTIME_MAIN_SECRET_RESOLVER_CONFIG_JSON='{"enable_convention_fallback":false,"mappings":[{"logical_name":"llm.openrouter.api_key","source":{"source_path":"OPEN_ROUTER_API_KEY","source_type":"env"}},{"logical_name":"llm.glm.api_key","source":{"source_path":"LLM_GLM_API_KEY","source_type":"env"}},{"logical_name":"llm.gemini.api_key","source":{"source_path":"GEMINI_API_KEY","source_type":"env"}}]}' +DEV_RUNTIME_MAIN_SECRET_RESOLVER_CONFIG_JSON='{"enable_convention_fallback":false,"mappings":[{"logical_name":"llm.openrouter.api_key","source":{"source_path":"OPEN_ROUTER_API_KEY","source_type":"env"}},{"logical_name":"llm.glm.api_key","source":{"source_path":"LLM_GLM_API_KEY","source_type":"env"}},{"logical_name":"llm.gemini.api_key","source":{"source_path":"GEMINI_API_KEY","source_type":"env"}},{"logical_name":"llm.vertex.access_token","source":{"source_path":"/run/secrets/vertex_access_token","source_type":"file"}}]}' DEV_RUNTIME_MAIN_SECRET_RESOLVER_CONFIG_PATH=/app/data/delegation/secret_resolver.yaml DEV_RUNTIME_WORKER_BIFROST_VERIFY_ENDPOINTS=1 DEV_RUNTIME_WORKER_CAPABILITIES=workflow.dispatch,contract.update,runtime.worker DEV_RUNTIME_WORKER_OMNIMEMORY_ENABLED=false DEV_RUNTIME_WORKER_OMNIMEMORY_MEMGRAPH_HOST= DEV_RUNTIME_WORKER_PUBLISH_INTROSPECTION=false -DEV_RUNTIME_WORKER_SECRET_RESOLVER_CONFIG_JSON='{"enable_convention_fallback":false,"mappings":[{"logical_name":"llm.openrouter.api_key","source":{"source_path":"OPEN_ROUTER_API_KEY","source_type":"env"}},{"logical_name":"llm.glm.api_key","source":{"source_path":"LLM_GLM_API_KEY","source_type":"env"}},{"logical_name":"llm.gemini.api_key","source":{"source_path":"GEMINI_API_KEY","source_type":"env"}}]}' +DEV_RUNTIME_WORKER_SECRET_RESOLVER_CONFIG_JSON='{"enable_convention_fallback":false,"mappings":[{"logical_name":"llm.openrouter.api_key","source":{"source_path":"OPEN_ROUTER_API_KEY","source_type":"env"}},{"logical_name":"llm.glm.api_key","source":{"source_path":"LLM_GLM_API_KEY","source_type":"env"}},{"logical_name":"llm.gemini.api_key","source":{"source_path":"GEMINI_API_KEY","source_type":"env"}},{"logical_name":"llm.vertex.access_token","source":{"source_path":"/run/secrets/vertex_access_token","source_type":"file"}}]}' DEV_RUNTIME_WORKER_SECRET_RESOLVER_CONFIG_PATH=/app/data/delegation/secret_resolver.yaml DEV_TOPIC_PROVISIONER_MAX_PARTITIONS=0 +DEV_WORKER_REPLICAS=1 +GOOGLE_CLOUD_LOCATION=us-central1 +GOOGLE_CLOUD_PROJECT=gen-lang-client-0084338881 JUDGE_COMPOSE_PROJECT=omnibase-infra-judge JUDGE_RUNTIME_EFFECTS_BIFROST_VERIFY_ENDPOINTS=0 JUDGE_RUNTIME_EFFECTS_CAPABILITIES=effects.consumer,market.skill-proof,runtime.effects @@ -34,7 +38,7 @@ JUDGE_RUNTIME_EFFECTS_OMNIMEMORY_ENABLED=false JUDGE_RUNTIME_EFFECTS_OMNIMEMORY_MEMGRAPH_HOST= JUDGE_RUNTIME_EFFECTS_PORT=48086 JUDGE_RUNTIME_EFFECTS_PUBLISH_INTROSPECTION=false -JUDGE_RUNTIME_EFFECTS_SECRET_RESOLVER_CONFIG_JSON='{"enable_convention_fallback":false,"mappings":[{"logical_name":"llm.openrouter.api_key","source":{"source_path":"OPEN_ROUTER_API_KEY","source_type":"env"}},{"logical_name":"llm.glm.api_key","source":{"source_path":"LLM_GLM_API_KEY","source_type":"env"}},{"logical_name":"llm.gemini.api_key","source":{"source_path":"GEMINI_API_KEY","source_type":"env"}}]}' +JUDGE_RUNTIME_EFFECTS_SECRET_RESOLVER_CONFIG_JSON='{"enable_convention_fallback":false,"mappings":[{"logical_name":"llm.openrouter.api_key","source":{"source_path":"OPEN_ROUTER_API_KEY","source_type":"env"}},{"logical_name":"llm.glm.api_key","source":{"source_path":"LLM_GLM_API_KEY","source_type":"env"}},{"logical_name":"llm.gemini.api_key","source":{"source_path":"GEMINI_API_KEY","source_type":"env"}},{"logical_name":"llm.vertex.access_token","source":{"source_path":"/run/secrets/vertex_access_token","source_type":"file"}}]}' JUDGE_RUNTIME_EFFECTS_SECRET_RESOLVER_CONFIG_PATH=/app/data/delegation/secret_resolver.yaml JUDGE_RUNTIME_MAIN_BIFROST_VERIFY_ENDPOINTS=0 JUDGE_RUNTIME_MAIN_CAPABILITIES=market.skill-proof,workflow.orchestration,runtime.main @@ -42,17 +46,18 @@ JUDGE_RUNTIME_MAIN_OMNIMEMORY_ENABLED=false JUDGE_RUNTIME_MAIN_OMNIMEMORY_MEMGRAPH_HOST= JUDGE_RUNTIME_MAIN_PORT=48085 JUDGE_RUNTIME_MAIN_PUBLISH_INTROSPECTION=true -JUDGE_RUNTIME_MAIN_SECRET_RESOLVER_CONFIG_JSON='{"enable_convention_fallback":false,"mappings":[{"logical_name":"llm.openrouter.api_key","source":{"source_path":"OPEN_ROUTER_API_KEY","source_type":"env"}},{"logical_name":"llm.glm.api_key","source":{"source_path":"LLM_GLM_API_KEY","source_type":"env"}},{"logical_name":"llm.gemini.api_key","source":{"source_path":"GEMINI_API_KEY","source_type":"env"}}]}' +JUDGE_RUNTIME_MAIN_SECRET_RESOLVER_CONFIG_JSON='{"enable_convention_fallback":false,"mappings":[{"logical_name":"llm.openrouter.api_key","source":{"source_path":"OPEN_ROUTER_API_KEY","source_type":"env"}},{"logical_name":"llm.glm.api_key","source":{"source_path":"LLM_GLM_API_KEY","source_type":"env"}},{"logical_name":"llm.gemini.api_key","source":{"source_path":"GEMINI_API_KEY","source_type":"env"}},{"logical_name":"llm.vertex.access_token","source":{"source_path":"/run/secrets/vertex_access_token","source_type":"file"}}]}' JUDGE_RUNTIME_MAIN_SECRET_RESOLVER_CONFIG_PATH=/app/data/delegation/secret_resolver.yaml JUDGE_RUNTIME_WORKER_BIFROST_VERIFY_ENDPOINTS=0 JUDGE_RUNTIME_WORKER_CAPABILITIES=workflow.dispatch,contract.update,runtime.worker JUDGE_RUNTIME_WORKER_OMNIMEMORY_ENABLED=false JUDGE_RUNTIME_WORKER_OMNIMEMORY_MEMGRAPH_HOST= JUDGE_RUNTIME_WORKER_PUBLISH_INTROSPECTION=false -JUDGE_RUNTIME_WORKER_SECRET_RESOLVER_CONFIG_JSON='{"enable_convention_fallback":false,"mappings":[{"logical_name":"llm.openrouter.api_key","source":{"source_path":"OPEN_ROUTER_API_KEY","source_type":"env"}},{"logical_name":"llm.glm.api_key","source":{"source_path":"LLM_GLM_API_KEY","source_type":"env"}},{"logical_name":"llm.gemini.api_key","source":{"source_path":"GEMINI_API_KEY","source_type":"env"}}]}' +JUDGE_RUNTIME_WORKER_SECRET_RESOLVER_CONFIG_JSON='{"enable_convention_fallback":false,"mappings":[{"logical_name":"llm.openrouter.api_key","source":{"source_path":"OPEN_ROUTER_API_KEY","source_type":"env"}},{"logical_name":"llm.glm.api_key","source":{"source_path":"LLM_GLM_API_KEY","source_type":"env"}},{"logical_name":"llm.gemini.api_key","source":{"source_path":"GEMINI_API_KEY","source_type":"env"}},{"logical_name":"llm.vertex.access_token","source":{"source_path":"/run/secrets/vertex_access_token","source_type":"file"}}]}' JUDGE_RUNTIME_WORKER_SECRET_RESOLVER_CONFIG_PATH=/app/data/delegation/secret_resolver.yaml JUDGE_TOPIC_PROVISIONER_MAX_PARTITIONS=1 -LLM_CLOUD_ENDPOINT_HOST_ALLOWLIST=generativelanguage.googleapis.com,api.z.ai +JUDGE_WORKER_REPLICAS=1 +LLM_CLOUD_ENDPOINT_HOST_ALLOWLIST=generativelanguage.googleapis.com,api.z.ai,aiplatform.googleapis.com OMNIMEMORY_ENABLED=false OMNIMEMORY_MEMGRAPH_HOST= OMNIMEMORY_MEMGRAPH_PORT=7687 @@ -82,6 +87,7 @@ PROD_RUNTIME_WORKER_PUBLISH_INTROSPECTION=false PROD_RUNTIME_WORKER_SECRET_RESOLVER_CONFIG_JSON= PROD_RUNTIME_WORKER_SECRET_RESOLVER_CONFIG_PATH= PROD_TOPIC_PROVISIONER_MAX_PARTITIONS=0 +PROD_WORKER_REPLICAS=1 STABILITY_TEST_COMPOSE_PROJECT=omnibase-infra-stability-test STABILITY_TEST_RUNTIME_EFFECTS_BIFROST_VERIFY_ENDPOINTS=0 STABILITY_TEST_RUNTIME_EFFECTS_CAPABILITIES=effects.consumer,market.skill-proof,runtime.effects @@ -89,7 +95,7 @@ STABILITY_TEST_RUNTIME_EFFECTS_OMNIMEMORY_ENABLED=false STABILITY_TEST_RUNTIME_EFFECTS_OMNIMEMORY_MEMGRAPH_HOST= STABILITY_TEST_RUNTIME_EFFECTS_PORT=18086 STABILITY_TEST_RUNTIME_EFFECTS_PUBLISH_INTROSPECTION=false -STABILITY_TEST_RUNTIME_EFFECTS_SECRET_RESOLVER_CONFIG_JSON='{"enable_convention_fallback":false,"mappings":[{"logical_name":"llm.openrouter.api_key","source":{"source_path":"OPEN_ROUTER_API_KEY","source_type":"env"}},{"logical_name":"llm.glm.api_key","source":{"source_path":"LLM_GLM_API_KEY","source_type":"env"}},{"logical_name":"llm.gemini.api_key","source":{"source_path":"GEMINI_API_KEY","source_type":"env"}}]}' +STABILITY_TEST_RUNTIME_EFFECTS_SECRET_RESOLVER_CONFIG_JSON='{"enable_convention_fallback":false,"mappings":[{"logical_name":"llm.openrouter.api_key","source":{"source_path":"OPEN_ROUTER_API_KEY","source_type":"env"}},{"logical_name":"llm.glm.api_key","source":{"source_path":"LLM_GLM_API_KEY","source_type":"env"}},{"logical_name":"llm.gemini.api_key","source":{"source_path":"GEMINI_API_KEY","source_type":"env"}},{"logical_name":"llm.vertex.access_token","source":{"source_path":"/run/secrets/vertex_access_token","source_type":"file"}}]}' STABILITY_TEST_RUNTIME_EFFECTS_SECRET_RESOLVER_CONFIG_PATH=/app/data/delegation/secret_resolver.yaml STABILITY_TEST_RUNTIME_MAIN_BIFROST_VERIFY_ENDPOINTS=0 STABILITY_TEST_RUNTIME_MAIN_CAPABILITIES=market.skill-proof,workflow.orchestration,runtime.main @@ -97,13 +103,14 @@ STABILITY_TEST_RUNTIME_MAIN_OMNIMEMORY_ENABLED=false STABILITY_TEST_RUNTIME_MAIN_OMNIMEMORY_MEMGRAPH_HOST= STABILITY_TEST_RUNTIME_MAIN_PORT=18085 STABILITY_TEST_RUNTIME_MAIN_PUBLISH_INTROSPECTION=true -STABILITY_TEST_RUNTIME_MAIN_SECRET_RESOLVER_CONFIG_JSON='{"enable_convention_fallback":false,"mappings":[{"logical_name":"llm.openrouter.api_key","source":{"source_path":"OPEN_ROUTER_API_KEY","source_type":"env"}},{"logical_name":"llm.glm.api_key","source":{"source_path":"LLM_GLM_API_KEY","source_type":"env"}},{"logical_name":"llm.gemini.api_key","source":{"source_path":"GEMINI_API_KEY","source_type":"env"}}]}' +STABILITY_TEST_RUNTIME_MAIN_SECRET_RESOLVER_CONFIG_JSON='{"enable_convention_fallback":false,"mappings":[{"logical_name":"llm.openrouter.api_key","source":{"source_path":"OPEN_ROUTER_API_KEY","source_type":"env"}},{"logical_name":"llm.glm.api_key","source":{"source_path":"LLM_GLM_API_KEY","source_type":"env"}},{"logical_name":"llm.gemini.api_key","source":{"source_path":"GEMINI_API_KEY","source_type":"env"}},{"logical_name":"llm.vertex.access_token","source":{"source_path":"/run/secrets/vertex_access_token","source_type":"file"}}]}' STABILITY_TEST_RUNTIME_MAIN_SECRET_RESOLVER_CONFIG_PATH=/app/data/delegation/secret_resolver.yaml STABILITY_TEST_RUNTIME_WORKER_BIFROST_VERIFY_ENDPOINTS=0 STABILITY_TEST_RUNTIME_WORKER_CAPABILITIES=workflow.dispatch,contract.update,runtime.worker STABILITY_TEST_RUNTIME_WORKER_OMNIMEMORY_ENABLED=false STABILITY_TEST_RUNTIME_WORKER_OMNIMEMORY_MEMGRAPH_HOST= STABILITY_TEST_RUNTIME_WORKER_PUBLISH_INTROSPECTION=false -STABILITY_TEST_RUNTIME_WORKER_SECRET_RESOLVER_CONFIG_JSON='{"enable_convention_fallback":false,"mappings":[{"logical_name":"llm.openrouter.api_key","source":{"source_path":"OPEN_ROUTER_API_KEY","source_type":"env"}},{"logical_name":"llm.glm.api_key","source":{"source_path":"LLM_GLM_API_KEY","source_type":"env"}},{"logical_name":"llm.gemini.api_key","source":{"source_path":"GEMINI_API_KEY","source_type":"env"}}]}' +STABILITY_TEST_RUNTIME_WORKER_SECRET_RESOLVER_CONFIG_JSON='{"enable_convention_fallback":false,"mappings":[{"logical_name":"llm.openrouter.api_key","source":{"source_path":"OPEN_ROUTER_API_KEY","source_type":"env"}},{"logical_name":"llm.glm.api_key","source":{"source_path":"LLM_GLM_API_KEY","source_type":"env"}},{"logical_name":"llm.gemini.api_key","source":{"source_path":"GEMINI_API_KEY","source_type":"env"}},{"logical_name":"llm.vertex.access_token","source":{"source_path":"/run/secrets/vertex_access_token","source_type":"file"}}]}' STABILITY_TEST_RUNTIME_WORKER_SECRET_RESOLVER_CONFIG_PATH=/app/data/delegation/secret_resolver.yaml STABILITY_TEST_TOPIC_PROVISIONER_MAX_PARTITIONS=1 +STABILITY_TEST_WORKER_REPLICAS=1 diff --git a/docs/audits/2026-06-11-contract-store-durability-audit.md b/docs/audits/2026-06-11-contract-store-durability-audit.md new file mode 100644 index 0000000000..635345e83b --- /dev/null +++ b/docs/audits/2026-06-11-contract-store-durability-audit.md @@ -0,0 +1,115 @@ +# Contract-Store Durability Audit — Cold-Runtime Census Reconstruction + +**Ticket:** OMN-12962 (P1.5) +**Date:** 2026-06-11 +**Scope:** Prove that a COLD runtime can reconstruct the full registered +contract census WITHOUT relying on retained registration events on +`onex.evt.platform.node-registration.v1` (`cleanup.policy=delete`, 7-day +retention). + +## Verdict + +**Durability is PROVEN.** The cold-start contract census is reconstructed +entirely from the **image-bundled filesystem manifest**, independent of Kafka +retention. The delete-retention `node-registration.v1` topic feeds ONLY a +post-freeze, additive dynamic-materialization listener that does not replay +history on a cold start. No fix to the contract store is required. A sweep +check ships to detect the one residual gap: a contract that exists ONLY via the +dynamic path and is therefore not durable. + +## Evidence + +### 1. Cold-start handler resolution is filesystem-sourced, not Kafka-sourced + +`omnibase_infra/src/omnibase_infra/runtime/runtime_host_process.py` +`_resolve_handler_descriptors()` resolves the boot census via +`HandlerSourceResolver`. The default mode is **HYBRID** +(`_load_handler_source_config()` returns `EnumHandlerSourceMode.HYBRID` when no +config is supplied — line ~2790). HYBRID merges: + +- `HandlerBootstrapSource` — hardcoded bootstrap handlers, and +- `PluginLoaderContractSource(contract_paths=...)` — filesystem contract scan. + +`KafkaContractSource` is the active discovery source **only** when +`effective_mode == EnumHandlerSourceMode.KAFKA_EVENTS`. In HYBRID/BOOTSTRAP/ +CONTRACT modes the boot census never consults Kafka. + +### 2. Live runtime confirms filesystem provenance + +Read-only probe of stability-test (`192.168.86.201:18085`): + +``` +GET /v1/introspection/manifest +contract_path = /app/.venv/lib/python3.12/site-packages/omnimarket/nodes/ + node_ab_compare_reducer/contract.yaml +package_name = omnimarket package_version = 0.4.3 +``` + +The registered census points at pip-installed package files baked into the +runtime image — durable, version-pinned, and unaffected by any topic purge. + +### 3. The delete-retention topic feeds only the dynamic listener + +`node-registration.v1` (`SUFFIX_NODE_REGISTRATION`) is consumed by +`_start_dynamic_contract_listener()` / +`_on_dynamic_contract_event()` (OMN-11247). These run AFTER the dispatch engine +is frozen and only ADD handlers on top of the already-resolved filesystem +census via `KafkaContractSource.on_contract_registered()` + +`materialize_cached_contract()`. + +Live topic config (stability-test redpanda, read-only `rpk topic describe`): + +``` +cleanup.policy = delete +retention.ms = 604800000 (7 days) +PARTITION LOG-START-OFFSET HIGH-WATERMARK +0 4 7 +1 1 5 +2 0 1 +3 4 6 +4 2 4 +5 2 4 +``` + +`LOG-START-OFFSET > 0` on five of six partitions proves the log head is ALREADY +truncated by retention — yet the live runtime census is intact, because the +census does not come from this topic. + +### 4. The dynamic listener never replays history on cold start + +The Kafka event bus default offset policy is `latest`: + +- `event_bus/configs/kafka_event_bus_config.yaml`: + `auto_offset_reset: "${KAFKA_AUTO_OFFSET_RESET:-latest}"` +- `ModelKafkaEventBusConfig.auto_offset_reset` default = `"latest"`. + +The dynamic-contract listener subscribes with `EnumConsumerGroupPurpose.CONSUME` +through this bus, so on a cold start with a fresh consumer group it reads from +`latest` — it does NOT replay retained registration events. Consequently a +compacted backup of `node-registration.v1` would NOT change cold-start behavior: +the runtime simply does not re-read that topic to rebuild census. + +## Residual gap (covered by the sweep check) + +A contract that is registered EXCLUSIVELY via the dynamic path — i.e. it is NOT +bundled in the image filesystem manifest — is non-durable: it vanishes on a cold +runtime restart unless its registrant re-publishes a registration event after +the new runtime is live. This is the only retention-adjacent failure mode, and +it is a *provisioning* gap, not a store-durability gap. + +**Enforcement:** `node_runtime_sweep` +(`omnimarket/src/omnimarket/nodes/node_runtime_sweep`) gains a +`NON_DURABLE_CONTRACT` check (`_check_census_durability`). Given the live +registered census and the durable `durable_node_names` manifest, it flags any +live-registered contract absent from the durable source as a CRITICAL finding. +When no manifest is supplied the check is skipped. Unit-tested in +`omnimarket/tests/test_golden_chain_runtime_sweep.py`. + +## Why no store fix ships + +The ticket conditions a fix (compaction or re-seed-on-boot) on the store being +retention-dependent. The audit proves it is NOT: cold-start census comes from +the durable image filesystem, and the dynamic listener resets to `latest` and +never replays history. Adding compaction or a re-seed would be dead code against +the actual cold-start path. The correct, minimal change is the sweep check that +detects dynamic-only contracts — the single real residual gap. diff --git a/docs/runbooks/stability-test-runtime-lane.md b/docs/runbooks/stability-test-runtime-lane.md index 4e4e5cc1b5..61a3adf21f 100644 --- a/docs/runbooks/stability-test-runtime-lane.md +++ b/docs/runbooks/stability-test-runtime-lane.md @@ -128,6 +128,56 @@ uv run pytest tests/unit/infra/test_stability_test_runtime_lane.py -q uv run pytest tests/integration/infra/test_stability_test_runtime_compose_render.py -q ``` +## Worker Replica Census (OMN-12990) + +The base compose sets the runtime-worker deploy replicas to +`${WORKER_REPLICAS:-0}` — a soft default of **zero**. The stability lane's +required state includes a running worker (`GATE_ZERO_PROOF.md`: 4 runtime +containers — main, effects, worker, projection-api). Any plain +`docker compose up`/`recreate` that does not supply the worker replica count +silently scales the worker to zero with no error and no signal. + +The lane override now references the ledgered policy value fail-fast: +`replicas: ${STABILITY_TEST_WORKER_REPLICAS:?...}`. The value lives in +`docker/runtime-policy.env` (rendered from +`contracts/services/runtime_policy.contract.yaml`, +`profiles.stability-test.services.worker.replicas`). A recreate that forgets to +pass `--env-file docker/runtime-policy.env` aborts loudly instead of dropping the +worker. + +Expected-container census for the lane (a missing worker is a FAILURE, not +silence): + +- `omninode-stability-test-runtime` +- `omninode-stability-test-runtime-effects` +- `omninode-stability-test-runtime-worker` +- `omnimarket-stability-test-projection-api` + +Confirm the rendered config keeps the worker before any recreate (validation +only — this does not run `docker compose up`): + +```bash +docker compose \ + --env-file docker/runtime-policy.env \ + -f docker/docker-compose.infra.yml \ + -f docker/docker-compose.stability-test.yml \ + --profile runtime \ + config | grep -A2 'runtime-worker:' | grep -q 'replicas' && echo "worker replicas pinned" +``` + +After an operator-approved recreate, assert the running census matches the +expected set (the worker must be present): + +```bash +uv run python -m omnibase_infra.scripts.verify_container_manifest \ + --catalog-dir docker/catalog \ + --bundles runtime \ + --json +``` + +A non-zero exit (missing `omninode-stability-test-runtime-worker`) is a census +failure that must block the deploy/verify procedure. + ## Explicit Non-Goals For This PR - It does not run `docker compose up`. diff --git a/docs/runbooks/volume-config-drift-and-reseed.md b/docs/runbooks/volume-config-drift-and-reseed.md new file mode 100644 index 0000000000..b7d1903487 --- /dev/null +++ b/docs/runbooks/volume-config-drift-and-reseed.md @@ -0,0 +1,145 @@ +# Volume Config Drift Gate + Re-seed Procedure (OMN-12958) + +## Problem + +The Bifrost delegation contract is rendered once at container startup to a Docker +named volume at `/app/data/delegation/bifrost_delegation.yaml`. That volume +survives `docker compose build` and container recreate, so the **deployed (volume) +copy can silently diverge from the packaged source** shipped in the image. Two +prior incidents (folded into OMN-12945) were caused by exactly this: + +- a stale gemini backend binding persisted on the volume after the packaged + source was updated, and +- an empty `frontier_api` endpoint persisted on the volume. + +There are two competing authorities: + +1. **Packaged source** — `omnimarket/configs/bifrost_delegation.yaml` baked into + the image (canonical home since OMN-10865; legacy fallback + `omnibase_infra/configs/bifrost_delegation.yaml`). +2. **Volume copy** — `/app/data/delegation/bifrost_delegation.yaml`, written once + and then trusted on subsequent boots when it already has populated endpoints + (see `render_bifrost_delegation_contract` short-circuit). + +Additionally, `BIFROST_CONTRACT_PATH=""` (empty) silently **disables** packaged +rendering entirely (the renderer returns `None`). + +> **Scope note.** Reconciling the two authorities into one (Infisical > bootstrap +> overlay precedence) is the P1.2b / OMN-12803 epic. This runbook is the tactical +> drift **gate + re-seed** stopgap: it makes drift *visible and ticketed*, and +> gives operators a deterministic re-seed step. It does not change which authority +> wins at render time. + +## Ratchet shipped with this runbook + +| Ratchet | Where | +|---------|-------| +| (a) Re-seed volume config from packaged source with ledgered overlay diffs only | this runbook (procedure below) | +| (b) Runtime startup logs config provenance (path + sha); health exposes it | `runtime/config_provenance.py`, `runtime/health/health_config_provenance.py`, `runtime/render_bifrost_delegation_contract.py::main` | +| (c) Sweep compares volume sha vs packaged sha → drift = auto-ticket | omnimarket `node_volume_config_drift_sweep` | +| (d) Config provenance appears in proof packets | provenance sidecar `/app/data/delegation/.config_provenance.json` + sweep output | + +## Provenance at runtime + +On every boot, after the contract is rendered, the entrypoint logs a single-line +provenance summary and writes a sidecar JSON next to the deployed contract: + +``` +[entrypoint] config_provenance config=bifrost_delegation status=in-sync \ + deployed_path=/app/data/delegation/bifrost_delegation.yaml \ + deployed_sha256= source_path= source_sha256= +``` + +- `status=DRIFT` and an additional `WARNING` line are emitted when the deployed + sha differs from the packaged-source sha. +- The sidecar `/app/data/delegation/.config_provenance.json` is the machine + surface the drift sweep and proof packets read (no need to re-resolve the + packaged source path inside the container). +- The health component `config_provenance` + (`check_config_provenance_health`) reports: + - `unhealthy` — deployed contract absent, + - `degraded` — drift detected (re-seed required) or packaged source absent, + - `healthy` — deployed sha == source sha. + +> The deployed sha reflects the **post-render** contract (endpoint_url values +> populated from env). Treat the sweep's drift signal as "investigate", then use +> the source-vs-volume diff below to decide whether the divergence is an intended +> overlay or stale config that must be re-seeded. + +## Re-seed procedure (operator) + +> **Live-lane safety.** Re-seeding mutates the named volume. Do **not** run the +> overwrite steps against stability (`18085/18086`), dev (`8085/8086`), or prod +> (`28085/28086`) without the lane's deploy approval. The diff/inspection steps +> are read-only and always safe. + +### 1. Inspect drift (read-only) + +```bash +# On the .201 host, against the target lane's runtime container: +docker exec cat /app/data/delegation/.config_provenance.json +``` + +If `has_drifted` is `true`, diff the deployed volume copy against the packaged +source to capture the **explicit overlay** before overwriting: + +```bash +docker exec sh -c \ + 'diff -u /app/data/delegation/bifrost_delegation.yaml || true' +``` + +### 2. Ledger the overlay (mandatory) + +Re-seed overwrites the volume with packaged source. Any divergence that must be +**kept** (an intentional per-lane overlay) has to be recorded first, or it is +lost. Record the diff and the keep/drop decision in the deploy evidence / proof +packet: + +``` +config: bifrost_delegation +deployed_sha256: +source_sha256: +overlay_diff: +decision: re-seed from packaged source; overlay +``` + +Overlays are expressed through **env-driven render inputs** (e.g. `base_url_env` +backends) or Infisical — never by hand-editing the volume copy. A hand-edited +volume copy is exactly the drift this runbook exists to eliminate. + +### 3. Re-seed (mutating — requires lane approval) + +```bash +# Remove the stale volume copy + sidecar so the next render writes fresh from source: +docker exec sh -c \ + 'rm -f /app/data/delegation/bifrost_delegation.yaml /app/data/delegation/.config_provenance.json' + +# Re-render from packaged source (re-runs the entrypoint render path): +docker exec \ + python -m omnibase_infra.runtime.render_bifrost_delegation_contract +``` + +Then restart the runtime container per the lane's standard deploy step so the +freshly rendered contract is loaded. + +### 4. Verify + +```bash +docker exec cat /app/data/delegation/.config_provenance.json +# expect: has_drifted == false, deployed_sha256 == source_sha256 +``` + +Re-run the drift sweep (below) and confirm zero drift findings. + +## Drift sweep (continuous) + +```bash +cd omnimarket +uv run python -m omnimarket.nodes.node_volume_config_drift_sweep --dry-run +``` + +The sweep reads the provenance sidecar on each runtime lane (or computes +provenance from a probed volume copy + packaged source), classifies +`IN_SYNC` / `DRIFTED` / `DEPLOYED_ABSENT` / `SOURCE_ABSENT`, and — outside +`--dry-run` — auto-creates a Linear ticket for any `DRIFTED` lane. Its JSON output +is suitable for inclusion in proof packets. diff --git a/docs/validation/README.md b/docs/validation/README.md index 37fa588261..89f568eb41 100644 --- a/docs/validation/README.md +++ b/docs/validation/README.md @@ -152,6 +152,16 @@ boundaries. validation. - Topic drift and completeness scripts remain runnable from the repo root. +**Topic authority**: +- Marketplace and runtime topic authority comes from contract YAML, + `topics.yaml` manifests, generated topic enums, and canonical topic constants. +- `scripts/validation/completeness-allowlist.yaml` is not marketplace policy + and must not be used as permanent topic authority. +- Any topic-completeness suppression must be temporary and include `owner`, + `reason`, `created_at`, `expires_at`, and `replacement_authority`. Expired + suppressions stop suppressing violations, and legacy bare `allowlist:` entries + are rejected. + ## Validation Results Each validator returns a `ModelValidationResult[None]` (or `CircularImportValidationResult` for imports) with: diff --git a/pyproject.toml b/pyproject.toml index c95c9d416a..48ec2f940c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,9 +122,11 @@ onex-status = "omnibase_infra.tui.__main__:run_status_tui" verify-container-manifest = "omnibase_infra.scripts.verify_container_manifest:main" [project.entry-points."onex.cli"] +delegate = "omnibase_infra.cli.cli_delegate:delegate_command" kafka = "omnibase_infra.cli.cli_kafka:kafka" node = "omnibase_infra.cli.cli_node:run_node_by_name" run = "omnibase_infra.cli.cli_node:run_node_by_name" +skill = "omnibase_infra.cli.cli_skill:run_skill_by_name" [project.entry-points."onex.backends"] event_bus_kafka = "omnibase_infra.event_bus.event_bus_kafka:EventBusKafka" @@ -169,9 +171,10 @@ override-dependencies = [ ] [tool.uv.sources] -# OMN-12763: bump core pin to dev HEAD (c7d6b5ec, OMN-12762) -# which accepts quality-gate DoD overlay criteria in delegation wire requests. -omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", rev = "c7d6b5ec010692b6724de1d49a0ee5a599d7ee06" } +# OMN-13094: pin core to the merged ArtifactStore (OMN-13093) + +# ModelArtifactRef/ModelSkillResult (OMN-13091) so `onex node --output receipt` +# consumes the canonical content-addressed capture surface. +omnibase-core = { git = "https://github.com/OmniNode-ai/omnibase_core.git", rev = "ae8793bdad96c12a0b47e2f4d1d0f179618553b8" } omnibase-spi = { git = "https://github.com/OmniNode-ai/omnibase_spi.git", branch = "main" } onex-change-control = { git = "https://github.com/OmniNode-ai/onex_change_control.git", branch = "main" } [tool.ruff] @@ -464,6 +467,7 @@ node_baseline_comparison_compute = "omnibase_infra.nodes.node_baseline_compariso node_baselines_batch_compute = "omnibase_infra.nodes.node_baselines_batch_compute" node_build_loop_projection_compute = "omnibase_infra.nodes.node_build_loop_projection_compute" node_build_loop_write_effect = "omnibase_infra.nodes.node_build_loop_write_effect" +node_bus_forwarder_effect = "omnibase_infra.nodes.node_bus_forwarder_effect" node_chain_orchestrator = "omnibase_infra.nodes.node_chain_orchestrator" node_chain_replay_compute = "omnibase_infra.nodes.node_chain_replay_compute" node_chain_retrieval_effect = "omnibase_infra.nodes.node_chain_retrieval_effect" diff --git a/scripts/check_dispatcher_route_coverage.py b/scripts/check_dispatcher_route_coverage.py new file mode 100644 index 0000000000..c747dba914 --- /dev/null +++ b/scripts/check_dispatcher_route_coverage.py @@ -0,0 +1,402 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Dispatcher route-coverage gate (OMN-12858 / OMN-12879 / OMN-12880). + +Static analysis: for every contract that subscribes to a command topic +(``onex.cmd.*``), assert that the contract declares a dispatcher route via +``handler_routing`` or ``runtime_dispatch``. Missing wiring causes the +command to go to DLQ silently — this gate catches that regression before +it reaches the runtime. + +Coverage included (OMN-12880): +- Primary ``event_bus.subscribe_topics`` entries. +- ``compatibility_publish_topics`` entries are treated as sender-side only + (a publisher does not require a handler route for topics it emits to + compatibility surfaces — routes are required only on the subscriber side). + +Changed-command-topic coverage (OMN-12879): +- When ``--changed-contracts`` paths are provided (space-separated list from + CI diff), only those contracts are checked for missing routes. This makes + failures actionable to the PR author without blocking on pre-existing + ratchet violations. + +Usage:: + + # Full scan (both repos checked out as siblings) + uv run python scripts/check_dispatcher_route_coverage.py \\ + --contracts-dir src/omnibase_infra/nodes \\ + --extra-contracts-dir ../omnimarket/src/omnimarket/nodes + + # CI changed-contract mode (OMN-12879) + uv run python scripts/check_dispatcher_route_coverage.py \\ + --contracts-dir src/omnibase_infra/nodes \\ + --extra-contracts-dir ../omnimarket/src/omnimarket/nodes \\ + --changed-contracts "src/.../contract.yaml src/.../contract.yaml" + + # Verbose output + uv run python scripts/check_dispatcher_route_coverage.py \\ + --contracts-dir src/omnibase_infra/nodes \\ + --extra-contracts-dir ../omnimarket/src/omnimarket/nodes \\ + --verbose + +Exit codes: + 0 = all command-topic subscriptions have a dispatcher route (or are allowlisted) + 1 = one or more command-topic subscriptions are missing dispatcher wiring + +[OMN-12858, OMN-12879, OMN-12880] +""" + +from __future__ import annotations + +import argparse +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml + +# --------------------------------------------------------------------------- +# Allow running as a standalone script when omnibase_infra is not installed. +# --------------------------------------------------------------------------- +_REPO_ROOT = Path(__file__).resolve().parent.parent +_SRC_DIR = _REPO_ROOT / "src" +if _SRC_DIR.is_dir() and str(_SRC_DIR) not in sys.path: + sys.path.insert(0, str(_SRC_DIR)) + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- +_CMD_TOPIC_PREFIX = "onex.cmd." + +# --------------------------------------------------------------------------- +# Allowlist: command-topic subscriptions that are expected to have no +# handler_routing / runtime_dispatch because they are consumed by an +# internal broker outside the standard dispatch surface. +# Format: "topic": "reason | owner | expiry" +# --------------------------------------------------------------------------- +_ALLOWLISTED_CMD_TOPICS: dict[str, str] = { + # Pattern-B dispatch broker owns its own consumption model; the contract + # declares the subscription but RuntimePatternBBroker processes the + # messages directly, bypassing handler_routing. + "onex.cmd.omnibase-infra.pattern-b-dispatch.v1": ( + "Consumed by RuntimePatternBBroker directly, not via handler_routing " + "| owner: jonah | expiry: 2027-01-01" + ), + # Transitional HTTP bridge (OMN-2756): subscribes for observability/future + # Kafka-native switch but is currently an HTTP-dispatch node. Dispatcher + # route will be added when the node is migrated to native Kafka dispatch. + # Ref: node_contract_resolver_bridge/contract.yaml — metadata.transitional=true + "onex.cmd.platform.contract-resolve-requested.v1": ( + "Transitional HTTP bridge node_contract_resolver_bridge (OMN-2756); " + "processes via FastAPI not Kafka handler_routing | owner: jonah | expiry: 2027-01-01" + ), +} + + +# --------------------------------------------------------------------------- +# Data models +# --------------------------------------------------------------------------- +@dataclass(frozen=True) +class ContractInfo: + """Parsed subset of a contract.yaml needed for route coverage checking.""" + + name: str + contract_path: Path + subscribe_topics: tuple[str, ...] + has_handler_routing: bool + has_runtime_dispatch: bool + + @property + def has_dispatcher_route(self) -> bool: + return self.has_handler_routing or self.has_runtime_dispatch + + +@dataclass +class RouteCoverageFailure: + """One missing-route finding.""" + + topic: str + contract_name: str + contract_path: Path + + +@dataclass +class RouteCoverageReport: + """Result of the full scan.""" + + scanned: list[ContractInfo] = field(default_factory=list) + failures: list[RouteCoverageFailure] = field(default_factory=list) + allowlisted: list[tuple[str, str]] = field(default_factory=list) + + @property + def passed(self) -> bool: + return len(self.failures) == 0 + + +# --------------------------------------------------------------------------- +# YAML parsing helpers +# --------------------------------------------------------------------------- +def _parse_topic_list(raw: Any) -> tuple[str, ...]: + """Parse a subscribe_topics or publish_topics value from YAML. + + Handles both flat-string lists and schema-B structured dicts:: + + subscribe_topics: + - "onex.cmd.foo.bar.v1" + - topic: "onex.cmd.foo.baz.v1" + operation: "ingest" + """ + if not raw: + return () + topics: list[str] = [] + for entry in raw: + if isinstance(entry, str): + topics.append(entry) + elif isinstance(entry, dict): + t = entry.get("topic", "") + if isinstance(t, str) and t: + topics.append(t) + return tuple(topics) + + +def _parse_contract(path: Path) -> ContractInfo | None: + """Parse a contract.yaml and return a ContractInfo, or None on error.""" + try: + raw: dict[str, Any] = yaml.safe_load(path.read_text()) or {} + except (OSError, yaml.YAMLError): + return None + + name: str = raw.get("name") or path.parent.name + + # event_bus.subscribe_topics + event_bus: dict[str, Any] = raw.get("event_bus") or {} + subscribe_topics = _parse_topic_list(event_bus.get("subscribe_topics")) + + # Dispatcher route declarations + has_handler_routing = bool(raw.get("handler_routing")) + has_runtime_dispatch = bool(raw.get("runtime_dispatch")) + + return ContractInfo( + name=name, + contract_path=path, + subscribe_topics=subscribe_topics, + has_handler_routing=has_handler_routing, + has_runtime_dispatch=has_runtime_dispatch, + ) + + +def _scan_contracts_dir(contracts_dir: Path) -> list[ContractInfo]: + """Recursively find and parse all contract.yaml files under a directory.""" + contracts: list[ContractInfo] = [] + for p in sorted(contracts_dir.rglob("contract.yaml")): + info = _parse_contract(p) + if info is not None: + contracts.append(info) + return contracts + + +# --------------------------------------------------------------------------- +# Core checker +# --------------------------------------------------------------------------- +def check_route_coverage( + contracts_dirs: list[Path], + *, + changed_contract_paths: frozenset[Path] | None = None, + verbose: bool = False, +) -> RouteCoverageReport: + """Run the dispatcher route-coverage check. + + Args: + contracts_dirs: List of directories to scan for contract.yaml files. + changed_contract_paths: When provided (OMN-12879 mode), only report + failures for contracts whose path is in this set. The full scan + populates ``scanned``; the filter only affects ``failures``. + Pass ``None`` to check all contracts (full-scan mode). + verbose: Print per-contract debug lines to stdout. + + Returns: + A :class:`RouteCoverageReport` with the scan results. + """ + report = RouteCoverageReport() + + for contracts_dir in contracts_dirs: + if not contracts_dir.is_dir(): + if verbose: + print(f" [SKIP missing dir] {contracts_dir}", flush=True) + continue + for info in _scan_contracts_dir(contracts_dir): + report.scanned.append(info) + + for topic in info.subscribe_topics: + if not topic.startswith(_CMD_TOPIC_PREFIX): + continue # only command topics require dispatch wiring + + if topic in _ALLOWLISTED_CMD_TOPICS: + report.allowlisted.append((topic, _ALLOWLISTED_CMD_TOPICS[topic])) + if verbose: + print( + f" [ALLOWLISTED] {info.name}: {topic}", + flush=True, + ) + continue + + # OMN-12879: changed-contract mode — skip unchanged contracts. + if changed_contract_paths is not None: + if info.contract_path not in changed_contract_paths: + if verbose: + print( + f" [SKIP unchanged] {info.name}: {topic}", + flush=True, + ) + continue + + if info.has_dispatcher_route: + if verbose: + route_kind = ( + "handler_routing" + if info.has_handler_routing + else "runtime_dispatch" + ) + print( + f" [OK {route_kind}] {info.name}: {topic}", + flush=True, + ) + continue + + # Failure: subscribed command topic with no dispatcher route. + report.failures.append( + RouteCoverageFailure( + topic=topic, + contract_name=info.name, + contract_path=info.contract_path, + ) + ) + if verbose: + print( + f" [FAIL] {info.name}: {topic} -- no dispatcher route", + flush=True, + ) + + return report + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Check that every command-topic subscription has a dispatcher route " + "(OMN-12858 / OMN-12879 / OMN-12880)." + ), + ) + parser.add_argument( + "--contracts-dir", + type=Path, + default=_REPO_ROOT / "src" / "omnibase_infra" / "nodes", + help="Primary contracts directory to scan (default: src/omnibase_infra/nodes)", + ) + parser.add_argument( + "--extra-contracts-dir", + dest="extra_contracts_dirs", + action="append", + type=Path, + default=[], + help=( + "Additional contract directories (repeatable; " + "e.g. ../omnimarket/src/omnimarket/nodes)" + ), + ) + parser.add_argument( + "--changed-contracts", + type=str, + default="", + help=( + "OMN-12879: space-separated list of contract.yaml paths that changed in " + "this PR diff. When non-empty, only those contracts are checked." + ), + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Emit per-contract status lines", + ) + args = parser.parse_args() + + contracts_dirs: list[Path] = [args.contracts_dir, *args.extra_contracts_dirs] + + # OMN-12879: parse changed-contract paths + changed_contract_paths: frozenset[Path] | None = None + if args.changed_contracts.strip(): + changed_contract_paths = frozenset( + Path(p).resolve() for p in args.changed_contracts.split() if p.strip() + ) + + print( + "================================================================", + flush=True, + ) + print( + "Dispatcher Route-Coverage Gate (OMN-12858 / OMN-12879 / OMN-12880)", + flush=True, + ) + print( + "================================================================", + flush=True, + ) + for d in contracts_dirs: + print(f" scanning: {d}", flush=True) + if changed_contract_paths is not None: + print( + f" changed-contract mode (OMN-12879): {len(changed_contract_paths)} paths", + flush=True, + ) + + report = check_route_coverage( + contracts_dirs, + changed_contract_paths=changed_contract_paths, + verbose=args.verbose, + ) + + print( + f"\nContracts scanned: {len(report.scanned)}" + f" | allowlisted: {len(report.allowlisted)}" + f" | failures: {len(report.failures)}", + flush=True, + ) + + if report.failures: + print( + "\nFAILURES — command topics subscribed without a dispatcher route:\n", + flush=True, + ) + for f in report.failures: + print(f" topic: {f.topic}") + print(f" contract: {f.contract_name}") + print(f" path: {f.contract_path}") + print(" fix: add handler_routing or runtime_dispatch to the contract") + print() + print( + f"ERROR: {len(report.failures)} command topic(s) lack dispatcher wiring. " + "Messages on these topics will go to DLQ at runtime.", + flush=True, + ) + return 1 + + if changed_contract_paths is not None: + print( + "\nAll changed command-topic subscriptions have dispatcher routes. PASS.", + flush=True, + ) + else: + print( + "\nAll command-topic subscriptions have dispatcher routes. PASS.", + flush=True, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check_runtime_image_identity.py b/scripts/check_runtime_image_identity.py new file mode 100644 index 0000000000..6b2b8a998f --- /dev/null +++ b/scripts/check_runtime_image_identity.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Enforce runtime image identity stamping (OMN-12965). + +The deep dive found the main runtime image stamped +``org.opencontainers.image.version=0.1.0`` with a blank +``org.opencontainers.image.revision`` after workspace rebuilds. A blank identity +degrades every proof packet — the runtime SHA + image digest are required +citations in accepted evidence. + +This is the enforcement ratchet (pre-commit + CI) that prevents the build paths +from regressing to a blank/placeholder identity. It is a static check: no docker +build, no network. It verifies: + +1. ``docker/Dockerfile.runtime`` labels ``org.opencontainers.image.revision`` + from ``VCS_REF`` and ``org.opencontainers.image.version`` from + ``RUNTIME_VERSION``, and carries a workspace-mode guard that fails the build + when either would be blank/``unknown``/placeholder ``0.1.0``. +2. The operator rebuild path ``cmd_up`` in + ``src/omnibase_infra/docker/catalog/cli.py`` stamps the identity quad via + ``_image_identity_build_args()`` rather than a bare ``GIT_SHA`` build-arg. + +Usage:: + + # From the repo root + uv run python scripts/check_runtime_image_identity.py + +Exit codes: + 0 — all checks passed + 1 — one or more checks failed (stderr has details) +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[1] +_DOCKERFILE = _REPO_ROOT / "docker" / "Dockerfile.runtime" +_CLI = _REPO_ROOT / "src" / "omnibase_infra" / "docker" / "catalog" / "cli.py" + + +def _collapse_continuations(text: str) -> str: + """Collapse shell ``\\``-continued physical lines into single logical lines.""" + return re.sub(r"\\\n\s*", " ", text) + + +def _check_dockerfile() -> list[str]: + errors: list[str] = [] + if not _DOCKERFILE.is_file(): + return [f"missing {_DOCKERFILE}"] + collapsed = _collapse_continuations(_DOCKERFILE.read_text(encoding="utf-8")) + + if 'org.opencontainers.image.revision="${VCS_REF}"' not in collapsed: + errors.append( + "Dockerfile.runtime must label org.opencontainers.image.revision " + 'from "${VCS_REF}" (OMN-12965).' + ) + if 'org.opencontainers.image.version="${RUNTIME_VERSION}"' not in collapsed: + errors.append( + "Dockerfile.runtime must label org.opencontainers.image.version " + 'from "${RUNTIME_VERSION}" (OMN-12965).' + ) + + guard = re.search( + r'if \[ "\$\{BUILD_SOURCE\}" = "workspace" \];.*?VCS_REF.*?' + r"exit 64.*?RUNTIME_VERSION.*?0\.1\.0.*?exit 64", + collapsed, + re.DOTALL, + ) + if not guard: + errors.append( + "Dockerfile.runtime must guard workspace builds against blank VCS_REF " + "and placeholder RUNTIME_VERSION=0.1.0 (OMN-12965)." + ) + return errors + + +def _function_body(source: str, name: str) -> str | None: + lines = source.splitlines() + start = next( + (i for i, line in enumerate(lines) if line.startswith(f"def {name}(")), + None, + ) + if start is None: + return None + end = next( + ( + i + for i in range(start + 1, len(lines)) + if lines[i] and not lines[i][0].isspace() + ), + len(lines), + ) + return "\n".join(lines[start:end]) + + +def _check_cli() -> list[str]: + errors: list[str] = [] + if not _CLI.is_file(): + return [f"missing {_CLI}"] + source = _CLI.read_text(encoding="utf-8") + + body = _function_body(source, "cmd_up") + if body is None: + return ["cmd_up not found in cli.py"] + + if "_image_identity_build_args()" not in body: + errors.append( + "cmd_up --build must stamp the identity quad via " + "_image_identity_build_args() (OMN-12965)." + ) + if "GIT_SHA=" in body: + errors.append( + "cmd_up must not hand-roll a GIT_SHA build-arg; delegate to " + "_image_identity_build_args() so VCS_REF + RUNTIME_VERSION are " + "stamped (OMN-12965)." + ) + return errors + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.parse_args(argv) + + errors = _check_dockerfile() + _check_cli() + if errors: + print("Runtime image identity check FAILED (OMN-12965):", file=sys.stderr) + for err in errors: + print(f" - {err}", file=sys.stderr) + return 1 + print("Runtime image identity check passed (OMN-12965).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/infra-node-allowlist.txt b/scripts/ci/infra-node-allowlist.txt index 0399e36cce..dc8182f126 100644 --- a/scripts/ci/infra-node-allowlist.txt +++ b/scripts/ci/infra-node-allowlist.txt @@ -25,6 +25,9 @@ src/omnibase_infra/nodes/node_build_loop_projection_compute/handlers/__init__.py src/omnibase_infra/nodes/node_build_loop_projection_compute/handlers/handler_build_loop_projection.py # OMN-10956 baseline: approved existing infra handler path; new business-domain handlers belong in omnimarket. src/omnibase_infra/nodes/node_build_loop_write_effect/handlers/__init__.py # OMN-10956 baseline: approved existing infra handler path; new business-domain handlers belong in omnimarket. src/omnibase_infra/nodes/node_build_loop_write_effect/handlers/handler_build_loop_append.py # OMN-10956 baseline: approved existing infra handler path; new business-domain handlers belong in omnimarket. +src/omnibase_infra/nodes/node_bus_forwarder_effect/handlers/__init__.py # OMN-12909: infra-owned tenant gateway bus bridge; enforces transport boundary, not business-domain logic. +src/omnibase_infra/nodes/node_bus_forwarder_effect/handlers/handler_consume_inbound.py # OMN-12909: infra-owned tenant gateway bus bridge; strips cloud wire prefixes before local bus republish. +src/omnibase_infra/nodes/node_bus_forwarder_effect/handlers/handler_forward_outbound.py # OMN-12909: infra-owned tenant gateway bus bridge; validates local topics before cloud bus forwarding. src/omnibase_infra/nodes/node_chain_orchestrator/handlers/__init__.py # OMN-10956 baseline: approved existing infra handler path; new business-domain handlers belong in omnimarket. src/omnibase_infra/nodes/node_chain_orchestrator/handlers/handler_chain_learn_initiate.py # OMN-10956 baseline: approved existing infra handler path; new business-domain handlers belong in omnimarket. src/omnibase_infra/nodes/node_chain_orchestrator/handlers/handler_chain_replay_complete.py # OMN-10956 baseline: approved existing infra handler path; new business-domain handlers belong in omnimarket. diff --git a/scripts/deploy-agent/tests/unit/test_executor_build_source.py b/scripts/deploy-agent/tests/unit/test_executor_build_source.py index 97237a7024..eaca3c868c 100644 --- a/scripts/deploy-agent/tests/unit/test_executor_build_source.py +++ b/scripts/deploy-agent/tests/unit/test_executor_build_source.py @@ -85,6 +85,18 @@ def fake_run(cmd: list[str], timeout: int, **kwargs) -> subprocess.CompletedProc assert "EXPECTED_BUILD_SOURCE=workspace" in build_cmd assert "OMNI_HOME=/data/omninode/omni_home" in build_cmd + # OMN-12965: the workspace build must stamp the full OCI image-identity quad + # so org.opencontainers.image.{version,revision,created} populate. A blank + # revision / placeholder version degrades every proof packet. + assert "GIT_SHA=abc1234" in build_cmd + assert "VCS_REF=abc1234" in build_cmd + runtime_version_args = [a for a in build_cmd if a.startswith("RUNTIME_VERSION=")] + assert runtime_version_args, "missing RUNTIME_VERSION build-arg (OMN-12965)" + assert runtime_version_args[0] != "RUNTIME_VERSION=0.1.0", ( + "RUNTIME_VERSION must come from pyproject, not the Dockerfile placeholder" + ) + assert any(a.startswith("BUILD_DATE=") for a in build_cmd) + def test_unknown_build_source_fails_before_compose_build( monkeypatch: pytest.MonkeyPatch, diff --git a/scripts/deploy-agent/tests/unit/test_executor_workspace_provenance.py b/scripts/deploy-agent/tests/unit/test_executor_workspace_provenance.py index 010bc7698d..f4c695e278 100644 --- a/scripts/deploy-agent/tests/unit/test_executor_workspace_provenance.py +++ b/scripts/deploy-agent/tests/unit/test_executor_workspace_provenance.py @@ -20,6 +20,10 @@ PROVENANCE_SCRIPT = ( REPO_ROOT / "scripts" / "runtime_build" / "compute_workspace_provenance.py" ) +# Pin-resolution helper imported by the provenance script (OMN-12989). Isolated +# test copies of the provenance script must place this alongside the copy so the +# sys.path-relative import resolves — mirrors the Dockerfile's dual COPY. +PIN_SCRIPT = REPO_ROOT / "scripts" / "runtime_build" / "resolve_workspace_pins.py" STAGE_SCRIPT = REPO_ROOT / "scripts" / "runtime_build" / "stage_workspace.sh" @@ -222,7 +226,15 @@ def fake_run(cmd: list[str], timeout: int, **kwargs) -> subprocess.CompletedProc build_cmd = captured_cmds[0] assert "VCS_REF=deadbeef" in build_cmd - assert "RUNTIME_VERSION=0.38.0" in build_cmd + # RUNTIME_VERSION is sourced from the repo's own pyproject — read it the same + # way the executor does so this assertion never drifts on a version bump. + import tomllib + + pyproject = tomllib.loads( + (REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8") + ) + expected_runtime_version = pyproject["project"]["version"] + assert f"RUNTIME_VERSION={expected_runtime_version}" in build_cmd assert any(a.startswith("BUILD_DATE=") for a in build_cmd) @@ -261,6 +273,10 @@ def test_provenance_script_passes_with_valid_local_installs( patched_script = tmp_path / "compute_workspace_provenance_test.py" patched_script.write_text(script_src, encoding="utf-8") + # Co-locate the pin-resolution helper so the script's import resolves. + (tmp_path / "resolve_workspace_pins.py").write_text( + PIN_SCRIPT.read_text(encoding="utf-8"), encoding="utf-8" + ) # Patch importlib.metadata to simulate local installs mock_dist = MagicMock() @@ -322,6 +338,9 @@ def test_provenance_script_missing_sibling_repo_exits_nonzero( patched_script = tmp_path / "compute_prov_missing.py" patched_script.write_text(script_src, encoding="utf-8") + (tmp_path / "resolve_workspace_pins.py").write_text( + PIN_SCRIPT.read_text(encoding="utf-8"), encoding="utf-8" + ) import sys @@ -378,3 +397,147 @@ def test_dockerfile_workspace_copy_and_provenance_present() -> None: assert "build-provenance.json" in dockerfile assert "com.omninode.workspace_provenance_manifest" in dockerfile assert "--if-present" not in dockerfile + + +def test_dockerfile_copies_pin_resolution_helper() -> None: + """OMN-12989: the pin-resolution helper must be copied beside the provenance + script so the in-image import resolves.""" + dockerfile = (REPO_ROOT / "docker" / "Dockerfile.runtime").read_text( + encoding="utf-8" + ) + assert "scripts/runtime_build/resolve_workspace_pins.py" in dockerfile + assert "/workspace/resolve_workspace_pins.py" in dockerfile + + +def _patched_provenance(tmp_path: Path, sibling_dir: Path, manifest_path: Path) -> Path: + """Write an isolated copy of the provenance script + helper, path-patched.""" + script_src = PROVENANCE_SCRIPT.read_text(encoding="utf-8") + script_src = script_src.replace( + 'SIBLING_REPOS_DIR = Path("/workspace/sibling-repos")', + f'SIBLING_REPOS_DIR = Path("{sibling_dir}")', + ).replace( + 'OUTPUT_MANIFEST = Path("/app/build-provenance.json")', + f'OUTPUT_MANIFEST = Path("{manifest_path}")', + ) + patched = tmp_path / "compute_prov_pins.py" + patched.write_text(script_src, encoding="utf-8") + (tmp_path / "resolve_workspace_pins.py").write_text( + PIN_SCRIPT.read_text(encoding="utf-8"), encoding="utf-8" + ) + return patched + + +_OMNIMARKET_LOCK = """ +version = 1 +requires-python = ">=3.12" + +[[package]] +name = "omnibase-compat" +version = "0.5.1" +source = { git = "https://github.com/OmniNode-ai/omnibase_compat.git?rev=4d887307aae34d9d40d389ba91070cb411ce3df5#4d887307aae34d9d40d389ba91070cb411ce3df5" } + +[[package]] +name = "omnibase-infra" +version = "0.38.1" +source = { git = "https://github.com/OmniNode-ai/omnibase_infra.git?rev=e2dbdc950540df8bc59ca4370b2d4a0f5b8d6c59#e2dbdc950540df8bc59ca4370b2d4a0f5b8d6c59" } + +[[package]] +name = "onex-change-control" +version = "0.9.0" +source = { git = "https://github.com/OmniNode-ai/onex_change_control.git?rev=4877d3c223517cb0c7e1eca462ba0f4d38916314" } + +[[package]] +name = "omnimarket" +version = "0.4.3" +source = { editable = "." } +""" + + +def _stage_siblings(sibling_dir: Path) -> None: + """Create the three workspace siblings (compat, occ, omnimarket) at lock pin.""" + for repo, version in ( + ("omnibase_compat", "0.5.1"), + ("onex_change_control", "0.9.0"), + ("omnimarket", "0.4.3"), + ): + d = sibling_dir / repo + d.mkdir(parents=True) + name = repo.replace("_", "-") + (d / "pyproject.toml").write_text( + f'[project]\nname = "{name}"\nversion = "{version}"\n', encoding="utf-8" + ) + (sibling_dir / "omnimarket" / "uv.lock").write_text( + _OMNIMARKET_LOCK, encoding="utf-8" + ) + + +def test_provenance_emits_pin_comparison_block(tmp_path: Path) -> None: + """OMN-12989: the manifest carries an expected-vs-actual pin_comparison block.""" + import sys + + sibling_dir = tmp_path / "workspace" / "sibling-repos" + sibling_dir.mkdir(parents=True) + _stage_siblings(sibling_dir) + manifest_path = tmp_path / "build-provenance.json" + patched = _patched_provenance(tmp_path, sibling_dir, manifest_path) + + # Host infra version resolved from importlib.metadata is at/above pin in the + # test venv (>=0.38.1), so the infra self-check passes; here we only assert + # the comparison block is present and the staged siblings are recorded. + subprocess.run( + [sys.executable, str(patched)], capture_output=True, text=True, check=False + ) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + assert "pin_comparison" in manifest + by_pkg = {e["package"]: e for e in manifest["pin_comparison"]} + # omnibase-compat staged at exactly the lock pin. + assert by_pkg["omnibase-compat"]["status"] == "exact" + assert by_pkg["omnibase-compat"]["match"] is True + # The host infra self-check appears only when omnibase-infra is installed in + # the running venv (always true in the build image; not in the deploy-agent + # venv). When present it must carry the lock-pinned expected version. + if "omnibase-infra" in by_pkg: + assert by_pkg["omnibase-infra"]["expected_version"] == "0.38.1" + + +def test_build_comparisons_flags_host_infra_regression(tmp_path: Path) -> None: + """OMN-12989 (unit): the exact crash — host infra 0.37.0 vs lock pin 0.38.1. + + Exercises build_comparisons directly with an explicit host-infra actual + version so the regression detection does not depend on the test venv. + """ + mod = _load_resolve_module() + omnimarket = tmp_path / "omnimarket" + omnimarket.mkdir() + (omnimarket / "uv.lock").write_text(_OMNIMARKET_LOCK, encoding="utf-8") + + # Synthesize a "host infra" pyproject tree at the stale crash version. + infra = tmp_path / "omnibase_infra" + infra.mkdir() + (infra / "pyproject.toml").write_text( + '[project]\nname = "omnibase-infra"\nversion = "0.37.0"\n', encoding="utf-8" + ) + + comparisons = mod.build_comparisons( + lock_path=omnimarket / "uv.lock", + siblings={"omnibase-infra": infra}, + ) + infra_cmp = next(c for c in comparisons if c.package == "omnibase-infra") + assert infra_cmp.status == "regression" + with pytest.raises(mod.WorkspacePinError): + mod.assert_pins_satisfied(comparisons) + + +def _load_resolve_module(): + import importlib.util + import sys + + spec = importlib.util.spec_from_file_location( + "resolve_workspace_pins_provtest", str(PIN_SCRIPT) + ) + assert spec is not None + mod = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[spec.name] = mod + spec.loader.exec_module(mod) # type: ignore[attr-defined] + return mod diff --git a/scripts/deploy-agent/tests/unit/test_runtime_worker_census.py b/scripts/deploy-agent/tests/unit/test_runtime_worker_census.py new file mode 100644 index 0000000000..5db099627a --- /dev/null +++ b/scripts/deploy-agent/tests/unit/test_runtime_worker_census.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Census ratchet for the runtime-worker (OMN-12988, config-drift OMN-12945). + +The stability-test lane's required state includes a running ``runtime-worker`` +container (4-container census: main, effects, worker, projection-api). The base +``docker-compose.infra.yml`` defaults the worker to ``replicas: 0`` +(``${WORKER_REPLICAS:-0}``), so a plain compose ``up``/recreate that does not +set replicas silently drops the worker — zero errors, zero signal. + +Two ratchets guard against that: + +1. ``runtime-worker`` MUST remain in the deploy-agent RUNTIME-scope census so + ``verify_containers_up`` flags it ``missing`` (deploy failure) rather than + silently tolerating its absence. A ``replicas: 0`` worker produces no + container, so ``docker compose ps`` omits it and the census check fails — + which is the desired loud signal. +2. The stability-test compose override MUST pin the worker to ``replicas: 1`` as + a literal value (no ``${VAR:-default}`` env indirection), so a lane recreate + cannot silently scale it to 0 via a stray exported env var. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +from deploy_agent.events import SCOPE_SERVICES, Scope, services_for_scope + +pytestmark = pytest.mark.unit + +REPO_ROOT = Path(__file__).resolve().parents[4] +STABILITY_COMPOSE_PATH = REPO_ROOT / "docker" / "docker-compose.stability-test.yml" + + +def test_runtime_worker_in_runtime_scope_census() -> None: + """A missing worker must be a deploy failure, not silence. + + ``runtime-worker`` is part of the required runtime census; the deploy-agent + verifier polls for every service in this list and raises when any is + missing. A ``replicas: 0`` worker is absent from ``docker compose ps`` and + is therefore reported missing — the loud signal we want. + """ + assert "runtime-worker" in SCOPE_SERVICES[Scope.RUNTIME] + assert "runtime-worker" in services_for_scope(Scope.RUNTIME) + assert "runtime-worker" in services_for_scope(Scope.FULL) + + +def test_stability_override_pins_worker_replicas_to_literal_one() -> None: + """The stability override pins worker ``replicas: 1`` as a literal. + + Env indirection (``${STABILITY_TEST_WORKER_REPLICAS:-1}``) is a silent-drop + surface: an exported ``STABILITY_TEST_WORKER_REPLICAS=0`` or a future edit + removing the ``:-1`` fallback would scale the worker to 0 with no signal. + The pin must be the literal integer ``1``. + """ + # The stability override uses docker compose custom tags (!override, + # !!merge) that yaml.safe_load cannot parse, so assert against the raw text: + # locate the runtime-worker service block and its deploy.replicas line. + raw = STABILITY_COMPOSE_PATH.read_text(encoding="utf-8") + + worker_block = re.search( + r"^ runtime-worker:\n((?: .*\n|\n)*)", + raw, + re.MULTILINE, + ) + assert worker_block is not None, ( + f"could not locate the runtime-worker service block in {STABILITY_COMPOSE_PATH}" + ) + + replicas_line = re.search( + r"^ replicas:\s*(.+?)\s*$", + worker_block.group(1), + re.MULTILINE, + ) + assert replicas_line is not None, ( + "stability-test runtime-worker must declare deploy.replicas explicitly; " + f"none found in:\n{worker_block.group(1)}" + ) + + value = replicas_line.group(1) + assert value == "1", ( + "stability-test runtime-worker must pin deploy.replicas to the literal " + f"integer 1 (no env-interpolation default that could silently scale to " + f"0), got {value!r}" + ) diff --git a/scripts/deploy-runtime.sh b/scripts/deploy-runtime.sh index 5082c4de7a..e20c3575a0 100755 --- a/scripts/deploy-runtime.sh +++ b/scripts/deploy-runtime.sh @@ -564,6 +564,78 @@ stage_workspace_if_needed() { log_step "Stage Workspace Sibling Repos" log_cmd "OMNI_HOME=${omni_home} bash ${stage_script}" (cd "${repo_root}" && OMNI_HOME="${omni_home}" bash "${stage_script}") + + check_sibling_lock_pins "${repo_root}" "${omni_home}" +} + +check_sibling_lock_pins() { + # Fail-fast preflight (OMN-12987): every vendored sibling's version/SHA must + # match the consuming repo's (omnimarket) uv.lock pin. The 2026-06-11 + # stability crash shipped a 13-day-stale infra 0.37.0 because the build + # ignored the dev lock; this guard refuses to build a stale image. + local repo_root="$1" + local omni_home="$2" + local guard="${repo_root}/scripts/runtime_build/check_sibling_lock_pins.py" + if [[ ! -f "${guard}" ]]; then + log_error "Sibling lock-pin preflight not found: ${guard}" + log_error "Cannot verify vendored siblings match the consuming lock. Aborting." + exit 1 + fi + + log_step "Sibling Lock-Pin Preflight (OMN-12987)" + # Write under sibling-repos/ so it rides along with the directory the + # Dockerfile already COPYs into the build image (no extra COPY needed). + mkdir -p "${repo_root}/workspace/sibling-repos" + local provenance_out="${repo_root}/workspace/sibling-repos/.sibling-lock-pins.json" + local python_bin + if [[ -x "${repo_root}/.venv/bin/python" ]]; then + python_bin="${repo_root}/.venv/bin/python" + elif command -v uv &>/dev/null; then + python_bin="uv-run" + elif command -v python3 &>/dev/null; then + python_bin="python3" + else + log_error "No Python interpreter available to run the sibling lock-pin preflight." + exit 1 + fi + + # The check_sibling_lock_pins.py interface changed under OMN-12977/12987: + # the original single-output flag was removed in favor of --lock (required, + # the pin authority), repeatable --repo PACKAGE=PATH (the canonical clones + # the build vendors), and --output (where to write the comparison JSON). + # The consuming repo's uv.lock (omnimarket) is the pin authority. + local lock_path="${omni_home}/omnimarket/uv.lock" + local guard_args=( + --lock "${lock_path}" + --repo "omnibase-infra=${omni_home}/omnibase_infra" + --repo "omnibase-core=${omni_home}/omnibase_core" + --repo "omnibase-spi=${omni_home}/omnibase_spi" + --repo "omnibase-compat=${omni_home}/omnibase_compat" + --repo "onex-change-control=${omni_home}/onex_change_control" + --output "${provenance_out}" + ) + # Operator override (OMN-12977): ALLOW_SIBLING_PIN_DRIFT=1 records drift in + # the provenance artifact and proceeds instead of aborting. Never the default. + if [[ "${ALLOW_SIBLING_PIN_DRIFT:-0}" == "1" ]]; then + guard_args+=(--allow-drift) + log_warn "ALLOW_SIBLING_PIN_DRIFT=1 -- passing --allow-drift to sibling lock-pin preflight (OMN-12977)" + fi + + log_cmd "OMNI_HOME=${omni_home} ${guard} ${guard_args[*]}" + if [[ "${python_bin}" == "uv-run" ]]; then + if ! OMNI_HOME="${omni_home}" uv run --project "${repo_root}" python "${guard}" \ + "${guard_args[@]}"; then + log_error "Sibling lock-pin preflight FAILED. Refusing to build a stale image." + exit 1 + fi + else + if ! OMNI_HOME="${omni_home}" "${python_bin}" "${guard}" \ + "${guard_args[@]}"; then + log_error "Sibling lock-pin preflight FAILED. Refusing to build a stale image." + exit 1 + fi + fi + log_info "Sibling lock-pin preflight passed: all vendored siblings match the lock." } check_git_dirty() { @@ -634,6 +706,88 @@ guard_prod_promotion_lineage() { log_info "Prod promotion-lineage guard passed: source clean + promoted." } +guard_hotpatch_ledger() { + # Hot-patch ledger rebuild preflight (OMN-13014, retro B-1). + # + # In-container hot-patches (.prepatch sibling discipline) silently revert + # on any image rebuild / force-recreate. When a hot-patch ledger exists on + # this host, refuse to build a lane whose recorded patches have source PRs + # not merged into the build ref, or whose containers carry unledgered + # .prepatch files. Delegates to scripts/preflight_hotpatch_ledger.py. + # Sole bypass: HOTPATCH_PREFLIGHT_BYPASS with a Rule-10 user-approval + # receipt ('# skip-token-allowed: '), validated by the gate. + local repo_root="$1" + local git_sha="$2" + local compose_project="$3" + + log_step "Hot-Patch Ledger Preflight (OMN-13014)" + + local ledger_path="${HOTPATCH_LEDGER_PATH:-/data/omninode/hotpatch-ledger/ledger.yaml}" + if [[ ! -f "${ledger_path}" ]]; then + log_warn "No hot-patch ledger at ${ledger_path} — nothing recorded on this host; gate skipped." + log_warn "If containers here carry live hot-patches, STOP and write the ledger first." + return 0 + fi + + local gate="${repo_root}/scripts/preflight_hotpatch_ledger.py" + if [[ ! -f "${gate}" ]]; then + log_error "Hot-patch ledger exists at ${ledger_path} but the gate script is missing: ${gate}" + log_error "Refusing to rebuild over recorded hot-patches without the preflight." + exit 1 + fi + + # Lane = compose project suffix (omnibase-infra-stability-test -> stability-test); + # the bare dev project (omnibase-infra) maps to lane 'dev'. + local lane="${compose_project#omnibase-infra}" + lane="${lane#-}" + if [[ -z "${lane}" ]]; then + lane="dev" + fi + + # Workspace builds vendor sibling repos from OMNI_HOME clones; the gate + # resolves each ledger row's repo build ref (clone HEAD unless overridden + # via --build-ref) and runs git merge-base --is-ancestor per merge commit. + local clones_root="${OMNI_HOME:-}" + if [[ -z "${clones_root}" ]]; then + log_error "Hot-patch ledger present but OMNI_HOME is unset." + log_error "Cannot resolve build-input clones for the hot-patch preflight." + exit 1 + fi + + local python_bin="" + if [[ -x "${repo_root}/.venv/bin/python" ]]; then + python_bin="${repo_root}/.venv/bin/python" + elif command -v uv &>/dev/null; then + python_bin="uv-run" + elif command -v python3 &>/dev/null; then + python_bin="python3" + else + log_error "No Python interpreter available to run the hot-patch ledger preflight." + exit 1 + fi + + local gate_args=( + --lane "${lane}" + --ledger "${ledger_path}" + --clones-root "${clones_root}" + --build-ref "omnibase_infra=${git_sha}" + ) + log_cmd "${gate} ${gate_args[*]}" + if [[ "${python_bin}" == "uv-run" ]]; then + if ! uv run --project "${repo_root}" python "${gate}" "${gate_args[@]}"; then + log_error "Hot-patch ledger preflight FAILED. Refusing to rebuild over live hot-patches." + exit 1 + fi + else + if ! "${python_bin}" "${gate}" "${gate_args[@]}"; then + log_error "Hot-patch ledger preflight FAILED. Refusing to rebuild over live hot-patches." + exit 1 + fi + fi + + log_info "Hot-patch ledger preflight passed: all recorded patches merged into the build ref." +} + # ============================================================================= # Concurrency Lock # ============================================================================= @@ -1115,6 +1269,27 @@ if spec and spec.origin: log_cmd "rsync -a --delete workspace/sibling-repos/ -> deployed" rsync -a --delete \ "${repo_root}/workspace/sibling-repos/" "${deploy_target}/workspace/sibling-repos/" + # The lock-pin preflight result (OMN-12987) lives under sibling-repos/ as + # .sibling-lock-pins.json, so the rsync above already carries it into the + # build context for the in-image provenance merge. + + # Carry the root-level workspace/ file that Dockerfile.runtime COPYs + # (workspace/sibling-pin-comparison.json, line ~278). The sibling-repos/ + # rsync above only covers the subdirectory; without this the deployed build + # context lacks the comparison file and `docker build` fails with + # "failed to calculate checksum of ref ...:/workspace/sibling-pin-comparison.json: + # not found" -- the bug fixed in OMN-12987 (the dev compose build only worked + # because it runs from the repo root where the committed placeholder exists). + # Release mode ships the committed placeholder; workspace mode ships the real + # expected-vs-actual comparison stage_workspace.sh wrote into the repo root + # (OMN-12977). The regression test + # tests/scripts/test_deploy_runtime_build_context.py asserts every + # COPY-from-workspace path the Dockerfile references is staged here, so a + # future Dockerfile COPY without a matching rsync fails CI. + log_cmd "rsync -a workspace/sibling-pin-comparison.json -> deployed" + rsync -a \ + "${repo_root}/workspace/sibling-pin-comparison.json" \ + "${deploy_target}/workspace/sibling-pin-comparison.json" # 6. Migration scripts (bind-mounted by docker-compose.infra.yml) log_info "Syncing migration scripts..." @@ -1349,6 +1524,11 @@ build_images() { local build_date build_date="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + # OMN-12965: stamp org.opencontainers.image.version from pyproject so the + # runtime image carries a real version instead of the Dockerfile placeholder + # (0.1.0). A placeholder version degrades every proof packet. + local runtime_version + runtime_version="$(read_version "${deploy_target}")" local omni_home="${OMNI_HOME:-}" local build_source build_source="$(resolve_build_source)" @@ -1375,7 +1555,9 @@ build_images() { --profile "${COMPOSE_PROFILE}" build --progress=plain + --build-arg "GIT_SHA=${git_sha}" --build-arg "VCS_REF=${git_sha}" + --build-arg "RUNTIME_VERSION=${runtime_version}" --build-arg "BUILD_DATE=${build_date}" --build-arg "RUNTIME_SOURCE_HASH=${git_sha}" --build-arg "COMPOSE_PROJECT=${compose_project}" @@ -1387,7 +1569,7 @@ build_images() { --build-arg "ONEX_CHANGE_CONTROL_REF=${occ_ref}" ) - log_info "Building images with VCS_REF=${git_sha} RUNTIME_SOURCE_HASH=${git_sha} COMPOSE_PROJECT=${compose_project}..." + log_info "Building images with VCS_REF=${git_sha} RUNTIME_VERSION=${runtime_version} RUNTIME_SOURCE_HASH=${git_sha} COMPOSE_PROJECT=${compose_project}..." log_info "Build source: BUILD_SOURCE=${build_source} EXPECTED_BUILD_SOURCE=${expected_build_source} OMNI_HOME=${omni_home}" log_info "Plugin refs: OMNIBASE_COMPAT_REF=${compat_ref} OMNIMARKET_REF=${omnimarket_ref} ONEX_CHANGE_CONTROL_REF=${occ_ref}" log_info "Build timeout: ${build_timeout}s (set DOCKER_BUILD_TIMEOUT_SECONDS to override)" @@ -1430,7 +1612,14 @@ run_runtime_migration_preflight() { log_cmd "${cmd[*]}" "${cmd[@]}" if [[ "${service}" == "forward-migration" ]]; then - local wait_cmd=(docker wait omnibase-forward-migration) + # Derive the container name from the compose project so the wait + # targets the lane being deployed, not a fixed name. The base + # compose names it -forward-migration and each + # lane overlay follows the same form (OMN-12987), so this resolves + # to omnibase-infra-forward-migration for dev and + # omnibase-infra-stability-test-forward-migration for stability. + local forward_migration_container="${compose_project}-forward-migration" + local wait_cmd=(docker wait "${forward_migration_container}") log_cmd "${wait_cmd[*]}" if [[ "$("${wait_cmd[@]}")" != "0" ]]; then log_error "forward-migration did not complete successfully." @@ -1439,9 +1628,15 @@ run_runtime_migration_preflight() { fi done + # Postgres follows the same lane-derivable naming as forward-migration: + # -postgres (omnibase-infra-postgres for dev, + # omnibase-infra-stability-test-postgres for stability). Deriving it keeps + # the projection-table probe pointed at the lane being deployed instead of + # always hitting the dev-lane postgres (OMN-12987). + local postgres_container="${compose_project}-postgres" for table_name in "${REQUIRED_PROJECTION_TABLES[@]}"; do local check_cmd=( - docker exec omnibase-infra-postgres + docker exec "${postgres_container}" psql -U postgres -d omnidash_analytics @@ -1552,6 +1747,20 @@ verify_deployment() { log_warn "Could not read image label (container may not exist yet)." fi + # OMN-12965: verify org.opencontainers.image.version is a real version, not + # the Dockerfile placeholder (0.1.0) or blank. A placeholder/blank identity + # degrades every proof packet (runtime SHA + image digest are required + # citations in accepted evidence). + local version_label + version_label="$(docker inspect "${container_id}" \ + --format='{{index .Config.Labels "org.opencontainers.image.version"}}' 2>/dev/null || true)" + if [[ -z "${version_label}" || "${version_label}" == "0.1.0" ]]; then + log_error "Image version label is blank/placeholder: org.opencontainers.image.version='${version_label}'" + log_error "Runtime image identity is degraded (OMN-12965). Rebuild with RUNTIME_VERSION from pyproject." + exit 1 + fi + log_info "Image version label OK: org.opencontainers.image.version=${version_label}" + # 4. Log sentinel: entrypoint ran log_info "Checking log sentinels..." local logs @@ -1759,6 +1968,11 @@ main() { local compose_project compose_project="$(resolve_compose_project)" + # Hot-patch ledger preflight: refuse to rebuild over live in-container + # hot-patches whose source PRs are not merged into the build ref. + # Runs in both dry-run and execute modes (OMN-13014, retro B-1). + guard_hotpatch_ledger "${repo_root}" "${git_sha}" "${compose_project}" + # --print-compose-cmd: show commands and exit if [[ "${PRINT_COMPOSE_CMD}" == true ]]; then print_compose_commands "${deploy_target}" "${compose_project}" "${git_sha}" diff --git a/scripts/disk-gc.sh b/scripts/disk-gc.sh new file mode 100755 index 0000000000..79b5f81493 --- /dev/null +++ b/scripts/disk-gc.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +# disk-gc.sh — Conservative scheduled docker/disk garbage collection for .201 (OMN-13008). +# +# WHY: On 2026-06-11 /data on .201 reached ~95% (weeks of unpruned docker images +# + builder cache) and killed all three runtime lanes mid-demo. This script is the +# scheduled, conservative GC that keeps /data from detonating, driven by a +# VERSIONED keep-list config (deploy/disk-gc/keep-list.yaml) — nothing is hardcoded. +# +# It reaps, in increasing order of caution: +# 1. docker builder cache (cache mounts are always safe to drop) +# 2. dangling images (untagged, ) older than min_age_days +# 3. stopped containers older than min_age_days +# 4. superseded image generations of kept repos, KEEPING keep_image_tags, +# keeping protect_running references, keeping the newest +# superseded_image_keep_generations, and only removing those older than +# min_age_days. +# +# It NEVER removes: +# - any image whose repo matches keep_image_repos +# - any image whose tag matches keep_image_tags +# - any image referenced by a container (when protect_running: true) +# - anything younger than min_age_days +# - any volume (volumes are out of scope — data safety) +# +# Usage: +# ./scripts/disk-gc.sh # DRY RUN (default): print what WOULD be removed +# ./scripts/disk-gc.sh --execute # actually remove +# ./scripts/disk-gc.sh --keep-list /path/to/keep-list.yaml +# ./scripts/disk-gc.sh --json # machine-readable plan to stdout (dry-run plan) +# +# Exit codes: 0 success (plan printed or executed), 2 bad args, 3 missing deps. +# +# Runs on .201 via deploy/disk-gc.timer (systemd user timer). Log: ~/.local/log/onex/disk-gc.log + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +KEEP_LIST="${SCRIPT_DIR}/../deploy/disk-gc/keep-list.yaml" +EXECUTE=false +EMIT_JSON=false +LOG_FILE="${HOME}/.local/log/onex/disk-gc.log" + +while [[ $# -gt 0 ]]; do + case "$1" in + --execute) EXECUTE=true; shift ;; + --keep-list) KEEP_LIST="$2"; shift 2 ;; + --json) EMIT_JSON=true; shift ;; + --help|-h) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "Unknown argument: $1" >&2; exit 2 ;; + esac +done + +mkdir -p "$(dirname "$LOG_FILE")" +log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] [disk-gc] $*" | tee -a "$LOG_FILE" >&2; } + +command -v docker >/dev/null 2>&1 || { echo "ERROR: docker not found" >&2; exit 3; } +command -v python3 >/dev/null 2>&1 || { echo "ERROR: python3 not found" >&2; exit 3; } +[[ -f "$KEEP_LIST" ]] || { echo "ERROR: keep-list not found: $KEEP_LIST" >&2; exit 3; } + +log "Starting ($( [[ "$EXECUTE" == true ]] && echo EXECUTE || echo DRY-RUN )), keep-list=$KEEP_LIST" + +# --------------------------------------------------------------------------- +# Resolve the removal PLAN in Python (deterministic, testable). The plan is the +# only thing that decides what gets removed; bash only executes it. +# +# Docker inventory is written to per-run scratch files and handed to the planner +# on stdin as a JSON envelope. We do NOT pass it via env vars: a host with many +# images blows past ARG_MAX (`Argument list too long`). Scratch lives under the +# log dir (never /tmp), and is cleaned on exit. +# --------------------------------------------------------------------------- +SCRATCH="$(mktemp -d "$(dirname "$LOG_FILE")/disk-gc.XXXXXX")" +trap 'rm -rf "$SCRATCH"' EXIT +docker image ls --all --no-trunc --format '{{json .}}' >"$SCRATCH/images.ndjson" 2>/dev/null || : >"$SCRATCH/images.ndjson" +docker ps --all --no-trunc --format '{{json .}}' >"$SCRATCH/ps.ndjson" 2>/dev/null || : >"$SCRATCH/ps.ndjson" +docker ps --all --format '{{.Image}}' 2>/dev/null | sort -u >"$SCRATCH/inuse.txt" || : >"$SCRATCH/inuse.txt" + +# Build the stdin JSON envelope from the scratch files (SCRATCH_DIR env tells the +# encoder where to read), then pipe it straight into the planner. Two simple +# processes, no nested subprocess, no env-var size limit. +PLAN_JSON="$( + SCRATCH_DIR="$SCRATCH" python3 -c ' +import json, os +d = os.environ["SCRATCH_DIR"] +print(json.dumps({ + "images_ndjson": open(os.path.join(d, "images.ndjson")).read(), + "ps_ndjson": open(os.path.join(d, "ps.ndjson")).read(), + "inuse": open(os.path.join(d, "inuse.txt")).read(), +})) +' | KEEP_LIST="$KEEP_LIST" python3 "${SCRIPT_DIR}/disk_gc_plan.py" +)" + +if [[ "$EMIT_JSON" == true ]]; then + echo "$PLAN_JSON" +fi + +# Builder cache prune (always conservative — drops only reclaimable build cache). +# Honor min_age via docker's own filter so we don't drop a cache layer from a build +# that's seconds old. +MIN_AGE_DAYS="$(echo "$PLAN_JSON" | python3 -c 'import json,sys;print(json.load(sys.stdin)["min_age_days"])')" +IMAGE_IDS="$(echo "$PLAN_JSON" | python3 -c 'import json,sys;[print(i) for i in json.load(sys.stdin)["remove_image_ids"]]')" +CONTAINER_IDS="$(echo "$PLAN_JSON" | python3 -c 'import json,sys;[print(c) for c in json.load(sys.stdin)["remove_container_ids"]]')" + +log "Plan: $(echo "$IMAGE_IDS" | grep -c . || true) image(s), $(echo "$CONTAINER_IDS" | grep -c . || true) stopped container(s), builder cache > ${MIN_AGE_DAYS}d" + +if [[ "$EXECUTE" != true ]]; then + log "DRY-RUN — would remove the above. Re-run with --execute to act." + [[ -n "$IMAGE_IDS" ]] && { echo "IMAGES TO REMOVE:"; echo "$IMAGE_IDS"; } >&2 + [[ -n "$CONTAINER_IDS" ]] && { echo "STOPPED CONTAINERS TO REMOVE:"; echo "$CONTAINER_IDS"; } >&2 + exit 0 +fi + +# --- Execute --------------------------------------------------------------- +log "Pruning builder cache older than ${MIN_AGE_DAYS}d" +docker builder prune --force --filter "until=${MIN_AGE_DAYS}h0m0s" >>"$LOG_FILE" 2>&1 || \ + docker builder prune --force >>"$LOG_FILE" 2>&1 || log "builder prune failed (non-fatal)" + +if [[ -n "$CONTAINER_IDS" ]]; then + while IFS= read -r cid; do + [[ -z "$cid" ]] && continue + if docker rm "$cid" >>"$LOG_FILE" 2>&1; then log "removed container $cid"; else log "FAILED to remove container $cid"; fi + done <<< "$CONTAINER_IDS" +fi + +if [[ -n "$IMAGE_IDS" ]]; then + while IFS= read -r iid; do + [[ -z "$iid" ]] && continue + if docker rmi "$iid" >>"$LOG_FILE" 2>&1; then log "removed image $iid"; else log "kept/failed image $iid (likely still referenced)"; fi + done <<< "$IMAGE_IDS" +fi + +log "Done. df after:" +df -h /data 2>/dev/null | tee -a "$LOG_FILE" >&2 || df -h / | tee -a "$LOG_FILE" >&2 +exit 0 diff --git a/scripts/disk-watermark-check.sh b/scripts/disk-watermark-check.sh new file mode 100755 index 0000000000..075a5776ef --- /dev/null +++ b/scripts/disk-watermark-check.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +# disk-watermark-check.sh — Disk-usage watermark alert ratchet for .201 (OMN-13008). +# +# WHY: the 2026-06-11 outage had NO pre-detonation alert — /data silently filled +# until all three lanes crashed. This is the "pages before it detonates" fix. +# +# Behavior (df on the target mount): +# usage >= warn_pct (default 85): emit a disk-watermark bus event with +# severity=warning. A downstream consumer (node_runtime_sweep +# auto-ticket path) turns warning events into a Linear ticket. +# usage >= crit_pct (default 90): ALSO emit severity=critical — the loud event. +# usage < warn_pct : no-op (exit 0, quiet). +# +# The bus is the transport (ONEX doctrine): this script publishes the typed event +# and does not itself talk to Linear. Ticket creation is the consumer's job, which +# keeps a single auto-ticket authority (the sweep) instead of a second one here. +# +# Usage: +# ./scripts/disk-watermark-check.sh # check /data, publish if over +# ./scripts/disk-watermark-check.sh --mount / # different mount +# ./scripts/disk-watermark-check.sh --warn 85 --crit 90 # thresholds +# ./scripts/disk-watermark-check.sh --dry-run # print event, do NOT publish +# +# Exit codes: 0 ok (under warn, or published), 10 warn breached, 20 crit breached, +# 2 bad args. (Non-zero breach codes let the timer surface state in +# `systemctl --user status`.) +# +# Runs on .201 via deploy/disk-gc.timer (shares the GC timer). Log: ~/.local/log/onex/disk-watermark.log + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MOUNT="/data" +WARN_PCT=85 +CRIT_PCT=90 +DRY_RUN=false +TOPIC="onex.evt.infra.disk-watermark.v1" +LOG_FILE="${HOME}/.local/log/onex/disk-watermark.log" +HOSTNAME_TAG="$(hostname -s 2>/dev/null || echo unknown)" + +while [[ $# -gt 0 ]]; do + case "$1" in + --mount) MOUNT="$2"; shift 2 ;; + --warn) WARN_PCT="$2"; shift 2 ;; + --crit) CRIT_PCT="$2"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + --help|-h) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "Unknown argument: $1" >&2; exit 2 ;; + esac +done + +mkdir -p "$(dirname "$LOG_FILE")" +log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] [disk-watermark] $*" | tee -a "$LOG_FILE" >&2; } + +# Resolve usage percentage for the mount (fall back to / if MOUNT absent). +target="$MOUNT" +df -P "$target" >/dev/null 2>&1 || target="/" +USED_PCT="$(df -P "$target" | awk 'NR==2 {gsub(/%/,"",$5); print $5}')" +AVAIL_KB="$(df -P "$target" | awk 'NR==2 {print $4}')" + +if ! [[ "$USED_PCT" =~ ^[0-9]+$ ]]; then + log "ERROR: could not parse df usage for $target" + exit 2 +fi + +log "mount=$target used=${USED_PCT}% avail_kb=${AVAIL_KB} warn=${WARN_PCT} crit=${CRIT_PCT}" + +if (( USED_PCT < WARN_PCT )); then + log "under watermark — quiet" + exit 0 +fi + +SEVERITY="warning" +EXIT_CODE=10 +if (( USED_PCT >= CRIT_PCT )); then + SEVERITY="critical" + EXIT_CODE=20 +fi + +# Build the typed event payload deterministically in Python so the schema is +# stable and unit-testable. +EVENT_JSON="$( + USED_PCT="$USED_PCT" AVAIL_KB="$AVAIL_KB" MOUNT="$target" \ + SEVERITY="$SEVERITY" WARN_PCT="$WARN_PCT" CRIT_PCT="$CRIT_PCT" \ + HOSTNAME_TAG="$HOSTNAME_TAG" TOPIC="$TOPIC" \ + python3 "${SCRIPT_DIR}/disk_watermark_event.py" +)" + +log "event: $EVENT_JSON" + +if [[ "$DRY_RUN" == true ]]; then + echo "$EVENT_JSON" + log "DRY-RUN — not publishing (severity=$SEVERITY)" + exit "$EXIT_CODE" +fi + +# Publish to the bus. Prefer rpk (present on .201). The broker address MUST come +# from KAFKA_BOOTSTRAP_SERVERS — fail-fast, no localhost/default fallback (Rule 8, +# OMN-10741). The systemd unit injects it; an operator running by hand must export +# it. We never hardcode a broker address or LAN IP in source. +if [[ -z "${KAFKA_BOOTSTRAP_SERVERS:-}" ]]; then + log "KAFKA_BOOTSTRAP_SERVERS unset — cannot publish. Event logged above for manual replay." + exit "$EXIT_CODE" +fi +BOOTSTRAP="$KAFKA_BOOTSTRAP_SERVERS" +if command -v rpk >/dev/null 2>&1; then + if echo "$EVENT_JSON" | rpk topic produce "$TOPIC" --brokers "$BOOTSTRAP" >>"$LOG_FILE" 2>&1; then + log "published $SEVERITY event to $TOPIC via rpk" + else + log "FAILED to publish via rpk (broker=$BOOTSTRAP) — event logged above for manual replay" + fi +else + log "rpk not found — event logged above; install rpk or wire a producer to publish" +fi + +exit "$EXIT_CODE" diff --git a/scripts/disk_gc_plan.py b/scripts/disk_gc_plan.py new file mode 100644 index 0000000000..05a59f9822 --- /dev/null +++ b/scripts/disk_gc_plan.py @@ -0,0 +1,210 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""disk_gc_plan.py — Pure, deterministic removal-plan resolver for disk-gc.sh (OMN-13008). + +This module is the *only* place that decides what the .201 docker/disk GC removes. +`disk-gc.sh` collects docker inventory and pipes it in via env vars; this planner +emits a JSON plan; the shell only executes that plan. Keeping the decision logic +here (not in bash) makes the safety guarantees unit-testable without docker. + +Inputs: + KEEP_LIST (env) path to keep-list.yaml (small — env is fine) + stdin (JSON) {"images_ndjson": "...", "ps_ndjson": "...", "inuse": "..."} + where each value is the raw multi-line output of the matching + `docker ... --format '{{json .}}'` command. Inventory is passed + on stdin (NOT env) because a host with many images blows past + ARG_MAX when the blobs are env vars (`Argument list too long`). + +Output: a single JSON object on stdout: + { + "min_age_days": int, + "remove_image_ids": [str, ...], + "remove_container_ids": [str, ...], + "kept_reasons": {image_id: reason, ...} # for dry-run transparency + } + +Safety invariants (enforced + tested in tests/unit/scripts/test_disk_gc_plan.py): + - never remove an image whose repo matches a keep_image_repos substring + - never remove an image whose tag matches keep_image_tags exactly + - never remove an image referenced by any container when protect_running is true + - never remove anything younger than min_age_days + - retain the newest superseded_image_keep_generations of each kept repo +""" + +from __future__ import annotations + +import json +import os +import re +import sys +from datetime import UTC, datetime, timezone +from typing import Any + +import yaml + +# docker prints CreatedAt like: "2026-05-29 14:03:11 -0400 EDT" +_CREATED_RE = re.compile(r"^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) ([+-]\d{4})") + + +def _parse_created_at(value: str, now: datetime) -> float: + """Return age in days for a docker CreatedAt string. Unparseable → 0.0 (treat as new → keep).""" + if not value: + return 0.0 + m = _CREATED_RE.match(value.strip()) + if not m: + return 0.0 + dt = datetime.strptime(f"{m.group(1)} {m.group(2)}", "%Y-%m-%d %H:%M:%S %z") + return max(0.0, (now - dt).total_seconds() / 86400.0) + + +def _load_ndjson(blob: str) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + for line in (blob or "").splitlines(): + line = line.strip() + if not line: + continue + try: + out.append(json.loads(line)) + except json.JSONDecodeError: + continue + return out + + +def _repo_protected(repo: str, keep_repos: list[str]) -> bool: + return any(sub in repo for sub in keep_repos) + + +def build_plan( + keep_list: dict[str, Any], + images: list[dict[str, Any]], + containers: list[dict[str, Any]], + inuse_refs: set[str], + now: datetime | None = None, +) -> dict[str, Any]: + """Pure planner. See module docstring for invariants.""" + now = now or datetime.now(UTC) + + keep_repos: list[str] = keep_list.get("keep_image_repos", []) or [] + keep_tags: set[str] = set(keep_list.get("keep_image_tags", []) or []) + protect_running: bool = bool(keep_list.get("protect_running", True)) + keep_generations: int = int(keep_list.get("superseded_image_keep_generations", 2)) + min_age_days: int = int(keep_list.get("min_age_days", 3)) + + remove_image_ids: list[str] = [] + kept_reasons: dict[str, str] = {} + + # Group superseded candidates per repo so we can keep the N newest. + superseded_by_repo: dict[str, list[tuple[float, str]]] = {} + + for img in images: + image_id = img.get("ID", "") or img.get("Id", "") + repo = img.get("Repository", "") or "" + tag = img.get("Tag", "") or "" + created = img.get("CreatedAt", "") or "" + age = _parse_created_at(created, now) + ref = f"{repo}:{tag}" if repo and tag and repo != "" else image_id + + if age < min_age_days: + kept_reasons[image_id] = ( + f"younger than min_age_days ({age:.1f}<{min_age_days})" + ) + continue + if tag in keep_tags and tag and tag != "": + kept_reasons[image_id] = f"tag '{tag}' in keep_image_tags" + continue + if protect_running and (ref in inuse_refs or image_id in inuse_refs): + kept_reasons[image_id] = "referenced by a container (protect_running)" + continue + + is_dangling = repo == "" or repo == "" or tag == "" + + if is_dangling: + # Dangling + old + not in use → safe to remove. + remove_image_ids.append(image_id) + continue + + if _repo_protected(repo, keep_repos): + # Kept repo: this is a superseded generation candidate (older than + # min_age, not a keep_tag, not in use). Defer to per-repo retention. + superseded_by_repo.setdefault(repo, []).append((age, image_id)) + continue + + # Untracked repo, old, not in use, not dangling. Conservative default: + # KEEP. We only reap explicitly-tracked (keep_image_repos) superseded + # generations and dangling images, never random third-party images. + kept_reasons[image_id] = "repo not in keep_image_repos (conservative keep)" + + # Per-repo retention: keep the newest `keep_generations`, mark the rest for removal. + for repo, entries in superseded_by_repo.items(): + # newest first (smallest age first) + entries.sort(key=lambda e: e[0]) + for idx, (age, image_id) in enumerate(entries): + if idx < keep_generations: + kept_reasons[image_id] = ( + f"within newest {keep_generations} generations of '{repo}'" + ) + else: + remove_image_ids.append(image_id) + + # Stopped containers older than min_age_days are removable. + remove_container_ids: list[str] = [] + for c in containers: + state = (c.get("State", "") or "").lower() + status = (c.get("Status", "") or "").lower() + created = c.get("CreatedAt", "") or "" + cid = c.get("ID", "") or c.get("Id", "") + is_stopped = state in {"exited", "dead", "created"} or status.startswith( + "exited" + ) + if not is_stopped: + continue + if _parse_created_at(created, now) < min_age_days: + continue + remove_container_ids.append(cid) + + # Reconcile: the SAME image id can surface in multiple `docker image ls` rows + # (one per repo:tag). If any row produced a keep reason for an id, KEEP WINS — + # a protected id must never be in the remove list even if another tag routed it + # to removal. Also dedupe while preserving order. + seen: set[str] = set() + safe_remove: list[str] = [] + for image_id in remove_image_ids: + if image_id in kept_reasons: + continue + if image_id in seen: + continue + seen.add(image_id) + safe_remove.append(image_id) + remove_image_ids = safe_remove + + return { + "min_age_days": min_age_days, + "remove_image_ids": remove_image_ids, + "remove_container_ids": remove_container_ids, + "kept_reasons": kept_reasons, + } + + +def main() -> int: + keep_list_path = os.environ["KEEP_LIST"] + with open(keep_list_path, encoding="utf-8") as fh: + keep_list = yaml.safe_load(fh) or {} + + raw = sys.stdin.read() + payload = json.loads(raw) if raw.strip() else {} + + images = _load_ndjson(payload.get("images_ndjson", "")) + containers = _load_ndjson(payload.get("ps_ndjson", "")) + inuse = { + line.strip() for line in payload.get("inuse", "").splitlines() if line.strip() + } + + plan = build_plan(keep_list, images, containers, inuse) + json.dump(plan, sys.stdout) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/disk_watermark_event.py b/scripts/disk_watermark_event.py new file mode 100644 index 0000000000..bfcb73792c --- /dev/null +++ b/scripts/disk_watermark_event.py @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""disk_watermark_event.py — Build the typed disk-watermark bus event (OMN-13008). + +Pure builder so the event schema is deterministic and unit-testable. Reads the +measured values from env (the shell measures via df) and emits one JSON object +on stdout for `rpk topic produce` to publish to onex.evt.infra.disk-watermark.v1. + +The event is the alert authority: a downstream consumer (node_runtime_sweep +auto-ticket path) creates the Linear ticket on `severity=warning`, and operators +get the loud signal on `severity=critical`. This keeps a single ticket-creation +authority rather than letting every cron script talk to Linear directly. +""" + +from __future__ import annotations + +import json +import os +import sys +from datetime import UTC, datetime, timezone + + +def build_event( + *, + mount: str, + used_pct: int, + avail_kb: int, + severity: str, + warn_pct: int, + crit_pct: int, + host: str, + topic: str, + now: datetime | None = None, +) -> dict[str, object]: + """Construct the typed disk-watermark event payload.""" + now = now or datetime.now(UTC) + if severity not in {"warning", "critical"}: + raise ValueError(f"severity must be 'warning' or 'critical', got {severity!r}") + return { + "schema_version": "1.0.0", + "event_type": "disk-watermark", + "topic": topic, + "host": host, + "mount": mount, + "used_pct": used_pct, + "avail_kb": avail_kb, + "warn_pct": warn_pct, + "crit_pct": crit_pct, + "severity": severity, + "emitted_at": now.isoformat(), + # Stable dedupe key so the consumer can collapse repeated alerts for the + # same host/mount/severity into one open ticket instead of N. + "alert_key": f"disk-watermark:{host}:{mount}:{severity}", + "message": ( + f"{host}:{mount} at {used_pct}% used (warn>={warn_pct}%, crit>={crit_pct}%) " + f"severity={severity}" + ), + } + + +def main() -> int: + event = build_event( + mount=os.environ["MOUNT"], + used_pct=int(os.environ["USED_PCT"]), + avail_kb=int(os.environ["AVAIL_KB"]), + severity=os.environ["SEVERITY"], + warn_pct=int(os.environ["WARN_PCT"]), + crit_pct=int(os.environ["CRIT_PCT"]), + host=os.environ["HOSTNAME_TAG"], + topic=os.environ["TOPIC"], + ) + json.dump(event, sys.stdout) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/lane-census-check.sh b/scripts/lane-census-check.sh new file mode 100755 index 0000000000..25df8e09c7 --- /dev/null +++ b/scripts/lane-census-check.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +# lane-census-check.sh — Reconcile declared desired-state lanes vs actual (OMN-13011). +# +# THE CLASS FIX. Nothing reconciled desired vs actual lane state, so the same +# drift kept recurring with zero signal: +# - volume config drift (OMN-12945 family) +# - WORKER_REPLICAS silent zero (OMN-12988 / OMN-12990) +# - 2026-06-11: prod runtime containers + broker network silently absent, no alert +# +# This script gathers the live docker inventory, diffs it against the versioned +# lane manifest (deploy/lane-census/lane-manifest.yaml) via the pure planner +# (scripts/lane_census_plan.py), and on drift publishes a typed bus event +# (scripts/lane_census_event.py) so the sweep auto-ticket path opens a Linear +# ticket naming exactly what is missing/extra. +# +# Fail-fast, NO warn-only mode (gates-block policy): a drift on a non-optional +# lane exits non-zero. The systemd unit treats that as a maintenance signal. +# +# Usage: +# ./scripts/lane-census-check.sh # reconcile ALL non-optional lanes +# ./scripts/lane-census-check.sh --lane prod # one lane +# ./scripts/lane-census-check.sh --dry-run # print event/plan, do NOT publish +# ./scripts/lane-census-check.sh --json # emit the plan JSON to stdout +# +# Exit codes: 0 no drift, 30 drift detected (event emitted), 2 bad args, 3 missing deps. +# +# Runs on .201 via the SHARED onex-disk-gc.timer (4th ExecStart — coordinated with +# OMN-13008 rather than a second timer). Log: ~/.local/log/onex/lane-census.log + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +LANE="" +DRY_RUN=false +EMIT_JSON=false +LOG_FILE="${HOME}/.local/log/onex/lane-census.log" +DRIFT_TOPIC="onex.evt.infra.lane-census-drift.v1" + +while [[ $# -gt 0 ]]; do + case "$1" in + --lane) LANE="$2"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + --json) EMIT_JSON=true; shift ;; + --help|-h) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "Unknown argument: $1" >&2; exit 2 ;; + esac +done + +mkdir -p "$(dirname "$LOG_FILE")" +log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] [lane-census] $*" | tee -a "$LOG_FILE" >&2; } + +command -v docker >/dev/null 2>&1 || { echo "ERROR: docker not found" >&2; exit 3; } +command -v python3 >/dev/null 2>&1 || { echo "ERROR: python3 not found" >&2; exit 3; } + +HOST="${LANE_CENSUS_HOST:-$(hostname)}" +log "Starting (lane=${LANE:-ALL}, host=$HOST, $( [[ "$DRY_RUN" == true ]] && echo DRY-RUN || echo LIVE ))" + +# --------------------------------------------------------------------------- +# Gather actual state. Inventory goes to per-run scratch files (never /tmp) and +# is handed to the pure planner on stdin as a JSON envelope. No env-var size +# limit, no decision logic in bash. +# --------------------------------------------------------------------------- +SCRATCH="$(mktemp -d "$(dirname "$LOG_FILE")/lane-census.XXXXXX")" +trap 'rm -rf "$SCRATCH"' EXIT + +docker ps -a --no-trunc --format '{{json .}}' >"$SCRATCH/ps.ndjson" 2>/dev/null || : >"$SCRATCH/ps.ndjson" +docker network ls --format '{{.Name}}' >"$SCRATCH/networks.txt" 2>/dev/null || : >"$SCRATCH/networks.txt" + +# Resolve the runtime tag from the deploy-agent runtime version when available +# (relaxes to the default pattern if unresolvable — see the planner). +RUNTIME_TAG="${RUNTIME_TAG:-}" + +ENVELOPE_JSON="$( + SCRATCH_DIR="$SCRATCH" LANE="$LANE" RUNTIME_TAG="$RUNTIME_TAG" python3 -c ' +import json, os +d = os.environ["SCRATCH_DIR"] +lane = os.environ.get("LANE") or None +containers = [] +with open(os.path.join(d, "ps.ndjson")) as fh: + for line in fh: + line = line.strip() + if line: + containers.append(json.loads(line)) +networks = [n.strip() for n in open(os.path.join(d, "networks.txt")) if n.strip()] +print(json.dumps({ + "lane": lane, + "containers": containers, + "networks": networks, + "runtime_tag": os.environ.get("RUNTIME_TAG") or None, +})) +' +)" + +PLAN_JSON="$(echo "$ENVELOPE_JSON" | python3 "${SCRIPT_DIR}/lane_census_plan.py")" + +if [[ "$EMIT_JSON" == true ]]; then + echo "$PLAN_JSON" +fi + +HAS_DRIFT="$(echo "$PLAN_JSON" | python3 -c 'import json,sys;print(json.load(sys.stdin)["has_drift"])')" + +if [[ "$HAS_DRIFT" != "True" ]]; then + log "No lane drift. Desired == actual." + exit 0 +fi + +# Build the typed drift event from the plan. +EVENT_JSON="$(echo "$PLAN_JSON" | LANE_CENSUS_HOST="$HOST" python3 "${SCRIPT_DIR}/lane_census_event.py")" +log "DRIFT detected:" +# Render each finding to stderr. The event JSON is passed via env (EVENT_JSON) so +# the single-quoted heredoc body needs no shell escaping of inner Python quotes. +EVENT_JSON="$EVENT_JSON" python3 <<'PYEOF' >&2 || true +import json, os +event = json.loads(os.environ["EVENT_JSON"]) +for finding in event["findings"]: + print(" [{severity}] {kind} {container} ({lane}): {detail}".format(**finding)) +PYEOF + +if [[ "$DRY_RUN" == true ]]; then + echo "$EVENT_JSON" + log "DRY-RUN — not publishing drift event." + exit 30 +fi + +# Publish to the bus. The broker address MUST come from KAFKA_BOOTSTRAP_SERVERS — +# fail-fast, no localhost/default fallback (Rule 8). The systemd unit injects it; +# an operator running by hand must export it. We never hardcode a broker / LAN IP. +if [[ -z "${KAFKA_BOOTSTRAP_SERVERS:-}" ]]; then + log "KAFKA_BOOTSTRAP_SERVERS unset — cannot publish. Drift event logged above for manual replay." + exit 30 +fi +BOOTSTRAP="$KAFKA_BOOTSTRAP_SERVERS" +if command -v rpk >/dev/null 2>&1; then + if echo "$EVENT_JSON" | rpk topic produce "$DRIFT_TOPIC" --brokers "$BOOTSTRAP" >>"$LOG_FILE" 2>&1; then + log "published lane-census-drift event to $DRIFT_TOPIC via rpk" + else + log "FAILED to publish via rpk (broker=$BOOTSTRAP) — event logged above for manual replay" + fi +else + log "rpk not found — event logged above; install rpk or wire a producer to publish" +fi + +# Fail-fast on drift (gates-block policy, no warn-only mode). +exit 30 diff --git a/scripts/lane_census_event.py b/scripts/lane_census_event.py new file mode 100644 index 0000000000..489ca5c7aa --- /dev/null +++ b/scripts/lane_census_event.py @@ -0,0 +1,145 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""lane_census_event.py — Build the typed lane-census-drift bus event (OMN-13011). + +Pure builder so the drift-event schema is deterministic and unit-testable. Takes +the drift plan emitted by lane_census_plan.py and produces one JSON object on +stdout for `rpk topic produce` to publish to onex.evt.infra.lane-census-drift.v1. + +The event is the alert authority: a downstream consumer (the runtime_sweep / +sweep auto-ticket path) creates the Linear ticket naming exactly what is missing +or extra. This keeps a single ticket-creation authority rather than letting every +cron script talk to Linear directly — the same pattern as the disk-watermark +event (OMN-13008). + +The `alert_key` deduplicates: one open ticket per (host, lane, kind-set) so a +persistent outage does not spam a new ticket every reconcile tick. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import sys +from datetime import UTC, datetime +from typing import Any + +DRIFT_TOPIC = "onex.evt.infra.lane-census-drift.v1" + +# A drift finding is a flat string mapping (lane, kind, container, detail, severity) +# as emitted by lane_census_plan.py. The plan envelope is loosely typed (str/Any) +# because it round-trips through JSON; the helpers below narrow it locally. +Finding = dict[str, str] +Plan = dict[str, Any] + + +def _findings(plan: Plan) -> list[Finding]: + """Narrow the loosely-typed plan envelope to its findings list.""" + raw = plan.get("findings", []) + return list(raw) + + +def _alert_key(host: str, plan: Plan) -> str: + """Stable dedupe key over the (host, lane, kind, container) finding set.""" + signature = sorted( + f"{f['lane']}:{f['kind']}:{f['container']}" for f in _findings(plan) + ) + digest = hashlib.sha256("|".join(signature).encode("utf-8")).hexdigest()[:16] + return f"lane-census-drift:{host}:{digest}" + + +def _ticket_title(plan: Plan) -> str: + """One-line title naming exactly what is wrong.""" + findings = _findings(plan) + if not findings: + return "lane census: no drift" + lanes = sorted({f["lane"] for f in findings}) + # Summarize by kind for a precise, greppable title. + kinds: dict[str, int] = {} + for f in findings: + kinds[f["kind"]] = kinds.get(f["kind"], 0) + 1 + kind_summary = ", ".join( + f"{count}x {kind}" for kind, count in sorted(kinds.items()) + ) + return f"fix(infra): lane drift [{', '.join(lanes)}] — {kind_summary}" + + +def _ticket_body(host: str, plan: Plan) -> str: + findings = _findings(plan) + lanes_checked: list[str] = list(plan.get("lanes_checked", [])) + lines = [ + "## Lane census drift detected", + "", + f"Host: `{host}`", + f"Lanes checked: {', '.join(lanes_checked)}", + "", + "The desired-state lane census (deploy/lane-census/lane-manifest.yaml)", + "does not match the live runtime. Drift items below name exactly what is", + "missing or extra. This is the regression class the lane-census ratchet", + "(OMN-13011) exists to catch — it would have fired on 2026-06-11 when prod", + "runtime containers and the broker network were silently absent.", + "", + "## Findings", + "", + ] + for f in findings: + lines.append( + f"- **[{f['severity']}] {f['kind']}** `{f['container']}` ({f['lane']}): {f['detail']}" + ) + lines.append("") + lines.append("## Action") + lines.append( + "Bring the lane back to declared state (compose up the absent containers / " + "reattach the network), or update the lane manifest in the same PR if the " + "desired state legitimately changed. Do not silence the check." + ) + return "\n".join(lines) + + +def build_event( + *, + host: str, + plan: Plan, + topic: str = DRIFT_TOPIC, + now: datetime | None = None, +) -> dict[str, Any]: + """Construct the typed lane-census-drift event payload.""" + now = now or datetime.now(UTC) + findings = _findings(plan) + severity = ( + "critical" if any(f["severity"] == "critical" for f in findings) else "warning" + ) + return { + "schema_version": "1.0.0", + "event_type": "lane-census-drift", + "topic": topic, + "host": host, + "emitted_at": now.isoformat(), + "severity": severity, + "lanes_checked": plan.get("lanes_checked", []), + "drift_count": len(findings), + "findings": findings, + "alert_key": _alert_key(host, plan), + "ticket_title": _ticket_title(plan), + "ticket_body": _ticket_body(host, plan), + } + + +def main() -> int: + + host = os.environ.get("LANE_CENSUS_HOST", "") + if not host: + # Fail-fast: never silently fabricate a host into the alert_key. + print("ERROR: LANE_CENSUS_HOST must be set", file=sys.stderr) + return 2 + plan = json.load(sys.stdin) + event = build_event(host=host, plan=plan) + json.dump(event, sys.stdout) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/lane_census_plan.py b/scripts/lane_census_plan.py new file mode 100644 index 0000000000..c2c859bea8 --- /dev/null +++ b/scripts/lane_census_plan.py @@ -0,0 +1,346 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""lane_census_plan.py — Pure desired-vs-actual lane reconciler (OMN-13011). + +THE CLASS FIX for the recurring lane-drift regression. Nothing reconciled the +declared desired state of a runtime lane against what is actually running, so the +same failures kept recurring with zero signal: + + - volume config drift (OMN-12945 family) + - WORKER_REPLICAS silent zero — worker scaled to 0, no alert (OMN-12988/12990) + - 2026-06-11: prod runtime containers + broker network silently absent for hours + +This module is a *pure* planner: it takes the versioned lane manifest +(deploy/lane-census/lane-manifest.yaml) as the DESIRED state and a JSON envelope +of the live docker inventory as the ACTUAL state, and emits a deterministic list +of typed drift findings. It performs NO I/O — the shell driver +(scripts/lane-census-check.sh) gathers the docker inventory and publishes the +bus event / ticket. Keeping the decision logic here makes drift detection fully +unit-testable, including the 2026-06-11 prod red fixture. + +Drift kinds (named exactly so the auto-ticket says precisely what is wrong): + container_absent — a required `service` container is not running + replicas_zero — a required service resolved to 0 running replicas + network_detached — the lane's declared network does not exist + oneshot_failed — a run-to-completion container Exited non-zero + oneshot_stuck — a run-to-completion container is still Running + image_tag_mismatch — a running container's image tag fails the lane pattern + unexpected_container — a container labeled for the lane that the manifest does + not declare + +Fail-fast policy: any drift on a non-optional lane is a hard signal. There is no +warn-only mode (gates-block policy). The driver exits non-zero on drift. + +Input envelope (stdin JSON): + { + "lane": "" | null, # null => reconcile all non-optional lanes + "containers": [ # `docker ps -a --format '{{json .}}'` rows, + {"Names": "...", "State": "running"|"exited"|..., + "Image": "repo:tag", "Labels": "k=v,k2=v2", + "Status": "Up 3 hours" | "Exited (0) ..."}, + ... + ], + "networks": ["omnibase-infra-prod-network", ...], # `docker network ls` + "runtime_tag": "0.37.0" | null # resolved runtime tag (optional) + } + +Output (stdout JSON): + { + "schema_version": "1.0.0", + "lanes_checked": ["prod", ...], + "findings": [ {lane, kind, container, detail, severity}, ... ], + "has_drift": true|false + } +""" + +from __future__ import annotations + +import json +import os +import re +import sys +from pathlib import Path +from typing import Any + +import yaml + +SCHEMA_VERSION = "1.0.0" + +# Drift severities. critical => a required service is down / network gone (the +# 2026-06-11 class of outage). warning => degraded but not lane-down. +_SEVERITY_CRITICAL = "critical" +_SEVERITY_WARNING = "warning" + +_DEFAULT_MANIFEST = ( + Path(__file__).resolve().parent.parent + / "deploy" + / "lane-census" + / "lane-manifest.yaml" +) + + +def load_manifest(path: Path | None = None) -> dict[str, Any]: + """Load the versioned lane manifest (desired state).""" + manifest_path = path or _DEFAULT_MANIFEST + with open(manifest_path, encoding="utf-8") as fh: + data = yaml.safe_load(fh) + if not isinstance(data, dict) or "lanes" not in data: + raise ValueError(f"lane manifest missing 'lanes': {manifest_path}") + return data + + +def _running(state: str, status: str) -> bool: + """A container counts as running only when docker reports it running.""" + return state.lower() == "running" or status.lower().startswith("up") + + +def _exit_code(status: str) -> int | None: + """Parse the exit code out of a docker status string, e.g. 'Exited (0) ...'.""" + match = re.search(r"Exited \((\d+)\)", status) + return int(match.group(1)) if match else None + + +def _tag_of(image: str) -> str: + """Return the tag portion of a 'repo:tag' image reference (default 'latest').""" + # Strip a registry host:port (the first colon may belong to the host). + # docker image refs: [registry[:port]/]repo[:tag][@digest] + ref = image.split("@", 1)[0] + last_segment = ref.rsplit("/", 1)[-1] + if ":" in last_segment: + return last_segment.rsplit(":", 1)[1] + return "latest" + + +def _finding( + lane: str, kind: str, container: str, detail: str, severity: str +) -> dict[str, str]: + return { + "lane": lane, + "kind": kind, + "container": container, + "detail": detail, + "severity": severity, + } + + +def reconcile_lane( + lane_name: str, + lane_spec: dict[str, Any], + *, + actual_by_name: dict[str, dict[str, Any]], + networks: set[str], + lane_labeled_names: set[str], + default_tag_pattern: str, + runtime_tag: str | None, +) -> list[dict[str, str]]: + """Diff one lane's desired state against the actual inventory.""" + findings: list[dict[str, str]] = [] + optional = bool(lane_spec.get("optional", False)) + + declared = lane_spec.get("services", []) + declared_names = {svc["name"] for svc in declared} + + # An optional lane that is entirely down is NOT drift (developer lane). We + # detect "entirely down" as: none of its declared service containers are + # running. If ANY are running, it is partially up and we reconcile it. + if optional: + any_running = any( + _running( + actual_by_name.get(svc["name"], {}).get("State", ""), + actual_by_name.get(svc["name"], {}).get("Status", ""), + ) + for svc in declared + if svc.get("kind", "service") == "service" + ) + if not any_running: + return findings # lane legitimately down; no ticket + + # 1. Network presence — the broker/lane network must exist. + declared_network = lane_spec.get("network") + if declared_network and declared_network not in networks: + findings.append( + _finding( + lane_name, + "network_detached", + declared_network, + f"lane network {declared_network!r} is not present in " + f"`docker network ls` — containers cannot reach the broker", + _SEVERITY_CRITICAL, + ) + ) + + tag_pattern = lane_spec.get("image_tag_pattern") or default_tag_pattern + # ${RUNTIME_TAG} is resolved from the deploy-agent runtime version when + # available; an unresolvable token relaxes to the default pattern (a missing + # runtime tag is a separate signal, not a census drift). + if "${RUNTIME_TAG}" in tag_pattern: + tag_pattern = re.escape(runtime_tag) if runtime_tag else default_tag_pattern + compiled_pattern = re.compile(tag_pattern) + + # 2. Per-declared-service diff. + for svc in declared: + name = svc["name"] + kind = svc.get("kind", "service") + actual = actual_by_name.get(name) + + if kind == "oneshot": + # Run-to-completion container. Absence (compose removed it) is fine. + if actual is None: + continue + status = actual.get("Status", "") + if _running(actual.get("State", ""), status): + findings.append( + _finding( + lane_name, + "oneshot_stuck", + name, + f"migration/init container {name!r} is still Running " + f"(expected run-to-completion): {status}", + _SEVERITY_WARNING, + ) + ) + continue + code = _exit_code(status) + if code is not None and code != 0: + findings.append( + _finding( + lane_name, + "oneshot_failed", + name, + f"migration/init container {name!r} Exited non-zero " + f"(code {code}): {status}", + _SEVERITY_CRITICAL, + ) + ) + continue + + # kind == service: MUST be running with the required replica count. + required_replicas = int(svc.get("replicas", 1)) + if actual is None or not _running( + actual.get("State", ""), actual.get("Status", "") + ): + # Distinguish replicas_zero (declared scaled-out but down) only for + # clarity; for a single declared container, absent == container_absent. + kind_name = ( + "replicas_zero" if required_replicas == 0 else "container_absent" + ) + findings.append( + _finding( + lane_name, + kind_name, + name, + f"required service container {name!r} is not running " + f"(desired replicas={required_replicas}); lane " + f"{lane_name!r} is degraded", + _SEVERITY_CRITICAL, + ) + ) + continue + + # Running — verify image tag matches the lane pattern. + tag = _tag_of(actual.get("Image", "")) + if not compiled_pattern.fullmatch(tag): + findings.append( + _finding( + lane_name, + "image_tag_mismatch", + name, + f"container {name!r} runs image tag {tag!r} which does not " + f"match lane pattern {tag_pattern!r}", + _SEVERITY_WARNING, + ) + ) + + # 3. Unexpected containers — anything labeled for this lane but not declared. + for actual_name in lane_labeled_names: + if actual_name not in declared_names: + findings.append( + _finding( + lane_name, + "unexpected_container", + actual_name, + f"container {actual_name!r} carries com.omninode.lane=" + f"{lane_name!r} but is not declared in the lane manifest", + _SEVERITY_WARNING, + ) + ) + + return findings + + +def _labels_to_dict(labels: str) -> dict[str, str]: + out: dict[str, str] = {} + for pair in labels.split(","): + if "=" in pair: + key, value = pair.split("=", 1) + out[key.strip()] = value.strip() + return out + + +def build_plan(envelope: dict[str, Any], manifest: dict[str, Any]) -> dict[str, Any]: + """Build the deterministic drift plan from the desired manifest + actual state.""" + requested_lane = envelope.get("lane") + containers = envelope.get("containers", []) + networks = set(envelope.get("networks", [])) + runtime_tag = envelope.get("runtime_tag") + default_pattern = manifest.get("default_image_tag_pattern", ".+") + + actual_by_name: dict[str, dict[str, Any]] = {} + lane_labels: dict[str, set[str]] = {} + for row in containers: + name = (row.get("Names") or row.get("Name") or "").lstrip("/").strip() + if not name: + continue + actual_by_name[name] = row + labels = _labels_to_dict(row.get("Labels", "")) + lane_label = labels.get("com.omninode.lane") + if lane_label: + lane_labels.setdefault(lane_label, set()).add(name) + + lanes = manifest["lanes"] + if requested_lane: + if requested_lane not in lanes: + raise ValueError(f"unknown lane {requested_lane!r}") + target_lanes = {requested_lane: lanes[requested_lane]} + else: + target_lanes = lanes + + findings: list[dict[str, str]] = [] + checked: list[str] = [] + for lane_name, lane_spec in target_lanes.items(): + checked.append(lane_name) + findings.extend( + reconcile_lane( + lane_name, + lane_spec, + actual_by_name=actual_by_name, + networks=networks, + lane_labeled_names=lane_labels.get(lane_name, set()), + default_tag_pattern=default_pattern, + runtime_tag=runtime_tag, + ) + ) + + return { + "schema_version": SCHEMA_VERSION, + "lanes_checked": checked, + "findings": findings, + "has_drift": len(findings) > 0, + } + + +def main() -> int: + manifest_path = os.environ.get("LANE_MANIFEST") + manifest = load_manifest(Path(manifest_path) if manifest_path else None) + envelope = json.load(sys.stdin) + plan = build_plan(envelope, manifest) + json.dump(plan, sys.stdout) + sys.stdout.write("\n") + # Exit 0 always — the planner reports; the shell driver decides exit policy + # so the JSON plan is always emittable for dry-run/inspection. + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/preflight_hotpatch_ledger.py b/scripts/preflight_hotpatch_ledger.py new file mode 100644 index 0000000000..e65f44187a --- /dev/null +++ b/scripts/preflight_hotpatch_ledger.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Hot-patch ledger rebuild preflight (OMN-13014, night-plan retro B-1). + +Container-layer hot-patches (``.prepatch`` sibling discipline) silently revert +on any image rebuild or ``compose up --force-recreate``. This gate refuses to +rebuild a container when: + +1. any hot-patch ledger row for the target container/lane has a source PR + merge commit that is NOT an ancestor of the build ref for that repo + (``git merge-base --is-ancestor``), or +2. the running container carries ``.prepatch`` files that are NOT recorded in + the ledger (unledgered patches would be silently destroyed), or +3. with ``--post-rebuild``: any ``.prepatch`` file survives the rebuild. + +The canonical ledger lives at ``/data/omninode/hotpatch-ledger/ledger.yaml`` +on the runtime host (override with ``--ledger`` or ``HOTPATCH_LEDGER_PATH``). + +Sole bypass: export ``HOTPATCH_PREFLIGHT_BYPASS`` containing a line of the +exact Rule-10 form ``# skip-token-allowed: `` where +the receipt id is a real user-issued approval handle. Any other value of the +variable is a hard failure. + +Exit codes: 0 = pass (or authorized bypass), 1 = gate failure, 2 = usage or +configuration error (missing ledger, unknown commit, malformed bypass). +""" + +from __future__ import annotations + +import argparse +import os +import re +import subprocess +import sys +from pathlib import Path +from typing import Any + +import yaml + +DEFAULT_LEDGER_PATH = "/data/omninode/hotpatch-ledger/ledger.yaml" +BYPASS_PATTERN = re.compile(r"^# skip-token-allowed: (\S+)$") +TRIPWIRE_SEARCH_PATHS = ("/app", "/usr/local/lib", "/usr/lib/python3", "/opt") +SUPPORTED_SCHEMA = 1 + + +def fail(message: str, *, code: int = 1) -> int: + print(f"HOTPATCH-PREFLIGHT FAIL: {message}", file=sys.stderr) + return code + + +def load_ledger(ledger_path: Path) -> dict[str, Any]: + if not ledger_path.is_file(): + raise FileNotFoundError( + f"hot-patch ledger not found at {ledger_path}; refusing to guess. " + "If this host has never recorded a hot-patch, create an empty " + "ledger (schema: 1, rows: [])." + ) + raw = yaml.safe_load(ledger_path.read_text()) + if not isinstance(raw, dict): + raise ValueError(f"ledger at {ledger_path} is not a mapping") + schema = raw.get("schema") + if schema != SUPPORTED_SCHEMA: + raise ValueError( + f"unsupported ledger schema {schema!r} (expected {SUPPORTED_SCHEMA})" + ) + rows = raw.get("rows") + if rows is None: + raise ValueError("ledger has no 'rows' key") + if not isinstance(rows, list): + raise ValueError("ledger 'rows' must be a list") + return raw + + +def select_rows( + rows: list[dict[str, Any]], + container: str | None, + lane: str | None, +) -> list[dict[str, Any]]: + selected: list[dict[str, Any]] = [] + for row in rows: + if container is not None and row.get("container") != container: + continue + if lane is not None and row.get("lane") != lane: + continue + selected.append(row) + return selected + + +def resolve_build_ref( + repo: str, + explicit_refs: dict[str, str], + clones_root: Path, +) -> str: + """Return the build ref for *repo*, defaulting to the clone's HEAD. + + Workspace-mode builds vendor sibling repos from their clone working tree, + so the clone HEAD *is* the build ref when not explicitly overridden. The + resolved SHA is always printed so the deploy ledger can record it. + """ + if repo in explicit_refs: + return explicit_refs[repo] + clone = clones_root / repo + head = subprocess.run( + ["git", "-C", str(clone), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=False, + ) + if head.returncode != 0: + raise ValueError( + f"no --build-ref given for repo {repo!r} and could not resolve " + f"HEAD of clone {clone}: {head.stderr.strip()}" + ) + return head.stdout.strip() + + +def commit_known(clone: Path, commit: str) -> bool: + result = subprocess.run( + ["git", "-C", str(clone), "cat-file", "-e", f"{commit}^{{commit}}"], + capture_output=True, + text=True, + check=False, + ) + return result.returncode == 0 + + +def is_ancestor(clone: Path, commit: str, build_ref: str) -> bool: + result = subprocess.run( + ["git", "-C", str(clone), "merge-base", "--is-ancestor", commit, build_ref], + capture_output=True, + text=True, + check=False, + ) + if result.returncode == 0: + return True + if result.returncode == 1: + return False + raise ValueError( + f"git merge-base failed in {clone} for {commit}..{build_ref}: " + f"{result.stderr.strip()}" + ) + + +def tripwire_prepatch_files(container: str, docker_cmd: str) -> list[str]: + """Return every ``.prepatch`` path found inside the running container.""" + find_cmd = " ".join( + f'find {path} -name "*.prepatch" 2>/dev/null;' for path in TRIPWIRE_SEARCH_PATHS + ) + result = subprocess.run( + [docker_cmd, "exec", container, "sh", "-c", find_cmd], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + raise ValueError( + f"tripwire probe could not exec into container {container!r}: " + f"{result.stderr.strip()}" + ) + return [line.strip() for line in result.stdout.splitlines() if line.strip()] + + +def check_bypass() -> str | None: + """Return the receipt id when an authorized bypass is present. + + Raises ValueError when the bypass variable is set but malformed — a + malformed bypass is a hard failure, never a silent pass-through. + """ + raw = os.environ.get("HOTPATCH_PREFLIGHT_BYPASS") + if raw is None: + return None + match = BYPASS_PATTERN.fullmatch(raw.strip()) + if match is None: + raise ValueError( + "HOTPATCH_PREFLIGHT_BYPASS is set but does not match the required " + "form '# skip-token-allowed: '" + ) + return match.group(1) + + +def parse_build_refs(pairs: list[str]) -> dict[str, str]: + refs: dict[str, str] = {} + for pair in pairs: + repo, sep, ref = pair.partition("=") + if not sep or not repo or not ref: + raise ValueError(f"--build-ref must be =, got {pair!r}") + refs[repo] = ref + return refs + + +def run_preflight(args: argparse.Namespace) -> int: + try: + receipt = check_bypass() + except ValueError as exc: + return fail(str(exc), code=2) + if receipt is not None: + print( + "HOTPATCH-PREFLIGHT BYPASS: authorized by user approval receipt " + f"{receipt!r} — gate checks skipped." + ) + return 0 + + ledger_path = Path(args.ledger) + try: + ledger = load_ledger(ledger_path) + explicit_refs = parse_build_refs(args.build_ref) + except (FileNotFoundError, ValueError) as exc: + return fail(str(exc), code=2) + + rows = select_rows(ledger["rows"] or [], args.container, args.lane) + scope = args.container or args.lane + print( + f"HOTPATCH-PREFLIGHT: ledger {ledger_path} — {len(rows)} row(s) " + f"in scope {scope!r}" + ) + + failures: list[str] = [] + clones_root = Path(args.clones_root) + resolved_refs: dict[str, str] = {} + + for row in rows: + repo = row["source_repo"] + commit = row["merge_commit"] + clone = clones_root / repo + if repo not in resolved_refs: + try: + resolved_refs[repo] = resolve_build_ref( + repo, explicit_refs, clones_root + ) + except ValueError as exc: + return fail(str(exc), code=2) + print(f"HOTPATCH-PREFLIGHT: build ref {repo}={resolved_refs[repo]}") + build_ref = resolved_refs[repo] + if not commit_known(clone, commit): + return fail( + f"merge commit {commit} ({row['source_pr']}) is unknown in " + f"clone {clone} — the clone is stale or wrong; " + "run 'git fetch' on the build host first.", + code=2, + ) + try: + merged = is_ancestor(clone, commit, build_ref) + except ValueError as exc: + return fail(str(exc), code=2) + marker = "ancestor-of-build-ref" if merged else "NOT in build ref" + print( + f"HOTPATCH-PREFLIGHT: {row['container']} {row['file']} " + f"<- {row['source_pr']} ({commit[:12]}): {marker}" + ) + if not merged: + failures.append( + f"{row['file']} in {row['container']} is hot-patched from " + f"{row['source_pr']} (merge commit {commit}) which is NOT " + f"merged into build ref {build_ref} for repo {repo}; " + "rebuilding would silently destroy this patch." + ) + + if not args.skip_tripwire: + containers = sorted({str(row["container"]) for row in rows}) + if args.container is not None: + containers = sorted(set(containers) | {args.container}) + ledgered = {str(row["prepatch_path"]) for row in rows} + for container in containers: + try: + found = tripwire_prepatch_files(container, args.docker_cmd) + except ValueError as exc: + failures.append(str(exc)) + continue + unledgered = [path for path in found if path not in ledgered] + container_ledgered = { + str(row["prepatch_path"]) + for row in rows + if row["container"] == container + } + missing = sorted(path for path in container_ledgered if path not in found) + print( + f"HOTPATCH-PREFLIGHT: tripwire {container}: " + f"{len(found)} .prepatch file(s) live" + ) + if args.post_rebuild and found: + failures.append( + f"post-rebuild tripwire: {container} still carries " + f".prepatch files {found}; the rebuild did not start from " + "a clean image." + ) + if not args.post_rebuild and unledgered: + failures.append( + f"tripwire: {container} carries UNLEDGERED .prepatch " + f"files {unledgered}; record them in the ledger before " + "any rebuild." + ) + if not args.post_rebuild and missing: + print( + f"HOTPATCH-PREFLIGHT WARN: ledgered .prepatch missing from " + f"{container}: {missing} (patch may already be reverted)" + ) + + if failures: + for failure in failures: + fail(failure) + return 1 + print("HOTPATCH-PREFLIGHT PASS: all in-scope hot-patches merged into build ref.") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + scope = parser.add_mutually_exclusive_group(required=True) + scope.add_argument("--container", help="exact container name to gate") + scope.add_argument("--lane", help="gate every ledger row in this lane") + parser.add_argument( + "--build-ref", + action="append", + default=[], + metavar="REPO=REF", + help="build ref per repo (repeatable); defaults to clone HEAD", + ) + parser.add_argument( + "--clones-root", + required=True, + help="directory containing the build-input git clones, one per repo", + ) + parser.add_argument( + "--ledger", + default=os.environ.get("HOTPATCH_LEDGER_PATH", DEFAULT_LEDGER_PATH), + help="hot-patch ledger path (env HOTPATCH_LEDGER_PATH overrides default)", + ) + parser.add_argument("--docker-cmd", default="docker") + parser.add_argument( + "--skip-tripwire", + action="store_true", + help="skip the running-container .prepatch probe (offline analysis)", + ) + parser.add_argument( + "--post-rebuild", + action="store_true", + help="post-rebuild mode: any surviving .prepatch file is a failure", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + return run_preflight(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/provision-infisical.py b/scripts/provision-infisical.py index aaf94dd8be..48e6e25114 100644 --- a/scripts/provision-infisical.py +++ b/scripts/provision-infisical.py @@ -629,12 +629,18 @@ def main() -> int: resp.raise_for_status() try: body = resp.json() - if body.get("status") != "ok": + # Honor both editions: community returns {"message": "Ok"}, + # enterprise returns {"status": "ok"}. _is_infisical_ready is the + # single source of truth for readiness (the inline status-only + # check here was a bug that blocked provisioning against the + # community edition deployed on .201 — OMN-12966). + if not _is_infisical_ready(body): logger.error( - "Infisical at %s returned HTTP 200 but status field is %r " - "(expected 'ok'). The server may still be initialising.", + "Infisical at %s returned HTTP 200 but body is %r " + "(expected status=ok or message=Ok). " + "The server may still be initialising.", args.addr, - body.get("status"), + body, ) return 1 except (ValueError, KeyError): diff --git a/scripts/pull-all.sh b/scripts/pull-all.sh index e4ace645e7..e329b24f37 100755 --- a/scripts/pull-all.sh +++ b/scripts/pull-all.sh @@ -185,6 +185,7 @@ wait # Aggregate results OK=0 FAILED=() +WARNED=() for repo in "${REPOS[@]}"; do result_file="$RESULTS_DIR/$repo" @@ -193,7 +194,7 @@ for repo in "${REPOS[@]}"; do case "$status" in OK) (( OK++ )) || true ;; FAILED) FAILED+=("$repo") ;; - MISSING) FAILED+=("$repo (missing)") ;; + MISSING) WARNED+=("$repo (not cloned — skipped)") ;; # SKIPPED — don't count esac fi @@ -284,5 +285,11 @@ fi # === End plugin cache refresh === echo "" -echo "${OK} repo(s) up to date. ${#FAILED[@]} failed." +if [[ ${#WARNED[@]} -gt 0 ]]; then + echo "WARN: ${#WARNED[@]} repo(s) not found locally and were skipped:" + for w in "${WARNED[@]}"; do + echo " WARN $w" + done +fi +echo "${OK} repo(s) up to date. ${#FAILED[@]} failed. ${#WARNED[@]} absent (skipped)." [[ ${#FAILED[@]} -eq 0 ]] || exit 1 diff --git a/scripts/render_bifrost_lane_overlay_env.py b/scripts/render_bifrost_lane_overlay_env.py new file mode 100644 index 0000000000..6e11093312 --- /dev/null +++ b/scripts/render_bifrost_lane_overlay_env.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Render Bifrost lane overlay YAML to dotenv sidecar (OMN-12864). + +Usage: + # Render dev lane overlay to dev.bifrost.env (default) + uv run python scripts/render_bifrost_lane_overlay_env.py + + # Check that the rendered env file is in sync with the YAML source + uv run python scripts/render_bifrost_lane_overlay_env.py --check + + # Render a specific overlay file + uv run python scripts/render_bifrost_lane_overlay_env.py \\ + --overlay docker/lane-overlays/dev.bifrost.yaml \\ + --output docker/lane-overlays/dev.bifrost.env + +CI contract: the rendered .env sidecar MUST be committed alongside the YAML +source. The --check flag fails with exit code 1 when the sidecar is stale. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import yaml + +_REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_REPO_ROOT / "src")) + +from omnibase_infra.runtime.models.model_bifrost_lane_overlay import ( + ModelBifrostLaneOverlay, +) + +_DEFAULT_OVERLAY = _REPO_ROOT / "docker" / "lane-overlays" / "dev.bifrost.yaml" +_DEFAULT_OUTPUT = _REPO_ROOT / "docker" / "lane-overlays" / "dev.bifrost.env" + + +def load_overlay(path: Path) -> ModelBifrostLaneOverlay: + """Load and validate the Bifrost lane overlay YAML.""" + raw = yaml.safe_load(path.read_text(encoding="utf-8")) + return ModelBifrostLaneOverlay.model_validate(raw) + + +def render_env_file(overlay: ModelBifrostLaneOverlay) -> str: + """Render the env-var binding as a dotenv file.""" + env = overlay.as_env_dict() + lines = [ + f"# Generated from docker/lane-overlays/{overlay.lane}.bifrost.yaml.", + "# Do not edit directly — edit the YAML source and re-render:", + "# uv run python scripts/render_bifrost_lane_overlay_env.py", + "#", + "# OMN-12864: committed endpoint authority for BIFROST_LOCAL_* vars.", + "# OMN-12815: every URL is the COMPLETE final chat-completions URL.", + ] + lines.extend(f"{key}={value}" for key, value in sorted(env.items())) + return "\n".join(lines) + "\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--overlay", + type=Path, + default=_DEFAULT_OVERLAY, + help="Path to the bifrost lane overlay YAML.", + ) + parser.add_argument( + "--output", + type=Path, + default=_DEFAULT_OUTPUT, + help="Path to write the rendered dotenv sidecar.", + ) + parser.add_argument( + "--check", + action="store_true", + help="Check that the sidecar is in sync with the YAML; exit 1 if not.", + ) + args = parser.parse_args(argv) + + overlay = load_overlay(args.overlay) + rendered = render_env_file(overlay) + + if args.check: + if not args.output.exists(): + print( + f"ERROR: {args.output} does not exist. Run without --check to generate.", + file=sys.stderr, + ) + return 1 + existing = args.output.read_text(encoding="utf-8") + if rendered != existing: + print( + f"ERROR: {args.output} is out of sync with {args.overlay}. " + "Re-render: uv run python scripts/render_bifrost_lane_overlay_env.py", + file=sys.stderr, + ) + return 1 + print(f"OK: {args.output} matches {args.overlay}") + return 0 + + args.output.write_text(rendered, encoding="utf-8") + print(f"Rendered: {args.output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/render_runtime_policy_env.py b/scripts/render_runtime_policy_env.py index 1b35c4cd76..f057d392c0 100755 --- a/scripts/render_runtime_policy_env.py +++ b/scripts/render_runtime_policy_env.py @@ -70,6 +70,9 @@ def render_env(contract: ModelRuntimePolicyContract) -> dict[str, str]: "LLM_CLOUD_ENDPOINT_HOST_ALLOWLIST": ",".join( contract.llm_cloud_endpoint_host_allowlist ), + "BIFROST_VERTEX_GEMINI_ENDPOINT_URL": contract.bifrost_vertex_gemini_endpoint_url, + "GOOGLE_CLOUD_PROJECT": contract.google_cloud_project, + "GOOGLE_CLOUD_LOCATION": contract.google_cloud_location, "OMNIMEMORY_MEMGRAPH_PORT": str(contract.omnimemory_memgraph_port), } @@ -106,6 +109,12 @@ def render_env(contract: ModelRuntimePolicyContract) -> dict[str, str]: for process_name, process in profile.processes.items(): process_prefix = _PROCESS_ENV_PREFIX[process_name] prefix = f"{profile_prefix}_RUNTIME_{process_prefix}" + if process_name == "worker": + # OMN-12990: pin the worker replica count in the ledgered config + # surface so a plain compose recreate cannot silently drop the + # worker via the base ${WORKER_REPLICAS:-0} default. The lane + # overrides reference ${_WORKER_REPLICAS:?...} fail-fast. + env[f"{profile_prefix}_WORKER_REPLICAS"] = str(process.replicas) env[f"{prefix}_CAPABILITIES"] = ",".join(process.capabilities) env[f"{prefix}_BIFROST_VERIFY_ENDPOINTS"] = _bifrost_text( process.bifrost_verify_endpoints diff --git a/scripts/run-forward-migrations.sh b/scripts/run-forward-migrations.sh index 7b045c5991..3f11dbe90f 100755 --- a/scripts/run-forward-migrations.sh +++ b/scripts/run-forward-migrations.sh @@ -14,6 +14,27 @@ # mechanism for keeping warm Postgres volumes up-to-date with new migrations. # # Ticket: OMN-4175 (Forward migration runner for warm Postgres volumes) +# Ticket: OMN-13062 (migration-gate vacuity fix — retro A-10) +# +# --------------------------------------------------------------------------- +# Sentinel discipline (OMN-13062) +# --------------------------------------------------------------------------- +# migrations_complete is cleared to FALSE at the start of every runner +# invocation and set to TRUE only as the FINAL act after all infra and +# node migrations apply without error. Any nonzero exit from any migration +# leaves the flag FALSE, making the migration-gate healthcheck UNHEALTHY. +# +# The committed per-migration skip-manifest is the SOLE escape for migrations +# that must be intentionally skipped: +# docker/migrations/skip-manifest.yaml +# Format: +# skipped_migrations: +# - id: "docker/NNN_name.sql" +# reason: "..." +# ticket: "OMN-XXXX" +# The runner reads this at startup; a listed migration_id is treated as +# already-applied without executing the SQL. New entries must be committed +# in the same PR that deems the migration unrunnable. # # --------------------------------------------------------------------------- # Node-owned migration auto-discovery (OMN-12559) @@ -40,6 +61,7 @@ # MIGRATIONS_DIR (default: /migrations/forward) # NODE_MIGRATIONS_DIR (default: ${MIGRATIONS_DIR}/nodes) # NODE_POSTGRES_DB (default: POSTGRES_DB; compose sets omnidash_analytics) +# PG_WAIT_RETRIES (default: 30 — number of 2s waits for postgres ready) set -e @@ -50,9 +72,50 @@ PGDB="${POSTGRES_DB:-omnibase_infra}" MIGRATIONS_DIR="${MIGRATIONS_DIR:-/migrations/forward}" NODE_MIGRATIONS_DIR="${NODE_MIGRATIONS_DIR:-${MIGRATIONS_DIR}/nodes}" NODE_PGDB="${NODE_POSTGRES_DB:-${PGDB}}" +PG_WAIT_RETRIES="${PG_WAIT_RETRIES:-30}" export PGPASSWORD="${POSTGRES_PASSWORD}" +# --------------------------------------------------------------------------- +# Skip-manifest: load intentionally-skipped migration ids (OMN-13062) +# --------------------------------------------------------------------------- +# Format: YAML file with a top-level list "skipped_migrations" each entry has +# "id" (e.g. "docker/038_placeholder.sql") and optionally "reason" / "ticket". +# Only a committed manifest is honoured — operator env cannot inject skips. +SKIP_MANIFEST="${MIGRATIONS_DIR%/forward}/skip-manifest.yaml" +SKIPPED_IDS="" +if [ -f "${SKIP_MANIFEST}" ]; then + echo "[forward-migration] Loading skip-manifest: ${SKIP_MANIFEST}" + # Extract quoted id: values from YAML using portable sed (no yq/python/gawk). + # Handles lines of the form: - id: "docker/NNN_name.sql" + SKIPPED_IDS="$(sed -n 's/^[[:space:]]*-[[:space:]]*id:[[:space:]]*"\([^"]*\)".*/\1/p' \ + "${SKIP_MANIFEST}" 2>/dev/null || true)" +fi + +is_skipped_by_manifest() { + migration_id="$1" + if [ -z "${SKIPPED_IDS}" ]; then + return 1 + fi + echo "${SKIPPED_IDS}" | grep -Fxq "${migration_id}" +} + +# --------------------------------------------------------------------------- +# 0. Wait for Postgres to be ready (first-boot initdb race guard, OMN-13062) +# --------------------------------------------------------------------------- +echo "[forward-migration] Waiting for Postgres to accept connections..." +retries=0 +until psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDB" -c "SELECT 1" >/dev/null 2>&1; do + retries=$((retries + 1)) + if [ "$retries" -ge "$PG_WAIT_RETRIES" ]; then + echo "[forward-migration] ERROR: Postgres not ready after ${PG_WAIT_RETRIES} retries. Aborting." >&2 + exit 1 + fi + echo "[forward-migration] postgres not ready (attempt ${retries}/${PG_WAIT_RETRIES}), retrying in 2s..." + sleep 2 +done +echo "[forward-migration] Postgres is ready." + validate_database_identifier() { database="$1" if ! printf '%s' "$database" | grep -Eq '^[A-Za-z_][A-Za-z0-9_-]*$'; then @@ -96,6 +159,33 @@ CREATE TABLE IF NOT EXISTS public.schema_migrations ( ); " +# --------------------------------------------------------------------------- +# 1a. Clear the sentinel at the start of every run (OMN-13062) +# --------------------------------------------------------------------------- +# This ensures that any mid-run failure leaves migrations_complete=FALSE so +# the migration-gate healthcheck stays UNHEALTHY. The sentinel is only set +# TRUE as the very last act of this script (after all migrations succeed). +# We use a conditional UPDATE so this is a no-op on volumes that have not +# yet applied migration 037 (migrations_complete column may not exist yet). +echo "[forward-migration] Clearing migration sentinel (will be re-set on successful completion)..." +psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDB" -c " +DO \$\$ +BEGIN + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'db_metadata' + AND column_name = 'migrations_complete' + ) THEN + UPDATE public.db_metadata + SET migrations_complete = FALSE, + updated_at = NOW() + WHERE id = TRUE; + END IF; +END; +\$\$; +" 2>/dev/null || true + # --------------------------------------------------------------------------- # 2. Apply pending migrations in sorted order # --------------------------------------------------------------------------- @@ -108,6 +198,18 @@ for migration_file in $(ls "${MIGRATIONS_DIR}"/*.sql | sort); do filename=$(basename "$migration_file") migration_id="docker/${filename}" + # Honour skip-manifest: treat manifest-listed migrations as already applied + if is_skipped_by_manifest "${migration_id}"; then + echo "[forward-migration] skip ${filename} (skip-manifest)" + SKIPPED=$((SKIPPED + 1)) + # Record in schema_migrations so the table stays consistent. + psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDB" \ + -c "INSERT INTO public.schema_migrations (migration_id, checksum, source_set) + VALUES ('${migration_id}', 'skip-manifest', 'docker') + ON CONFLICT (migration_id) DO NOTHING;" + continue + fi + # Check if already applied already_applied=$(psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDB" \ -tAc "SELECT 1 FROM public.schema_migrations WHERE migration_id = '${migration_id}'" 2>/dev/null || true) @@ -199,3 +301,18 @@ else fi echo "[forward-migration] Complete: ${APPLIED} infra applied, ${SKIPPED} infra skipped; ${NODE_APPLIED} node applied, ${NODE_SKIPPED} node skipped." + +# --------------------------------------------------------------------------- +# 4. Set the sentinel TRUE only after ALL migrations succeed (OMN-13062) +# --------------------------------------------------------------------------- +# This is the FINAL act. Any earlier failure leaves migrations_complete=FALSE. +# runner_completed_at records the timestamp of this successful completion. +echo "[forward-migration] All migrations complete. Setting sentinel TRUE..." +psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDB" -v ON_ERROR_STOP=1 -c " +UPDATE public.db_metadata +SET migrations_complete = TRUE, + runner_completed_at = NOW(), + updated_at = NOW() +WHERE id = TRUE; +" +echo "[forward-migration] Sentinel set. Migration gate will report HEALTHY." diff --git a/scripts/runtime_build/check_sibling_lock_pins.py b/scripts/runtime_build/check_sibling_lock_pins.py new file mode 100644 index 0000000000..1774b7d85c --- /dev/null +++ b/scripts/runtime_build/check_sibling_lock_pins.py @@ -0,0 +1,391 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Workspace-build sibling-pin preflight + recurrence ratchet [OMN-12977]. + +The workspace-mode image build (BUILD_SOURCE=workspace) vendors sibling repos +from whatever the canonical clones under OMNI_HOME happen to be checked out at. +On 2026-06-11 that shipped a 13-day-stale omnibase_infra 0.37.0-dev (pre-OMN-12501 +Protocol-quarantine guard) + omnibase_core 0.42.0 even though omnimarket dev's +``uv.lock`` pinned omnibase-infra 0.38.1 @ e2dbdc95 / omnibase-core 0.44.0 @ +c97c2c9a. The stale image dropped the guard and crashed bootstrap fatally. + +This module makes the consuming repo's ``uv.lock`` the pin authority: + + - ``parse_lock_pins`` extracts the expected version (+ git rev) of the + foundation/sibling packages from the consuming repo's lock. + - ``resolve_actual_pin`` reads the version (from pyproject) + HEAD git SHA of + a canonical clone that the build is about to vendor. + - ``compare_pins`` reports match / mismatch with an identifying reason. + - ``check_pins`` orchestrates a fail-fast preflight and writes the + expected-vs-actual comparison so deploy verifiers and build-provenance.json + can assert on it. + +There are NO silent fallbacks: a requested package missing from the lock, an +unparseable source, or any version / SHA drift is a hard error. Run BEFORE the +docker build (preflight) and again inside the build to enrich provenance. + +Usage: + uv run python scripts/runtime_build/check_sibling_lock_pins.py \\ + --lock "$OMNI_HOME/omnimarket/uv.lock" \\ + --repo omnibase-infra="$OMNI_HOME/omnibase_infra" \\ + --repo omnibase-core="$OMNI_HOME/omnibase_core" \\ + --output workspace/sibling-pin-comparison.json + +Exit codes: + 0 every requested pin matches the lock + 1 one or more pins drifted from the lock (build must abort) + 2 malformed input (missing lock, unparseable source, missing repo) + +OMN-12977 +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +import tomllib +from pathlib import Path + +from pydantic import BaseModel, ConfigDict, Field + +# Foundation + sibling packages whose pins the workspace build must honor. +# Keys are uv.lock distribution names; values are the canonical clone dir names +# under OMNI_HOME that the build vendors / installs from. +DEFAULT_PACKAGE_REPO_DIRS: dict[str, str] = { + "omnibase-infra": "omnibase_infra", + "omnibase-core": "omnibase_core", + "omnibase-spi": "omnibase_spi", + "omnibase-compat": "omnibase_compat", + "onex-change-control": "onex_change_control", + "omnimarket": "omnimarket", +} + +# Matches a package's own ``source = { ... }`` line in a uv.lock block. The rev +# (when the source is a git source) is captured from the ``?rev=`` query. +# Scoping to the source line is required: a ``dependencies = [...]`` entry may +# also carry a ``?rev=`` for a transitive git dep, and a block-wide search would +# wrongly attribute that rev to an editable/registry package (e.g. omnimarket +# itself is ``source = { editable = "." }`` with no rev of its own). +_SOURCE_LINE_RE = re.compile(r"^source = \{(?P.*)\}\s*$", re.MULTILINE) +_GIT_REV_RE = re.compile(r"[?&]rev=(?P[0-9a-fA-F]{7,40})") + + +class ModelLockPin(BaseModel): + """Expected pin for a package as declared in the consuming repo's uv.lock.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + package: str = Field(..., description="uv.lock distribution name") + version: str = Field(..., description="Locked version string") + git_rev: str | None = Field( + default=None, + description="Locked git rev (full/short SHA) or None for registry pins", + ) + + +class ModelActualPin(BaseModel): + """Actual pin of a canonical clone the build is about to vendor/install.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + package: str = Field(..., description="uv.lock distribution name") + version: str = Field(..., description="Version from the clone's pyproject.toml") + git_sha: str = Field(..., description="HEAD commit SHA of the canonical clone") + + +class ModelPinComparison(BaseModel): + """Expected-vs-actual comparison for one package (provenance-serializable).""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + package: str = Field(...) + expected_version: str = Field(...) + actual_version: str = Field(...) + expected_git_rev: str | None = Field(default=None) + actual_git_sha: str = Field(...) + matches: bool = Field(...) + drift_direction: str = Field( + default="none", + description=( + "'none' (match), 'backward' (clone older than lock -- the stale-" + "sibling crash), 'forward' (clone newer than lock), or 'unknown' " + "(non-comparable versions)" + ), + ) + mismatch_reason: str = Field(default="", description="Empty when matches is True") + + +def _classify_drift(expected_version: str, actual_version: str) -> str: + """Classify version drift direction (backward is the stale-sibling crash).""" + if expected_version == actual_version: + return "none" + try: + exp = tuple(int(p) for p in expected_version.split(".")[:3]) + act = tuple(int(p) for p in actual_version.split(".")[:3]) + except ValueError: + return "unknown" + if act < exp: + return "backward" + if act > exp: + return "forward" + return "unknown" + + +def parse_lock_pins(lock_text: str, packages: list[str]) -> dict[str, ModelLockPin]: + """Parse a uv.lock and return expected pins for the requested packages. + + Each ``[[package]]`` block carries a ``name``, ``version`` and ``source``. + Git sources embed the resolved rev in the source URL; registry sources have + no rev (version-only pin). A requested package absent from the lock is a + hard error -- workspace builds must never silently skip a foundation pin. + """ + blocks = lock_text.split("[[package]]") + found: dict[str, ModelLockPin] = {} + requested = set(packages) + + for block in blocks: + name_match = re.search(r'^name = "(?P[^"]+)"', block, re.MULTILINE) + if name_match is None: + continue + name = name_match.group("name") + if name not in requested: + continue + + version_match = re.search( + r'^version = "(?P[^"]+)"', block, re.MULTILINE + ) + if version_match is None: + raise ValueError(f"uv.lock block for {name!r} has no version field") + version = version_match.group("version") + + # Only the package's OWN source line may contribute a git rev. Editable + # ("." path) and registry (PyPI) sources have no rev. + git_rev: str | None = None + source_match = _SOURCE_LINE_RE.search(block) + if source_match is not None: + rev_match = _GIT_REV_RE.search(source_match.group("body")) + if rev_match is not None: + git_rev = rev_match.group("rev") + + found[name] = ModelLockPin(package=name, version=version, git_rev=git_rev) + + missing = requested - set(found) + if missing: + raise KeyError( + f"packages absent from uv.lock: {sorted(missing)}. " + "Workspace build cannot resolve a pin for them." + ) + return found + + +def resolve_actual_pin(package: str, repo_root: Path) -> ModelActualPin: + """Read the canonical clone's version (pyproject) + HEAD SHA. + + No defaults: a missing pyproject, a missing version, or an unreadable git + HEAD is a hard error (exit 2 at the CLI boundary). + """ + pyproject_path = repo_root / "pyproject.toml" + if not pyproject_path.is_file(): + raise FileNotFoundError( + f"missing pyproject.toml for {package}: {pyproject_path}" + ) + data = tomllib.loads(pyproject_path.read_text(encoding="utf-8")) + version = data.get("project", {}).get("version") + if not version: + raise ValueError(f"no project.version in {pyproject_path}") + + head_sha = subprocess.run( + ["git", "-C", str(repo_root), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if not head_sha: + raise ValueError(f"could not resolve git HEAD for {repo_root}") + + return ModelActualPin(package=package, version=version, git_sha=head_sha) + + +def compare_pins(expected: ModelLockPin, actual: ModelActualPin) -> ModelPinComparison: + """Compare an expected lock pin against the actual clone pin. + + Version must always match. When the lock pin is a git source, the resolved + rev must be a prefix of the clone HEAD (or vice-versa) -- uv records the full + rev while a short SHA may appear elsewhere, so prefix-equality covers both. + Registry pins (git_rev is None) are version-only. + """ + reasons: list[str] = [] + + if expected.version != actual.version: + reasons.append( + f"version drift: lock pins {expected.version!r} but clone is " + f"{actual.version!r}" + ) + + if expected.git_rev is not None: + exp = expected.git_rev.lower() + act = actual.git_sha.lower() + if not (act.startswith(exp) or exp.startswith(act)): + reasons.append( + f"git SHA drift: lock pins {expected.git_rev} but clone HEAD is " + f"{actual.git_sha}" + ) + + matches = not reasons + return ModelPinComparison( + package=expected.package, + expected_version=expected.version, + actual_version=actual.version, + expected_git_rev=expected.git_rev, + actual_git_sha=actual.git_sha, + matches=matches, + drift_direction=_classify_drift(expected.version, actual.version), + mismatch_reason="; ".join(reasons), + ) + + +def check_pins( + lock_path: Path, + repo_roots: dict[str, Path], + output_path: Path | None, + allow_drift: bool = False, +) -> int: + """Preflight: compare every requested clone against the lock; fail fast. + + When ``allow_drift`` is True the drift is recorded in the provenance artifact + and a warning is printed, but the build is not aborted -- reserved for an + explicit operator decision (e.g. an intentional forward rebuild ahead of a + lock bump). The override is always durable in the artifact, never silent. + + Returns a process exit code (0 pass-or-allowed / 1 drift / 2 malformed input). + """ + if not lock_path.is_file(): + print(f"ERROR: lock file not found: {lock_path}", file=sys.stderr) + return 2 + + packages = list(repo_roots) + try: + expected_pins = parse_lock_pins( + lock_path.read_text(encoding="utf-8"), packages=packages + ) + except (KeyError, ValueError) as exc: + print(f"ERROR: cannot resolve lock pins: {exc}", file=sys.stderr) + return 2 + + comparisons: list[ModelPinComparison] = [] + for package, repo_root in repo_roots.items(): + try: + actual = resolve_actual_pin(package, repo_root) + except (FileNotFoundError, ValueError, subprocess.CalledProcessError) as exc: + print( + f"ERROR: cannot resolve clone pin for {package}: {exc}", file=sys.stderr + ) + return 2 + comparisons.append(compare_pins(expected_pins[package], actual)) + + drifted = [c for c in comparisons if not c.matches] + + if output_path is not None: + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text( + json.dumps( + { + "lock_source": str(lock_path), + "allow_drift": allow_drift, + "drift_count": len(drifted), + "comparisons": [c.model_dump() for c in comparisons], + }, + indent=2, + ), + encoding="utf-8", + ) + print(f"sibling-pin comparison written to {output_path}") + + for c in comparisons: + status = "OK" if c.matches else f"DRIFT/{c.drift_direction}" + line = f" [{status}] {c.package}: lock={c.expected_version} clone={c.actual_version}" + print( + line if c.matches else f"{line} -- {c.mismatch_reason}", + file=sys.stderr if not c.matches else sys.stdout, + ) + + if drifted: + if allow_drift: + print( + f"\nWARNING: {len(drifted)} sibling pin(s) drifted from " + f"{lock_path.name}; proceeding under explicit operator override " + "(ALLOW drift) -- recorded in provenance (OMN-12977).", + file=sys.stderr, + ) + return 0 + print( + f"\nFAIL: {len(drifted)} sibling pin(s) drifted from {lock_path.name}. " + "Workspace build would vendor stale code -- aborting (OMN-12977).", + file=sys.stderr, + ) + return 1 + + print(f"\nAll {len(comparisons)} sibling pins match {lock_path.name}.") + return 0 + + +def _parse_repo_arg(value: str) -> tuple[str, Path]: + """Parse a ``package=path`` --repo argument.""" + if "=" not in value: + raise argparse.ArgumentTypeError(f"--repo must be package=path, got {value!r}") + package, _, path = value.partition("=") + package = package.strip() + if package not in DEFAULT_PACKAGE_REPO_DIRS: + raise argparse.ArgumentTypeError( + f"unknown package {package!r}; expected one of " + f"{sorted(DEFAULT_PACKAGE_REPO_DIRS)}" + ) + return package, Path(path).expanduser() + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--lock", + required=True, + type=Path, + help="Path to the consuming repo's uv.lock (the pin authority)", + ) + parser.add_argument( + "--repo", + action="append", + default=[], + type=_parse_repo_arg, + metavar="PACKAGE=PATH", + help="Canonical clone the build will vendor (repeatable)", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Where to write the expected-vs-actual comparison JSON", + ) + parser.add_argument( + "--allow-drift", + action="store_true", + help=( + "Record drift in the provenance artifact and proceed instead of " + "aborting. Explicit operator override only -- never the default." + ), + ) + args = parser.parse_args(argv) + + if not args.repo: + print("ERROR: at least one --repo PACKAGE=PATH is required", file=sys.stderr) + return 2 + + repo_roots = dict(args.repo) + return check_pins(args.lock, repo_roots, args.output, allow_drift=args.allow_drift) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/runtime_build/compute_workspace_provenance.py b/scripts/runtime_build/compute_workspace_provenance.py index 86065feeac..bd46a5e1a3 100644 --- a/scripts/runtime_build/compute_workspace_provenance.py +++ b/scripts/runtime_build/compute_workspace_provenance.py @@ -6,12 +6,17 @@ Run during Docker build (workspace mode only) to: - Hash each staged sibling repo tree under /workspace/sibling-repos/ - Verify that the installed package content matches the recorded digest - - Write /app/build-provenance.json for operator inspection + - Enforce that every staged sibling honors the consuming repo's uv.lock pin + (fail-fast on a version regression below the pin — the 2026-06-11 + stability-crash condition, OMN-12989) + - Write /app/build-provenance.json for operator inspection, including the + expected-vs-actual pin comparison block - Print a human-readable provenance summary to stdout Exit codes: 0 all proofs passed - 1 digest mismatch (installed content != recorded workspace digest) + 1 digest mismatch (installed content != recorded workspace digest) OR a + sibling regressed below its uv.lock pin 2 missing sibling repo (expected path not found) 3 missing installed package (venv does not contain the expected dist) """ @@ -25,9 +30,30 @@ import sys from pathlib import Path +# Co-located pin-resolution helpers (OMN-12989). Inside the build image this +# module is copied next to the provenance script (see Dockerfile.runtime); in +# the repo it lives in the same scripts/runtime_build/ directory. +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from resolve_workspace_pins import ( + PinComparison, + WorkspacePinError, + assert_pins_satisfied, + build_comparisons, + compare_pin, + parse_lock_pins, +) + SIBLING_REPOS_DIR = Path("/workspace/sibling-repos") VENV_DIR = Path("/app/.venv") OUTPUT_MANIFEST = Path("/app/build-provenance.json") +# The consuming repo whose uv.lock pins the sibling versions. omnimarket dev's +# lock is the authority the runtime image must honor. +CONSUMING_REPO = "omnimarket" +# Expected-vs-actual sibling-pin comparison emitted by the preflight +# (check_sibling_lock_pins.py via stage_workspace.sh). Folded into the manifest +# so deploy verifiers can assert the build honored the consuming repo's lock +# (OMN-12977). Absent only for builds staged before the preflight existed. +PIN_COMPARISON_PATH = Path("/workspace/sibling-pin-comparison.json") # Canonical set of sibling repos that workspace mode must provision. # Keys are the directory names under sibling-repos/; values are installed @@ -57,7 +83,7 @@ def _hash_tree(root: Path) -> str: return h.hexdigest() -def _installed_direct_url(dist_name: str) -> dict | None: +def _installed_direct_url(dist_name: str) -> dict[str, object] | None: """Return the direct_url metadata dict for an installed package, or None.""" try: dist = importlib.metadata.distribution(dist_name) @@ -66,11 +92,59 @@ def _installed_direct_url(dist_name: str) -> dict | None: direct_url_text = dist.read_text("direct_url.json") if direct_url_text is None: return None - return json.loads(direct_url_text) + parsed: dict[str, object] = json.loads(direct_url_text) + return parsed + + +def _host_infra_comparison(lock_path: Path) -> PinComparison | None: + """Compare the runtime image's installed omnibase_infra against the lock pin. + + The host infra package is not staged under sibling-repos (it is the repo + being built), so it bypasses build_comparisons. Resolve the lock's + omnibase-infra pin and compare against the installed version — this is the + direct ratchet for the 2026-06-11 stale-infra crash. Returns a PinComparison + or None if the lock has no infra pin / infra is not installed. + """ + pins = parse_lock_pins(lock_path) + infra_pin = pins.get("omnibase-infra") + if infra_pin is None: + return None + try: + installed_version = importlib.metadata.version("omnibase-infra") + except importlib.metadata.PackageNotFoundError: + return None + return compare_pin( + package="omnibase-infra", + expected_version=infra_pin.version, + expected_rev=infra_pin.rev, + actual_version=installed_version, + actual_rev=None, + ) + + +def _load_pin_comparison(errors: list[str]) -> dict[str, object] | None: + """Load the preflight's expected-vs-actual sibling-pin comparison. + + Returns the parsed comparison dict, or None when the artifact is absent + (e.g. a build staged before the preflight existed). A present-but-drifted + comparison is recorded as an error so the proof surfaces it. + """ + if not PIN_COMPARISON_PATH.exists(): + return None + comparison: dict[str, object] = json.loads( + PIN_COMPARISON_PATH.read_text(encoding="utf-8") + ) + drift_count = comparison.get("drift_count", 0) + if drift_count and not comparison.get("allow_drift", False): + errors.append( + f"sibling-pin comparison reports {drift_count} unacknowledged " + "drift(s) -- the build vendored stale siblings (OMN-12977)." + ) + return comparison def main() -> int: - proofs: list[dict] = [] + proofs: list[dict[str, object]] = [] errors: list[str] = [] for repo_dir_name, pkg_name in WORKSPACE_PACKAGES.items(): @@ -98,10 +172,11 @@ def main() -> int: ) continue - install_url = direct_url.get("url", "") - is_local = direct_url.get("dir_info", {}).get("editable") is not None or ( - install_url.startswith("file://") - ) + install_url_raw = direct_url.get("url", "") + install_url = install_url_raw if isinstance(install_url_raw, str) else "" + dir_info = direct_url.get("dir_info", {}) + editable = dir_info.get("editable") if isinstance(dir_info, dict) else None + is_local = editable is not None or install_url.startswith("file://") if not is_local: errors.append( @@ -137,11 +212,56 @@ def main() -> int: f"install={install_url}" ) + # OMN-12989: compare every staged sibling against the consuming repo's + # uv.lock pins. A sibling regressed BELOW its lock pin is the exact + # 2026-06-11 stability-crash condition (vendored infra 0.37.0 vs locked + # 0.38.1) and aborts the build. The comparison is always recorded in the + # manifest so deploy verifiers can check it mechanically. + pin_comparison: list[dict[str, object]] = [] + pin_violation: str | None = None + lock_path = SIBLING_REPOS_DIR / CONSUMING_REPO / "uv.lock" + try: + comparisons = build_comparisons( + lock_path=lock_path, + siblings={ + "omnibase-compat": SIBLING_REPOS_DIR / "omnibase_compat", + "onex-change-control": SIBLING_REPOS_DIR / "onex_change_control", + "omnimarket": SIBLING_REPOS_DIR / "omnimarket", + }, + ) + # Host self-check (OMN-12989): the runtime image's OWN omnibase_infra is + # built from the build context, NOT from sibling-repos — that is exactly + # where the 2026-06-11 stale 0.37.0 entered. Compare the installed infra + # version against omnimarket's lock pin so a stale build context aborts. + host_infra = _host_infra_comparison(lock_path) + if host_infra is not None: + comparisons.append(host_infra) + pin_comparison = [c.to_dict() for c in comparisons] + try: + assert_pins_satisfied(comparisons) + except WorkspacePinError as exc: + pin_violation = str(exc) + except FileNotFoundError as exc: + # A missing consuming lock is itself a hard error — it means the build + # cannot prove sibling freshness. Fail rather than silently skip. + pin_violation = ( + f"Cannot resolve sibling lock pins: {exc}. " + "Workspace mode requires the consuming repo's uv.lock to be staged." + ) + + # Fold in the sibling-pin comparison so deploy verifiers can confirm the + # build honored the consuming repo's uv.lock (OMN-12977). A drift here would + # already have aborted the build in the default (no-override) path; when an + # operator override was used, this records expected-vs-actual durably. + sibling_pin_comparison = _load_pin_comparison(errors) + manifest = { "build_source": "workspace", "build_time": os.environ.get("BUILD_DATE", "unknown"), "vcs_ref": os.environ.get("VCS_REF", "unknown"), "proofs": proofs, + "pin_comparison": pin_comparison, + "sibling_pin_comparison": sibling_pin_comparison, } OUTPUT_MANIFEST.write_text(json.dumps(manifest, indent=2), encoding="utf-8") @@ -153,6 +273,21 @@ def main() -> int: print(f" - {err}", file=sys.stderr) return 1 + if pin_violation is not None: + print( + "\nWorkspace provenance FAILED (sibling lock-pin regression):", + file=sys.stderr, + ) + print(pin_violation, file=sys.stderr) + return 1 + + for entry in pin_comparison: + print( + f" pin OK: {entry['package']} " + f"expected>={entry['expected_version'] or '(unpinned)'} " + f"actual={entry['actual_version']} [{entry['status']}]" + ) + print(f"\nAll {len(proofs)} workspace provenance proofs passed.") return 0 diff --git a/scripts/runtime_build/resolve_workspace_pins.py b/scripts/runtime_build/resolve_workspace_pins.py new file mode 100644 index 0000000000..a12a71f689 --- /dev/null +++ b/scripts/runtime_build/resolve_workspace_pins.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Resolve + enforce workspace-mode sibling lock pins (OMN-12989). + +Background — the 2026-06-11 stability bootstrap crash +----------------------------------------------------- +A workspace-mode ``--no-cache`` rebuild staged its sibling repos from a stale +MERGE-SWEEP deploy worktree instead of the canonical clone, vendoring +omnibase_infra 0.37.0-dev (~2c1d672f, 13 days stale) even though omnimarket +dev's ``uv.lock`` pins infra 0.38.1 @ e2dbdc95. The downgraded sibling +predated the OMN-12501 Protocol-quarantine guard, so a latent contract defect +crashed bootstrap fatally and took the stability lane down. + +This module is the fail-fast ratchet for that hole. It: + + * parses the consuming repo's ``uv.lock`` to extract the expected sibling + pins (version + git rev), + * compares each staged/installed sibling against its lock pin, + * classifies the comparison as ``exact`` / ``ahead`` / ``regression`` / + ``unpinned``, + * raises ``WorkspacePinError`` (fail-fast, no silent fallback) when any + sibling has regressed below its lock pin, and + * exposes a serializable comparison block for ``build-provenance.json`` so + deploy verifiers can mechanically check it. + +Policy: the canonical clones track moving dev branches, so a staged sibling may +legitimately be **at or ahead of** the lock pin (dev advanced past the last +omnimarket lock). What must never happen is a sibling **below** the lock pin — +that is the downgrade that caused the crash and is the only violation enforced. +""" + +from __future__ import annotations + +import re +import tomllib +from dataclasses import dataclass +from pathlib import Path + +# `packaging` is the preferred PEP 440 comparator, but the minimal runtime build +# venv is not guaranteed to carry it. Fall back to a stdlib release-tuple +# comparator so this ratchet never fails-open on a missing dependency. +try: + from packaging.version import InvalidVersion as _InvalidVersion + from packaging.version import Version as _Version + + def _version_key(value: str) -> tuple[int, ...]: + return _Version(value).release + + _VersionError: type[Exception] = _InvalidVersion +except ModuleNotFoundError: # pragma: no cover - exercised only without packaging + + class _VersionError(Exception): # type: ignore[no-redef] + pass + + def _version_key(value: str) -> tuple[int, ...]: + # Parse the leading numeric release segment (e.g. "0.38.1-dev" -> (0,38,1)). + match = re.match(r"^\s*(\d+(?:\.\d+)*)", value) + if not match: + raise _VersionError(value) + return tuple(int(part) for part in match.group(1).split(".")) + + +# Directory-name -> uv.lock distribution name for every sibling that workspace +# mode provisions. Mirrors WORKSPACE_PACKAGES in compute_workspace_provenance.py. +SIBLING_DISTRIBUTIONS: dict[str, str] = { + "omnibase_compat": "omnibase-compat", + "onex_change_control": "onex-change-control", + "omnimarket": "omnimarket", +} + + +class WorkspacePinError(RuntimeError): + """Raised when a staged sibling has regressed below its lock pin.""" + + +@dataclass(frozen=True) +class LockPin: + """Expected version + git rev for one package, parsed from a uv.lock.""" + + package: str + version: str + rev: str | None + + +@dataclass(frozen=True) +class PinComparison: + """Expected-vs-actual comparison for one sibling, for the provenance block.""" + + package: str + expected_version: str + expected_rev: str | None + actual_version: str + actual_rev: str | None + match: bool + status: str # one of: exact | ahead | regression | unpinned + + def to_dict(self) -> dict[str, object]: + return { + "package": self.package, + "expected_version": self.expected_version, + "expected_rev": self.expected_rev, + "actual_version": self.actual_version, + "actual_rev": self.actual_rev, + "match": self.match, + "status": self.status, + } + + +def _rev_from_git_source(git_url: str) -> str | None: + """Extract the resolved commit SHA from a uv.lock git source URL. + + uv encodes git pins as ``...repo.git?rev=#`` (the + fragment is the resolved commit). Prefer the fragment; fall back to the + ``rev=`` query value. + """ + if "#" in git_url: + fragment = git_url.rsplit("#", 1)[1].strip() + if fragment: + return fragment + if "rev=" in git_url: + rev = git_url.split("rev=", 1)[1] + # rev runs until the next URL delimiter. + for delim in ("#", "&", " "): + rev = rev.split(delim, 1)[0] + rev = rev.strip() + if rev: + return rev + return None + + +def parse_lock_pins(lock_path: Path) -> dict[str, LockPin]: + """Parse a uv.lock into ``{distribution_name: LockPin}``. + + Raises FileNotFoundError if the lock file is absent — a missing lock is a + hard error, never a silent skip. + """ + if not lock_path.exists(): + raise FileNotFoundError(f"uv.lock not found: {lock_path}") + + data = tomllib.loads(lock_path.read_text(encoding="utf-8")) + pins: dict[str, LockPin] = {} + for package in data.get("package", []): + name = package.get("name") + if name is None: + continue + version = package.get("version", "") + rev: str | None = None + source = package.get("source", {}) + git_url = source.get("git") + if isinstance(git_url, str): + rev = _rev_from_git_source(git_url) + pins[name] = LockPin(package=name, version=str(version), rev=rev) + return pins + + +def read_repo_version(repo_dir: Path) -> str: + """Read the ``[project].version`` from a repo's pyproject.toml. + + Raises FileNotFoundError if the pyproject is absent. + """ + pyproject = repo_dir / "pyproject.toml" + if not pyproject.exists(): + raise FileNotFoundError(f"pyproject.toml not found: {pyproject}") + data = tomllib.loads(pyproject.read_text(encoding="utf-8")) + return str(data["project"]["version"]) + + +def read_repo_head_sha(repo_dir: Path) -> str | None: + """Return the staged repo's git HEAD SHA, or None if not a git tree. + + Staged sibling trees are rsync'd with ``--exclude=.git`` so HEAD is usually + unavailable inside the build context; this is best-effort provenance only + and never gates the build (version regression is the enforced signal). + """ + head = repo_dir / ".git" / "HEAD" + if not head.exists(): + return None + try: + ref = head.read_text(encoding="utf-8").strip() + except OSError: + return None + if ref.startswith("ref:"): + ref_path = repo_dir / ".git" / ref.split(" ", 1)[1].strip() + if ref_path.exists(): + return ref_path.read_text(encoding="utf-8").strip() + return None + return ref + + +def compare_pin( + *, + package: str, + expected_version: str, + expected_rev: str | None, + actual_version: str, + actual_rev: str | None, +) -> PinComparison: + """Classify one sibling against its lock pin. + + status: + * ``exact`` — actual version == expected version + * ``ahead`` — actual version > expected (dev advanced; allowed) + * ``regression`` — actual version < expected (the downgrade; violation) + * ``unpinned`` — lock pin carried no comparable version (advisory) + + ``match`` is True for exact/ahead/unpinned and False only for regression. + """ + if not expected_version: + return PinComparison( + package=package, + expected_version=expected_version, + expected_rev=expected_rev, + actual_version=actual_version, + actual_rev=actual_rev, + match=True, + status="unpinned", + ) + + try: + expected = _version_key(expected_version) + actual = _version_key(actual_version) + except _VersionError: + # Unorderable versions cannot be ranked; fall back to exact-string match. + status = "exact" if actual_version == expected_version else "regression" + return PinComparison( + package=package, + expected_version=expected_version, + expected_rev=expected_rev, + actual_version=actual_version, + actual_rev=actual_rev, + match=status != "regression", + status=status, + ) + + if actual < expected: + status = "regression" + elif actual > expected: + status = "ahead" + else: + status = "exact" + + return PinComparison( + package=package, + expected_version=expected_version, + expected_rev=expected_rev, + actual_version=actual_version, + actual_rev=actual_rev, + match=status != "regression", + status=status, + ) + + +def build_comparisons( + *, + lock_path: Path, + siblings: dict[str, Path], +) -> list[PinComparison]: + """Build expected-vs-actual comparisons for staged ``siblings``. + + ``siblings`` maps distribution name -> staged repo directory. The expected + pin comes from ``lock_path``; the actual version comes from the staged + tree's pyproject. A sibling absent from the lock is reported ``unpinned``. + """ + pins = parse_lock_pins(lock_path) + comparisons: list[PinComparison] = [] + for dist_name, repo_dir in siblings.items(): + actual_version = read_repo_version(repo_dir) + actual_rev = read_repo_head_sha(repo_dir) + pin = pins.get(dist_name) + if pin is None: + comparisons.append( + PinComparison( + package=dist_name, + expected_version="", + expected_rev=None, + actual_version=actual_version, + actual_rev=actual_rev, + match=True, + status="unpinned", + ) + ) + continue + comparisons.append( + compare_pin( + package=dist_name, + expected_version=pin.version, + expected_rev=pin.rev, + actual_version=actual_version, + actual_rev=actual_rev, + ) + ) + return comparisons + + +def assert_pins_satisfied(comparisons: list[PinComparison]) -> None: + """Fail-fast if any comparison is a regression. No silent fallback. + + Raises WorkspacePinError naming every regressed sibling with its + expected pin and the stale version actually staged. + """ + violations = [c for c in comparisons if c.status == "regression"] + if not violations: + return + lines = [ + "Workspace-mode build aborted: vendored sibling(s) regressed below the " + "consuming repo's uv.lock pin (this is the 2026-06-11 stability-crash " + "condition — a stale sibling was staged instead of the lock-pinned one).", + ] + for c in violations: + lines.append( + f" - {c.package}: lock pins >= {c.expected_version}" + f" (rev {c.expected_rev}), but staged tree is {c.actual_version}" + f" (rev {c.actual_rev}). Re-stage from the canonical clone at or " + "above the lock pin." + ) + raise WorkspacePinError("\n".join(lines)) diff --git a/scripts/runtime_build/stage_workspace.sh b/scripts/runtime_build/stage_workspace.sh index dc2c8c826d..8057fc6cc6 100644 --- a/scripts/runtime_build/stage_workspace.sh +++ b/scripts/runtime_build/stage_workspace.sh @@ -11,11 +11,25 @@ # # On success, creates: # workspace/sibling-repos// (rsync'd working tree copy) +# workspace/sibling-pin-comparison.json (expected-vs-actual pin proof) +# +# Sibling-pin preflight (OMN-12977): +# Before staging, the consuming repo's uv.lock is the pin authority. The +# build vendors the canonical OMNI_HOME clones of omnibase_infra / omnibase_core +# / siblings; if any clone's version or git SHA drifts from the lock, the build +# ABORTS rather than silently vendoring a stale sibling. This is the recurrence +# guard for the 2026-06-11 stability crash where a 13-day-stale infra 0.37.0-dev +# (pre-OMN-12501 guard) was vendored against an omnimarket dev lock pinning +# infra 0.38.1. Set CONSUMER_LOCK to the consuming repo's uv.lock (default: +# ${OMNI_HOME}/omnimarket/uv.lock). Set ALLOW_SIBLING_PIN_DRIFT=1 ONLY with an +# explicit operator decision (e.g. an intentional forward rebuild ahead of a +# lock bump); the override is recorded in the provenance artifact, never silent. # # Exit codes: # 0 all sibling repos staged successfully # 1 OMNI_HOME not set # 2 one or more sibling repos missing from OMNI_HOME +# 3 sibling pin drift from the consuming lock (preflight abort) set -euo pipefail SIBLING_REPOS=( @@ -29,6 +43,49 @@ if [[ -z "${OMNI_HOME:-}" ]]; then exit 1 fi +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# --------------------------------------------------------------------------- +# Sibling-pin preflight (OMN-12977): the consuming repo's uv.lock is authority. +# --------------------------------------------------------------------------- +CONSUMER_LOCK="${CONSUMER_LOCK:-${OMNI_HOME}/omnimarket/uv.lock}" +PIN_COMPARISON_OUT="workspace/sibling-pin-comparison.json" + +# Foundation + sibling packages the build vendors, mapped to OMNI_HOME clone dirs. +PREFLIGHT_REPO_ARGS=( + --repo "omnibase-infra=${OMNI_HOME}/omnibase_infra" + --repo "omnibase-core=${OMNI_HOME}/omnibase_core" + --repo "omnibase-spi=${OMNI_HOME}/omnibase_spi" + --repo "omnibase-compat=${OMNI_HOME}/omnibase_compat" + --repo "onex-change-control=${OMNI_HOME}/onex_change_control" + --repo "omnimarket=${OMNI_HOME}/omnimarket" +) + +preflight_extra=() +if [[ "${ALLOW_SIBLING_PIN_DRIFT:-0}" == "1" ]]; then + preflight_extra+=(--allow-drift) + echo "WARNING: ALLOW_SIBLING_PIN_DRIFT=1 -- pin drift will be recorded, not fatal" >&2 +fi + +if [[ -f "${CONSUMER_LOCK}" ]]; then + mkdir -p "$(dirname "${PIN_COMPARISON_OUT}")" + if ! python3 "${SCRIPT_DIR}/check_sibling_lock_pins.py" \ + --lock "${CONSUMER_LOCK}" \ + "${PREFLIGHT_REPO_ARGS[@]}" \ + --output "${PIN_COMPARISON_OUT}" \ + "${preflight_extra[@]}"; then + echo "ERROR: sibling-pin preflight failed against ${CONSUMER_LOCK}" >&2 + echo " canonical clones drift from the lock; sync clones to the" >&2 + echo " locked SHAs (or set ALLOW_SIBLING_PIN_DRIFT=1 with an" >&2 + echo " explicit operator decision) before rebuilding (OMN-12977)." >&2 + exit 3 + fi +else + echo "ERROR: consuming lock not found: ${CONSUMER_LOCK}" >&2 + echo " set CONSUMER_LOCK to the consuming repo's uv.lock (OMN-12977)." >&2 + exit 3 +fi + STAGING_DIR="workspace/sibling-repos" mkdir -p "${STAGING_DIR}" @@ -59,6 +116,15 @@ for repo in "${SIBLING_REPOS[@]}"; do --exclude='.venv' \ --exclude='*.egg-info' \ "${src}/" "${dst}/" + # Record the source HEAD SHA so the lock-pin preflight and provenance can + # identify exactly which commit was vendored. rsync drops .git, so without + # this marker the staged tree has no recoverable SHA (OMN-12987). + if git -C "${src}" rev-parse HEAD >/dev/null 2>&1; then + git -C "${src}" rev-parse HEAD > "${dst}/.build-sha" + else + echo "ERROR: cannot resolve HEAD SHA for ${src}; refusing to stage an unverifiable tree" >&2 + exit 3 + fi done echo "workspace staging complete: ${#SIBLING_REPOS[@]} repos staged to ${STAGING_DIR}" diff --git a/scripts/sync-node-migrations.sh b/scripts/sync-node-migrations.sh index 6cffcd8254..4f2d3ce5d0 100755 --- a/scripts/sync-node-migrations.sh +++ b/scripts/sync-node-migrations.sh @@ -95,17 +95,20 @@ PY OMK_RESOLVED="$(resolve_omnimarket_src || true)" if [ -z "${OMK_RESOLVED}" ]; then - # In --check mode (pre-commit / CI), the omnimarket source tree is frequently - # not checked out. The vendored files are the durable source of truth and are - # validated independently by the migration tests; a drift check is only - # meaningful when the source is present. Skip (success) rather than fail so the - # gate never produces false negatives on runners without omnimarket. - if [ "${CHECK_MODE}" -eq 1 ]; then - echo "[sync-node-migrations] omnimarket source not resolvable — skipping drift check." >&2 - exit 0 - fi echo "[sync-node-migrations] ERROR: could not resolve omnimarket source tree." >&2 echo " Set OMNIMARKET_SRC=, or OMNI_HOME=, or pip install omnimarket." >&2 + if [ "${CHECK_MODE}" -eq 1 ]; then + # OMN-13062 (retro A-10): --check with unresolvable source is a vacuous gate. + # Silently passing when the source is absent means drift is never detected. + # Exit 2 (source-unresolvable) so the CI gate fires and the operator is + # required to either supply OMNIMARKET_SRC / OMNI_HOME or explicitly skip + # this check via SYNC_NODE_MIGRATIONS_SKIP_UNRESOLVABLE=1. + if [ "${SYNC_NODE_MIGRATIONS_SKIP_UNRESOLVABLE:-0}" = "1" ]; then + echo "[sync-node-migrations] SYNC_NODE_MIGRATIONS_SKIP_UNRESOLVABLE=1 — skipping unresolvable-source error." >&2 + exit 0 + fi + exit 2 + fi exit 2 fi diff --git a/scripts/validation/check_topic_completeness.py b/scripts/validation/check_topic_completeness.py index e9cbb875f9..083450c7f0 100644 --- a/scripts/validation/check_topic_completeness.py +++ b/scripts/validation/check_topic_completeness.py @@ -8,7 +8,8 @@ # # Scans Python source files for ONEX topic string literals and validates that # every topic found in source is registered in the topic registry (union of -# contract YAML topics, generated enum values, and manual constants files). +# contract YAML topics, topics.yaml manifests, generated enum values, and manual +# constants files). # # This script catches the case where a developer adds a topic string literal # to Python code without registering it in the canonical topic registry. @@ -17,7 +18,8 @@ # - File types: .py only # - Exclude paths: tests/, docs/, fixtures/, files with '# GENERATED' header # - Match: string literals matching onex.(evt|cmd|dlq|intent).* (with version) -# - Allowlist: completeness-allowlist.yaml for known false positives +# - Temporary suppressions: completeness-allowlist.yaml entries must carry +# owner, reason, created_at, expires_at, and replacement_authority metadata # # Usage: # uv run python scripts/validation/check_topic_completeness.py @@ -31,14 +33,14 @@ # 1 one or more unregistered topics found in source # 2 runtime error (bad arguments, unreadable files) # -# NOTE: stdlib only — no third-party dependencies. - from __future__ import annotations import argparse import ast import re import sys +from dataclasses import dataclass +from datetime import UTC, date, datetime from pathlib import Path # --------------------------------------------------------------------------- @@ -63,6 +65,32 @@ "subscribe_topics", "publish_topics", ) +_SUPPRESSION_REQUIRED_FIELDS: frozenset[str] = frozenset( + { + "topic", + "owner", + "reason", + "created_at", + "expires_at", + "replacement_authority", + } +) + + +class SuppressionConfigError(ValueError): + """Raised when topic-completeness suppressions are malformed.""" + + +@dataclass(frozen=True) +class TopicSuppression: + """Temporary topic-completeness suppression with required ownership.""" + + topic: str + owner: str + reason: str + created_at: date + expires_at: date + replacement_authority: str # --------------------------------------------------------------------------- @@ -357,63 +385,142 @@ def scan_source_for_topics( # --------------------------------------------------------------------------- -# Allowlist loading +# Temporary suppression loading # --------------------------------------------------------------------------- -def load_allowlist(allowlist_path: Path) -> set[str]: - """Load topic allowlist from a YAML file. +def _parse_suppression_date(value: object, *, field: str, topic: str) -> date: + if isinstance(value, datetime): + return value.date() + if isinstance(value, date): + return value + if isinstance(value, str): + try: + return date.fromisoformat(value) + except ValueError as exc: + raise SuppressionConfigError( + f"{topic}: {field} must be an ISO date (YYYY-MM-DD)" + ) from exc + raise SuppressionConfigError(f"{topic}: {field} must be an ISO date (YYYY-MM-DD)") - The allowlist is a simple list of topic strings that are known false - positives (e.g., examples in validators, test fixtures in src/). - Format (YAML): - allowlist: - - onex.evt.example.test-topic.v1 - - onex.cmd.example.another-topic.v1 +def _require_non_empty_string( + item: dict[object, object], *, field: str, topic: str +) -> str: + value = item.get(field) + if not isinstance(value, str) or not value.strip(): + raise SuppressionConfigError(f"{topic}: {field} is required") + return value.strip() - Fallback format (plain text, one topic per line): - onex.evt.example.test-topic.v1 - onex.cmd.example.another-topic.v1 - """ - if not allowlist_path.exists(): - return set() +def _load_yaml_or_empty(path: Path) -> object: try: - text = allowlist_path.read_text(encoding="utf-8") - except OSError: - return set() + text = path.read_text(encoding="utf-8") + except OSError as exc: + raise SuppressionConfigError(f"could not read suppression file {path}") from exc + + if not text.strip(): + return {} - # Try YAML parsing first try: import yaml + except ImportError as exc: + raise SuppressionConfigError( + "PyYAML is required to parse topic-completeness suppressions" + ) from exc - data = yaml.safe_load(text) - if isinstance(data, dict) and "allowlist" in data: - items = data["allowlist"] - if isinstance(items, list): - return {str(item) for item in items if item} - except ImportError: - pass + loaded = yaml.safe_load(text) + return loaded if loaded is not None else {} - # Fallback: plain text format - topics: set[str] = set() - for line in text.splitlines(): - line = line.strip() - if not line or line.startswith("#"): - continue - # Skip YAML keys like "allowlist:" - if line.endswith(":"): - continue - # Strip leading "- " for YAML list items - if line.startswith("- "): - line = line[2:].strip() - # Strip quotes - line = line.strip("'\"") - if _TOPIC_PATTERN.match(line): - topics.add(line) - return topics +def load_topic_suppressions( + suppression_path: Path, *, today: date | None = None +) -> set[str]: + """Load active temporary topic suppressions from a YAML file. + + Permanent topic allowlists are not accepted. Every active suppression must + declare ownership, rationale, creation/expiry dates, and the replacement + authority that should eventually register the topic. + + Format (YAML): + suppressions: + - topic: onex.evt.example.test-topic.v1 + owner: "@team-or-person" + reason: "Temporary source literal while contract lands." + created_at: "2026-06-11" + expires_at: "2026-06-18" + replacement_authority: "contracts/example/contract.yaml" + """ + if not suppression_path.exists(): + return set() + + data = _load_yaml_or_empty(suppression_path) + if not isinstance(data, dict): + raise SuppressionConfigError( + "suppression file must be a mapping with a 'suppressions' list" + ) + + legacy_allowlist = data.get("allowlist") + if legacy_allowlist: + raise SuppressionConfigError( + "legacy 'allowlist' entries are no longer valid; use metadata-rich " + "'suppressions' entries with owner, reason, created_at, expires_at, " + "and replacement_authority" + ) + + suppressions = data.get("suppressions", []) + if suppressions is None: + suppressions = [] + if not isinstance(suppressions, list): + raise SuppressionConfigError("'suppressions' must be a list") + + active_topics: set[str] = set() + today = today or datetime.now(tz=UTC).date() + for index, item in enumerate(suppressions, start=1): + if not isinstance(item, dict): + raise SuppressionConfigError( + f"suppression #{index} must be a mapping with required metadata" + ) + + topic = _require_non_empty_string(item, field="topic", topic=f"#{index}") + missing = sorted(_SUPPRESSION_REQUIRED_FIELDS - {str(key) for key in item}) + if missing: + raise SuppressionConfigError( + f"{topic}: missing required field(s): {', '.join(missing)}" + ) + if not _TOPIC_PATTERN.match(topic): + raise SuppressionConfigError(f"{topic}: invalid ONEX topic name") + + created_at = _parse_suppression_date( + item.get("created_at"), field="created_at", topic=topic + ) + expires_at = _parse_suppression_date( + item.get("expires_at"), field="expires_at", topic=topic + ) + if expires_at < created_at: + raise SuppressionConfigError( + f"{topic}: expires_at must be on or after created_at" + ) + + suppression = TopicSuppression( + topic=topic, + owner=_require_non_empty_string(item, field="owner", topic=topic), + reason=_require_non_empty_string(item, field="reason", topic=topic), + created_at=created_at, + expires_at=expires_at, + replacement_authority=_require_non_empty_string( + item, field="replacement_authority", topic=topic + ), + ) + if suppression.expires_at >= today: + active_topics.add(suppression.topic) + + return active_topics + + +def load_allowlist(allowlist_path: Path) -> set[str]: + """Backward-compatible alias for temporary suppression loading.""" + return load_topic_suppressions(allowlist_path) # --------------------------------------------------------------------------- @@ -434,7 +541,7 @@ def check_completeness( src_root: Root directory of Python source to scan. contracts_root: Root directory containing contract.yaml files. constants_files: Additional Python files defining topic constants. - allowlist_path: Path to allowlist file for false positives. + allowlist_path: Path to temporary suppression file for false positives. manifest_roots: Additional directories with topics.yaml manifests (omniclaude skills, CLI relays, services). @@ -461,18 +568,18 @@ def check_completeness( if manifest_roots: registry.update(collect_registry_from_manifests(manifest_roots)) - # Load allowlist - allowlist: set[str] = set() + # Load active temporary suppressions. Permanent bare allowlists are rejected. + suppressed_topics: set[str] = set() if allowlist_path is not None: - allowlist = load_allowlist(allowlist_path) + suppressed_topics = load_topic_suppressions(allowlist_path) # Scan source for topic literals all_hits = scan_source_for_topics(src_root) - # Filter: keep only topics NOT in registry and NOT in allowlist + # Filter: keep only topics NOT in registry and NOT temporarily suppressed unregistered: list[tuple[Path, str, int]] = [] for path, topic, lineno in all_hits: - if topic not in registry and topic not in allowlist: + if topic not in registry and topic not in suppressed_topics: unregistered.append((path, topic, lineno)) return unregistered, registry @@ -520,7 +627,7 @@ def _build_parser() -> argparse.ArgumentParser: type=Path, default=None, help=( - "Path to topic allowlist file " + "Path to topic temporary suppression file " "(default: scripts/validation/completeness-allowlist.yaml)" ), ) @@ -627,17 +734,24 @@ def main(argv: list[str] | None = None) -> int: return 0 # Run completeness check - unregistered, registry = check_completeness( - src_root=src_root, - contracts_root=contracts_root, - constants_files=( - [cf.resolve() for cf in args.constants_file] - if args.constants_file - else None - ), - allowlist_path=allowlist_path, - manifest_roots=manifest_roots if manifest_roots else None, - ) + try: + unregistered, registry = check_completeness( + src_root=src_root, + contracts_root=contracts_root, + constants_files=( + [cf.resolve() for cf in args.constants_file] + if args.constants_file + else None + ), + allowlist_path=allowlist_path, + manifest_roots=manifest_roots if manifest_roots else None, + ) + except SuppressionConfigError as exc: + print( + f"ERROR: invalid topic-completeness suppression config: {exc}", + file=sys.stderr, + ) + return 2 if not unregistered: if not args.quiet: @@ -664,11 +778,12 @@ def main(argv: list[str] | None = None) -> int: ) print( "\nTo fix: register the topic in the appropriate contract.yaml or " - "topic constants file.", + "topics.yaml manifest.", file=sys.stderr, ) print( - "If this is a false positive, add it to the allowlist at:" + "Temporary suppressions must declare owner, reason, created_at, " + "expires_at, and replacement_authority in:" "\n scripts/validation/completeness-allowlist.yaml", file=sys.stderr, ) diff --git a/scripts/validation/completeness-allowlist.yaml b/scripts/validation/completeness-allowlist.yaml index 9c7e6786db..f86b9f5ff6 100644 --- a/scripts/validation/completeness-allowlist.yaml +++ b/scripts/validation/completeness-allowlist.yaml @@ -1,7 +1,19 @@ # SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. # SPDX-License-Identifier: MIT -# Topic Completeness Allowlist (OMN-3257) +# Topic Completeness Temporary Suppressions (OMN-12979) # -# All topics migrated to contract YAML or standalone manifests as of OMN-4622. -# This file is intentionally empty. -allowlist: [] +# Topic authority belongs in contract YAML and topics.yaml manifests, not in +# this file. This file is only for temporary validation false positives. +# +# Every suppression entry must include: +# topic, owner, reason, created_at, expires_at, replacement_authority +# +# Example: +# suppressions: +# - topic: onex.evt.example.temporary-topic.v1 +# owner: "@infra" +# reason: "Waiting for contract authority to land." +# created_at: "2026-06-11" +# expires_at: "2026-06-18" +# replacement_authority: "src/omnibase_infra/nodes/example/contract.yaml" +suppressions: [] diff --git a/scripts/worktree-gc.sh b/scripts/worktree-gc.sh new file mode 100755 index 0000000000..c0136d0b17 --- /dev/null +++ b/scripts/worktree-gc.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +# worktree-gc.sh — Merge-triggered worktree GC driver (OMN-13008). +# +# WHY: stale worktrees for already-merged PRs accumulate under the worktrees root +# and (alongside docker images) filled /data on .201 on 2026-06-11. This driver +# garbage-collects them on a schedule, removing ONLY worktrees whose PR is merged +# (or whose remote branch is gone) AND that are clean AND fully pushed. +# +# It does NOT reimplement the safety logic — it drives the canonical +# omniclaude/scripts/prune-worktrees.sh, which already enforces: +# - remove only if PR MERGED or remote branch gone +# - SKIP if working tree dirty +# - SKIP if unpushed commits (no-upstream defaults to SKIP, never DELETE) +# +# This is the single GC core used on BOTH machines: +# - Mac: invoked from the merge-sweep CronCreate tick path (launchd does not fire +# on this Mac; session crons are the working scheduler). +# - .201: invoked from deploy/worktree-gc.timer (systemd user timer) against +# /data/omninode/omni_worktrees. +# +# Usage: +# ./scripts/worktree-gc.sh # DRY RUN (default) +# ./scripts/worktree-gc.sh --execute # actually prune merged worktrees +# ./scripts/worktree-gc.sh --worktrees-root # override (default: $OMNI_HOME/omni_worktrees) +# ./scripts/worktree-gc.sh --prune-script # override path to prune-worktrees.sh +# +# Exit codes: 0 success, 2 bad args, 3 prune script not found. +# +# Log: ~/.local/log/onex/worktree-gc.log + +set -euo pipefail + +EXECUTE=false +# Resolve OMNI_HOME fail-fast-ish: prefer env, else derive from this script's repo root. +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" # .../omnibase_infra +DEFAULT_OMNI_HOME="$(cd "$REPO_ROOT/.." && pwd)" # .../omni_home (canonical registry sibling layout) +OMNI_HOME="${OMNI_HOME:-$DEFAULT_OMNI_HOME}" + +WORKTREES_ROOT="${ONEX_WORKTREES_ROOT:-$OMNI_HOME/omni_worktrees}" +# prune-worktrees.sh lives in the omniclaude canonical clone next to omnibase_infra. +PRUNE_SCRIPT="${OMNI_HOME}/omniclaude/scripts/prune-worktrees.sh" +LOG_FILE="${HOME}/.local/log/onex/worktree-gc.log" + +while [[ $# -gt 0 ]]; do + case "$1" in + --execute) EXECUTE=true; shift ;; + --worktrees-root) WORKTREES_ROOT="$2"; shift 2 ;; + --prune-script) PRUNE_SCRIPT="$2"; shift 2 ;; + --help|-h) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "Unknown argument: $1" >&2; exit 2 ;; + esac +done + +mkdir -p "$(dirname "$LOG_FILE")" +log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] [worktree-gc] $*" | tee -a "$LOG_FILE" >&2; } + +if [[ ! -f "$PRUNE_SCRIPT" ]]; then + log "ERROR: prune script not found at $PRUNE_SCRIPT (set --prune-script or OMNI_HOME)" + exit 3 +fi +if [[ ! -d "$WORKTREES_ROOT" ]]; then + log "worktrees root absent: $WORKTREES_ROOT — nothing to GC" + exit 0 +fi + +log "Starting ($( [[ "$EXECUTE" == true ]] && echo EXECUTE || echo DRY-RUN )), root=$WORKTREES_ROOT" + +ARGS=(--worktrees-root "$WORKTREES_ROOT") +[[ "$EXECUTE" == true ]] && ARGS+=(--execute) + +# Drive the canonical prune script. Its own safety checks are authoritative. +bash "$PRUNE_SCRIPT" "${ARGS[@]}" 2>&1 | tee -a "$LOG_FILE" + +log "Done." diff --git a/src/omnibase_infra/cli/cli_delegate.py b/src/omnibase_infra/cli/cli_delegate.py new file mode 100644 index 0000000000..0ea566c59d --- /dev/null +++ b/src/omnibase_infra/cli/cli_delegate.py @@ -0,0 +1,266 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""``onex delegate`` — single-command local-LLM delegation (OMN-13096). + +Phase 2b of the skill-output-suppression slice +(``docs/plans/2026-06-12-skill-output-suppression-plan.md``): the +governing simplification is that **a dispatch skill IS one CLI call**. + +``onex delegate "" [--task-type X] [--max-tokens N]`` wraps, inside +the CLI entrypoint: + +1. task-type classification (when ``--task-type`` is omitted), +2. typed payload construction written to ``/tmp/.json`` + (run_id-suffixed scratch — never ``/tmp``; ``feedback_no_tmp_use_workspace``), +3. resolution of the packaged ``node_delegate_skill_orchestrator`` contract, +4. dispatch through the OMN-13094 receipt-mode path + (:func:`omnibase_infra.cli.receipt_mode.run_receipt_mode`). + +stdout receives exactly ONE +:class:`~omnibase_core.models.dispatch.model_skill_result.ModelSkillResult` +JSON whose ``result`` is the FULL +``ModelDelegateSkillResponse`` (status, response, model_name, provider, +task_type, quality_gate_passed, metrics). RuntimeLocal logs, envelope dumps, +and progress go to the capture file + artifact store and never reach the +caller. Non-zero exit on failure. + +This replaces the multi-step ``omniclaude`` delegate shim +(payload temp file + ``cd omnimarket`` + ``onex node`` log flood + +``cat workflow_result.json``) with one command, one typed result. + +.. versionadded:: OMN-13096 +""" + +from __future__ import annotations + +import json +import sys +import uuid +from pathlib import Path + +import click + +from omnibase_infra.cli.cli_node import _resolve_packaged_contract +from omnibase_infra.cli.receipt_mode import ( + default_emit_socket_path, + run_receipt_mode, +) + +__all__ = [ + "DELEGATE_NODE_NAME", + "DELEGATE_SOURCE", + "DEFAULT_MAX_TOKENS", + "DEFAULT_TASK_TYPE", + "TASK_TYPE_CHOICES", + "classify_task_type", + "run_delegate", +] + +# The omnimarket node that owns the consumer-facing delegation contract. It is +# registered under the ``onex.nodes`` entry-point group and resolvable from +# this CLI's environment (the delegate node ships its packaged contract.yaml). +DELEGATE_NODE_NAME = "node_delegate_skill_orchestrator" + +# Registered adapter source for ``ModelDelegateSkillRequest`` (Literal field). +DELEGATE_SOURCE = "claude-code" + +# Default response budget when ``--max-tokens`` is omitted. Matches the +# historical delegate-shim default; the node enforces its own hard ceiling. +DEFAULT_MAX_TOKENS = 2048 + +# Fallback classification when no keyword matches (the prompt.md default). +DEFAULT_TASK_TYPE = "research" + +# The MVP task taxonomy the delegate node accepts (a subset of the +# ``ModelDelegateSkillRequest`` Literal — the surfaces the shim exposed). +# Source of truth for routing is the node contract's allowed_task_types; this +# CLI exposes the same classification mapping the legacy skill markdown used. +TASK_TYPE_CHOICES = ( + "test", + "document", + "research", + "code_generation", + "refactor", + "reasoning", + "review", +) + +# Ordered keyword → task_type rules (first match wins). Lifted verbatim from +# the legacy ``delegate/prompt.md`` classification table so behavior is +# unchanged; the glue now lives in the CLI, not skill markdown. +_CLASSIFICATION_RULES: tuple[tuple[tuple[str, ...], str], ...] = ( + (("test", "pytest", "unit test", "assert"), "test"), + (("document", "docstring", "readme", "explain how"), "document"), + (("refactor", "cleanup", "simplify"), "refactor"), + (("review", "audit", "check"), "review"), + (("reason", "think through", "decide", "compare"), "reasoning"), + (("write", "create", "implement", "build", "generate"), "code_generation"), +) + + +def classify_task_type(prompt: str) -> str: + """Classify ``prompt`` into a delegate task type via keyword match. + + First matching rule wins; falls back to :data:`DEFAULT_TASK_TYPE` when no + keyword is present. Matching is case-insensitive against the lowercased + prompt. This is the same table the legacy skill markdown carried — the + glue moved into the CLI per the plan's single-command simplification. + """ + lowered = prompt.lower() + for keywords, task_type in _CLASSIFICATION_RULES: + if any(keyword in lowered for keyword in keywords): + return task_type + return DEFAULT_TASK_TYPE + + +def _write_payload( + *, prompt: str, task_type: str, max_tokens: int, state_root: Path, run_id: uuid.UUID +) -> Path: + """Write the delegation input payload to run_id-suffixed scratch. + + Scratch lives under ``/tmp/`` (never ``/tmp`` — + ``feedback_no_tmp_use_workspace``). The payload validates against the + delegate node's input model (``ModelDelegateSkillRequest``); only the + fields the consumer supplies are set. + """ + tmp_dir = state_root / "tmp" + tmp_dir.mkdir(parents=True, exist_ok=True) + payload_path = tmp_dir / f"delegate-input-{run_id}.json" + payload_path.write_text( + json.dumps( + { + "prompt": prompt, + "task_type": task_type, + "source": DELEGATE_SOURCE, + "max_tokens": max_tokens, + } + ), + encoding="utf-8", + ) + return payload_path + + +@click.command("delegate") +@click.argument("prompt") +@click.option( + "--task-type", + "task_type", + type=click.Choice(TASK_TYPE_CHOICES), + default=None, + help=( + "Task classification for routing. Omit to auto-classify from the " + "prompt keywords (default fallback: research)." + ), +) +@click.option( + "--max-tokens", + "max_tokens", + type=click.IntRange(min=1), + default=DEFAULT_MAX_TOKENS, + show_default=True, + help="Maximum tokens for the delegated LLM response.", +) +@click.option( + "--state-root", + type=click.Path(path_type=Path), + default=".onex_state", + show_default=True, + help="Root directory for disk state, scratch payloads, and captures.", +) +@click.option( + "--timeout", + type=int, + default=300, + show_default=True, + help="Max delegation time in seconds.", +) +@click.option( + "--verbose", + "-v", + is_flag=True, + default=False, + help="Capture DEBUG-level logging (still routed to the capture file).", +) +@click.option( + "--emit-socket", + "emit_socket", + type=click.Path(path_type=Path), + default=None, + help=( + "Unix socket of the emit daemon for capture events (default: " + "~/.claude/emit.sock). Unreachable daemon => events spool under " + "/emit_spool/ for later replay." + ), +) +def delegate_command( + prompt: str, + task_type: str | None, + max_tokens: int, + state_root: Path, + timeout: int, + verbose: bool, + emit_socket: Path | None, +) -> None: + """Delegate PROMPT to a local LLM and print exactly one typed result. + + stdout carries exactly ONE ``ModelSkillResult[ModelDelegateSkillResponse]`` + JSON — the full LLM response and metrics, never truncated. RuntimeLocal + logs go to a capture file + the content-addressed artifact store, never to + stdout. Exits non-zero on failure. + + \b + Examples: + onex delegate "explain what a calendar app needs" + onex delegate "write a Python HTTP server" --task-type code_generation + onex delegate "analyze the routing architecture" --max-tokens 4096 + """ + sys.exit( + run_delegate( + prompt=prompt, + task_type=task_type, + max_tokens=max_tokens, + state_root=state_root, + timeout=timeout, + verbose=verbose, + emit_socket=emit_socket, + ) + ) + + +def run_delegate( + *, + prompt: str, + task_type: str | None, + max_tokens: int, + state_root: Path, + timeout: int, + verbose: bool, + emit_socket: Path | None, +) -> int: + """Build the payload, resolve the contract, and dispatch in receipt mode. + + Returns the process exit code. Payload construction, node dispatch, and + result extraction are all internal — the caller supplies a prompt and + receives one typed receipt on stdout. + """ + resolved_task_type = task_type or classify_task_type(prompt) + run_id = uuid.uuid4() + payload_path = _write_payload( + prompt=prompt, + task_type=resolved_task_type, + max_tokens=max_tokens, + state_root=state_root, + run_id=run_id, + ) + contract_path = _resolve_packaged_contract(DELEGATE_NODE_NAME) + return run_receipt_mode( + node_name=DELEGATE_NODE_NAME, + contract_path=contract_path, + input_path=payload_path, + state_root=state_root, + backend_overrides={"event_bus": "inmemory"}, + timeout=timeout, + verbose=verbose, + emit_socket=emit_socket or default_emit_socket_path(), + ) diff --git a/src/omnibase_infra/cli/cli_node.py b/src/omnibase_infra/cli/cli_node.py index 117b6988d7..f08f83b7e5 100644 --- a/src/omnibase_infra/cli/cli_node.py +++ b/src/omnibase_infra/cli/cli_node.py @@ -15,6 +15,7 @@ from __future__ import annotations import importlib.util +import json import logging import sys from importlib.metadata import entry_points @@ -22,11 +23,41 @@ import click +from omnibase_core.enums.enum_workflow_result import EnumWorkflowResult from omnibase_core.models.errors.model_onex_error import ModelOnexError +from omnibase_infra.cli.receipt_mode import ( + default_emit_socket_path, + run_receipt_mode, +) from omnibase_infra.runtime.runtime_local import RuntimeLocal, parse_backend_overrides from omnibase_infra.utils.util_error_sanitization import sanitize_error_message +def _emit_skill_routing_error( + node_id: str, + result: EnumWorkflowResult, + last_error: str | None, +) -> None: + """Emit the documented ``SkillRoutingError`` JSON envelope to stdout. + + Skill shells (e.g. ``golden_chain_sweep``) dispatch via ``onex node`` and + document that a non-zero exit surfaces a ``SkillRoutingError`` JSON envelope + rather than a raw traceback (OMN-8724). The default (non-receipt) path + previously left only the runtime's stderr traceback, so callers had no + machine-readable envelope to surface. This emits the same envelope shape as + ``omnibase_core.cli.cli_run_node._emit_error`` so both dispatch surfaces are + consistent. + """ + envelope: dict[str, str] = { + "error_type": "SkillRoutingError", + "message": last_error + or f"Node '{node_id}' run did not complete (result={result.value}).", + "node_id": node_id, + "result": result.value, + } + click.echo(json.dumps(envelope, indent=2)) + + def _resolve_packaged_contract(node_name: str) -> Path: """Resolve ``node_name`` via the ``onex.nodes`` entry-point group → packaged ``contract.yaml``. @@ -126,6 +157,30 @@ def _entry_point_module(value: str) -> str: default=False, help="Enable DEBUG-level logging (default is INFO).", ) +@click.option( + "--output", + "output_mode", + type=click.Choice(["default", "receipt"]), + default="default", + show_default=True, + help=( + "Output mode. 'receipt' routes ALL runtime logging to a capture " + "file under the state root, content-addresses the capture log and " + "handler result in the artifact store, and prints exactly one typed " + "ModelSkillResult JSON to stdout (OMN-13094)." + ), +) +@click.option( + "--emit-socket", + "emit_socket", + type=click.Path(path_type=Path), + default=None, + help=( + "Unix socket of the emit daemon for receipt-mode capture events " + "(default: ~/.claude/emit.sock). Unreachable daemon => events spool " + "under /emit_spool/ for later replay." + ), +) def run_node_by_name( node_name: str, contract_path: Path | None, @@ -134,6 +189,8 @@ def run_node_by_name( backend: tuple[str, ...], timeout: int, verbose: bool, + output_mode: str, + emit_socket: Path | None, ) -> None: """Run a packaged ONEX node on the local runtime, resolved by NAME. @@ -151,13 +208,8 @@ def run_node_by_name( onex node merge_sweep onex node merge_sweep --input fixtures/real_prs.json onex node merge_sweep --contract ./custom_contract.yaml --state-root ./state + onex node merge_sweep --output receipt # one typed result JSON on stdout """ - logging.basicConfig( - level=logging.DEBUG if verbose else logging.INFO, - format="%(asctime)s %(levelname)-8s %(name)s — %(message)s", - datefmt="%H:%M:%S", - ) - resolved_contract = contract_path or _resolve_packaged_contract(node_name) try: @@ -166,6 +218,28 @@ def run_node_by_name( click.echo(f"Error: {sanitize_error_message(exc)}", err=True) sys.exit(1) + if output_mode == "receipt": + # Receipt mode (OMN-13094): runtime logging goes to a capture file, + # never the console; stdout carries exactly one ModelSkillResult JSON. + sys.exit( + run_receipt_mode( + node_name=node_name, + contract_path=resolved_contract, + input_path=input_path, + state_root=state_root, + backend_overrides=backend_overrides, + timeout=timeout, + verbose=verbose, + emit_socket=emit_socket or default_emit_socket_path(), + ) + ) + + logging.basicConfig( + level=logging.DEBUG if verbose else logging.INFO, + format="%(asctime)s %(levelname)-8s %(name)s — %(message)s", + datefmt="%H:%M:%S", + ) + runtime = RuntimeLocal( workflow_path=resolved_contract, state_root=state_root, @@ -173,5 +247,7 @@ def run_node_by_name( input_path=input_path, timeout=timeout, ) - runtime.run() + result = runtime.run() + if result is not EnumWorkflowResult.COMPLETED: + _emit_skill_routing_error(node_name, result, runtime.last_error) sys.exit(runtime.exit_code) diff --git a/src/omnibase_infra/cli/cli_skill.py b/src/omnibase_infra/cli/cli_skill.py new file mode 100644 index 0000000000..a7f8b629fd --- /dev/null +++ b/src/omnibase_infra/cli/cli_skill.py @@ -0,0 +1,279 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""``onex skill [args]`` — single-command dispatch surface (OMN-13097). + +Phase 4a of the skill-output-suppression slice +(``docs/plans/2026-06-12-skill-output-suppression-plan.md`` Phase 4 item 1). + +A dispatch skill IS one CLI call (user directive 2026-06-12). This command +replaces the 24 hand-written dispatch shims: it resolves the skill via the +declarative ``skill_mapping.yaml`` registry, builds the backing node's input +payload from the skill's CLI arguments, and dispatches through the proven +receipt-mode path (``run_receipt_mode``, OMN-13094). stdout carries exactly +one typed :class:`~omnibase_core.models.dispatch.model_skill_result.ModelSkillResult` +JSON with the FULL handler result; runtime logs go to the capture file. + +The skill→node mapping is DECLARATIVE DATA in ``skill_mapping.yaml`` — never +hardcoded Python branching here (ticket deliverable 2). Adding a skill is a +YAML edit plus a fixture, not a code change. + +.. versionadded:: OMN-13097 +""" + +from __future__ import annotations + +import json +import logging +import uuid +from functools import lru_cache +from pathlib import Path + +import click +import yaml +from pydantic import JsonValue, ValidationError + +from omnibase_infra.cli.cli_node import _resolve_packaged_contract +from omnibase_infra.cli.enum_skill_arg_type import EnumSkillArgType +from omnibase_infra.cli.model_skill_arg_spec import ModelSkillArgSpec +from omnibase_infra.cli.model_skill_mapping import ModelSkillMapping +from omnibase_infra.cli.model_skill_mapping_registry import ModelSkillMappingRegistry +from omnibase_infra.cli.receipt_mode import ( + default_emit_socket_path, + run_receipt_mode, +) + +__all__ = ["MAPPING_FILENAME", "load_skill_registry", "run_skill_by_name"] + +logger = logging.getLogger(__name__) + +# Declarative skill→node mapping ships beside this module. +MAPPING_FILENAME = "skill_mapping.yaml" + +# Internal scratch for the constructed input payload lives under the state +# root, NEVER /tmp (feedback_no_tmp_use_workspace.md). run_id-suffixed names +# avoid collisions between parallel dispatches. +_TMP_DIR_NAME = "tmp" + + +@lru_cache(maxsize=1) +def load_skill_registry() -> ModelSkillMappingRegistry: + """Load and validate the declarative skill→node mapping registry. + + Raises: + click.ClickException: when the mapping file is missing or invalid. + """ + mapping_path = Path(__file__).parent / MAPPING_FILENAME + if not mapping_path.is_file(): + raise click.ClickException( + f"Skill mapping registry not found at {mapping_path}. " + "This file is required for 'onex skill'." + ) + raw = yaml.safe_load(mapping_path.read_text(encoding="utf-8")) + try: + return ModelSkillMappingRegistry.model_validate(raw) + except ValidationError as exc: + raise click.ClickException( + f"Invalid skill mapping registry at {mapping_path}: {exc}" + ) from exc + + +def _coerce_value(spec: ModelSkillArgSpec, raw: str) -> JsonValue: + """Coerce a raw CLI string to the spec's declared type. Fail fast.""" + if spec.arg_type is EnumSkillArgType.STRING: + return raw + if spec.arg_type is EnumSkillArgType.INTEGER: + try: + return int(raw) + except ValueError as exc: + raise click.ClickException( + f"--{spec.name} expects an integer, got {raw!r}" + ) from exc + if spec.arg_type is EnumSkillArgType.STRING_LIST: + return [item.strip() for item in raw.split(",") if item.strip()] + # BOOLEAN is a presence flag and never reaches value coercion. + raise click.ClickException( + f"--{spec.name}: value coercion not supported for {spec.arg_type}" + ) + + +def _parse_skill_args( + mapping: ModelSkillMapping, extra: tuple[str, ...] +) -> dict[str, JsonValue]: + """Parse trailing ``onex skill`` tokens into a node-input payload. + + Boolean specs are presence flags (``--dry-run``); typed specs take the + next token (``--repos a,b``). A single positional spec collects all + non-flag trailing tokens, joined with spaces. + + Raises: + click.ClickException: on unknown flags, missing values, missing + required args, or coercion failure. + """ + by_flag = {spec.name: spec for spec in mapping.args if not spec.positional} + positional_spec = next((s for s in mapping.args if s.positional), None) + + payload: dict[str, JsonValue] = dict(mapping.static_payload) + positional_tokens: list[str] = [] + seen: set[str] = set() + + tokens = list(extra) + i = 0 + while i < len(tokens): + token = tokens[i] + if token.startswith("--"): + flag = token[2:] + spec = by_flag.get(flag) + if spec is None: + raise click.ClickException( + f"Unknown argument '--{flag}' for skill '{mapping.skill_name}'." + ) + if spec.arg_type is EnumSkillArgType.BOOLEAN: + payload[spec.payload_field] = True + seen.add(spec.name) + i += 1 + continue + if i + 1 >= len(tokens): + raise click.ClickException(f"--{flag} requires a value.") + payload[spec.payload_field] = _coerce_value(spec, tokens[i + 1]) + seen.add(spec.name) + i += 2 + continue + positional_tokens.append(token) + i += 1 + + if positional_tokens: + if positional_spec is None: + raise click.ClickException( + f"Skill '{mapping.skill_name}' takes no positional arguments; " + f"got {positional_tokens!r}." + ) + payload[positional_spec.payload_field] = " ".join(positional_tokens) + seen.add(positional_spec.name) + + # Apply defaults for unset, non-required args; enforce required args. + for spec in mapping.args: + if spec.name in seen: + continue + if spec.required: + label = "positional argument" if spec.positional else f"--{spec.name}" + raise click.ClickException( + f"Skill '{mapping.skill_name}' requires {label}." + ) + if spec.arg_type is EnumSkillArgType.BOOLEAN: + payload.setdefault(spec.payload_field, bool(spec.default)) + elif spec.default is not None: + payload[spec.payload_field] = spec.default + + return payload + + +def _apply_classifiers( + mapping: ModelSkillMapping, payload: dict[str, JsonValue] +) -> None: + """Assign classifier target fields that remain unset after arg parsing.""" + for classifier in mapping.classifiers: + if payload.get(classifier.target_field) is not None: + continue + source = payload.get(classifier.source_field) + source_text = str(source).lower() if source is not None else "" + assigned = classifier.fallback + for keywords, value in classifier.rules: + if any(keyword.lower() in source_text for keyword in keywords): + assigned = value + break + payload[classifier.target_field] = assigned + + +def _write_payload( + state_root: Path, skill_name: str, payload: dict[str, JsonValue] +) -> Path: + """Write the input payload to a run_id-suffixed file under the state root.""" + tmp_dir = state_root / _TMP_DIR_NAME + tmp_dir.mkdir(parents=True, exist_ok=True) + payload_path = tmp_dir / f"{skill_name}-{uuid.uuid4()}.json" + payload_path.write_text(json.dumps(payload), encoding="utf-8") + return payload_path + + +@click.command( + "skill", + context_settings={"ignore_unknown_options": True, "allow_extra_args": True}, +) +@click.argument("skill_name") +@click.option( + "--state-root", + type=click.Path(path_type=Path), + default=".onex_state", + show_default=True, + help="Root directory for disk state and capture files.", +) +@click.option( + "--timeout", + type=int, + default=None, + help="Override the skill's default dispatch timeout (seconds).", +) +@click.option( + "--verbose", + "-v", + is_flag=True, + default=False, + help="Capture DEBUG-level runtime logging (still never on stdout).", +) +@click.option( + "--emit-socket", + "emit_socket", + type=click.Path(path_type=Path), + default=None, + help="Unix socket of the emit daemon (default: ~/.claude/emit.sock).", +) +@click.argument("skill_args", nargs=-1, type=click.UNPROCESSED) +def run_skill_by_name( + skill_name: str, + state_root: Path, + timeout: int | None, + verbose: bool, + emit_socket: Path | None, + skill_args: tuple[str, ...], +) -> None: + """Dispatch a skill to its backing node and print one typed result. + + \b + A dispatch skill is exactly one CLI call: this resolves the declarative + skill→node mapping, builds the node-input payload from SKILL_ARGS, and + runs the dispatch in receipt mode — stdout is exactly one + ModelSkillResult JSON carrying the FULL handler result; runtime logs go + to the capture file under the state root. + + \b + Examples: + onex skill compliance_sweep --repos omnibase_core,omnibase_infra + onex skill dod_verify OMN-1234 + onex skill delegate "summarize this paragraph" --task-type document + """ + registry = load_skill_registry() + mapping = registry.get(skill_name) + if mapping is None: + known = ", ".join(sorted(s.skill_name for s in registry.skills)) or "(none)" + raise click.ClickException( + f"Unknown skill '{skill_name}'. Known skills: {known}" + ) + + payload = _parse_skill_args(mapping, skill_args) + _apply_classifiers(mapping, payload) + + contract_path = _resolve_packaged_contract(mapping.node_name) + payload_path = _write_payload(state_root, skill_name, payload) + + exit_code = run_receipt_mode( + node_name=mapping.node_name, + contract_path=contract_path, + input_path=payload_path, + state_root=state_root, + backend_overrides={"event_bus": mapping.event_bus}, + timeout=timeout if timeout is not None else mapping.timeout, + verbose=verbose, + emit_socket=emit_socket or default_emit_socket_path(), + ) + raise SystemExit(exit_code) diff --git a/src/omnibase_infra/cli/enum_skill_arg_type.py b/src/omnibase_infra/cli/enum_skill_arg_type.py new file mode 100644 index 0000000000..3be5c4e278 --- /dev/null +++ b/src/omnibase_infra/cli/enum_skill_arg_type.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Argument value types for the declarative ``onex skill`` mapping (OMN-13097). + +Each ``onex skill`` argument declares the type its raw CLI string is coerced +to before it lands in the node-input payload. Keeping this an explicit enum +(never a free-form string) means the mapping loader fails fast on an unknown +type rather than silently passing a string through. +""" + +from __future__ import annotations + +from enum import StrEnum + +__all__ = ["EnumSkillArgType"] + + +class EnumSkillArgType(StrEnum): + """Coercion target for a declarative ``onex skill`` argument.""" + + STRING = "string" + INTEGER = "integer" + BOOLEAN = "boolean" + STRING_LIST = "string_list" diff --git a/src/omnibase_infra/cli/model_receipt_runtime_summary.py b/src/omnibase_infra/cli/model_receipt_runtime_summary.py new file mode 100644 index 0000000000..9ebc163af8 --- /dev/null +++ b/src/omnibase_infra/cli/model_receipt_runtime_summary.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Typed receipt ``result`` payload for runs without a handler result (OMN-13094). + +When ``onex node``/``onex run`` executes in ``--output receipt`` mode, the +receipt's ``result`` field carries the node's full handler result whenever one +exists. Runs that produce no handler result (event-only orchestrations) or +that fail carry this summary model instead, so the receipt's ``result`` is +always a typed model and errors are never hidden: on failure the FULL capture +log and error text travel inline in the receipt (parent invariant — see +``docs/plans/2026-06-12-skill-output-suppression-plan.md``, Phase 2 item 1). + +.. versionadded:: OMN-13094 +""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field, JsonValue + +__all__ = ["ModelReceiptRuntimeSummary"] + + +class ModelReceiptRuntimeSummary(BaseModel): + """Receipt ``result`` payload when no handler result exists or the run failed. + + Carries the full ``workflow_result.json`` content plus — on non-success — + the FULL capture log and error text inline. Errors are never hidden + behind an artifact ref. + """ + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + workflow_result: str = Field( + ..., + description=( + "Terminal workflow result value ('completed', 'partial', " + "'failed', 'timeout') or 'error' when the runtime raised before " + "producing a result." + ), + ) + exit_code: int = Field( + ..., + description="CLI exit code corresponding to the workflow result.", + ) + workflow: str = Field( + ..., + description="Path of the contract that was executed.", + ) + terminal_payload: JsonValue = Field( + default=None, + description="Terminal event payload from workflow_result.json, if any.", + ) + handler_result: JsonValue = Field( + default=None, + description="Handler result from workflow_result.json, if any.", + ) + error: str = Field( + default="", + description=( + "Full error text (including traceback) when the runtime raised. " + "Empty when the run produced a terminal workflow result." + ), + ) + capture_log: str = Field( + default="", + description=( + "FULL runtime capture log text, inlined on non-success outcomes " + "so errors are never hidden behind an artifact ref. Empty on " + "success (the log remains retrievable via the receipt's " + "artifact_refs)." + ), + ) diff --git a/src/omnibase_infra/cli/model_skill_arg_spec.py b/src/omnibase_infra/cli/model_skill_arg_spec.py new file mode 100644 index 0000000000..03ed897d6a --- /dev/null +++ b/src/omnibase_infra/cli/model_skill_arg_spec.py @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Declarative ``onex skill`` CLI-argument spec (OMN-13097). + +One arg of an ``onex skill`` invocation, mapping a ``--flag`` (or positional) +onto a field of the backing node's contract input model. +""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, model_validator + +from omnibase_infra.cli.enum_skill_arg_type import EnumSkillArgType + +__all__ = ["ModelSkillArgSpec"] + + +class ModelSkillArgSpec(BaseModel): + """One CLI argument of an ``onex skill`` invocation. + + ``boolean`` args are presence flags (``--dry-run``); all others take a + value (``--repos a,b,c``). + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + name: str = Field( + ..., + min_length=1, + description="CLI flag name without leading dashes (e.g. 'repos', 'dry-run').", + ) + payload_field: str = Field( + ..., + min_length=1, + description="Field name in the backing node's input-model payload.", + ) + arg_type: EnumSkillArgType = Field( + ..., + description="Coercion target for the raw CLI string value.", + ) + required: bool = Field( + default=False, + description="Whether the argument must be supplied (no default applies).", + ) + default: JsonValue = Field( + default=None, + description=( + "Default payload value when the arg is omitted. Ignored when " + "'required' is True. None means the field is omitted entirely " + "(the node-input model supplies its own default)." + ), + ) + positional: bool = Field( + default=False, + description=( + "When True the arg is read from trailing positional tokens " + "(joined with spaces) rather than a --flag. At most one " + "positional arg per skill." + ), + ) + + @model_validator(mode="after") + def _validate_required_has_no_default(self) -> ModelSkillArgSpec: + if self.required and self.default is not None: + raise ValueError( + f"arg '{self.name}': required args must not declare a default" + ) + return self diff --git a/src/omnibase_infra/cli/model_skill_classifier.py b/src/omnibase_infra/cli/model_skill_classifier.py new file mode 100644 index 0000000000..789e2a7d32 --- /dev/null +++ b/src/omnibase_infra/cli/model_skill_classifier.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Declarative ``onex skill`` keyword classifier (OMN-13097). + +Models the delegate skill's ``task_type`` auto-classification as DATA: if the +classified target field is still unset after arg parsing, the source field's +text is scanned (case-insensitive) for the first matching keyword group and +the corresponding value is assigned. +""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field + +__all__ = ["ModelSkillClassifier"] + + +class ModelSkillClassifier(BaseModel): + """Keyword → payload-value classification for an unset argument.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + target_field: str = Field( + ..., + min_length=1, + description="Payload field this classifier assigns when unset.", + ) + source_field: str = Field( + ..., + min_length=1, + description="Payload field whose text is scanned for keywords.", + ) + rules: tuple[tuple[tuple[str, ...], str], ...] = Field( + ..., + description=( + "Ordered ((keyword, ...), value) pairs. First group with any " + "keyword present in the source text wins." + ), + ) + fallback: str = Field( + ..., + min_length=1, + description="Value assigned when no keyword group matches.", + ) diff --git a/src/omnibase_infra/cli/model_skill_mapping.py b/src/omnibase_infra/cli/model_skill_mapping.py new file mode 100644 index 0000000000..1e332bfa2d --- /dev/null +++ b/src/omnibase_infra/cli/model_skill_mapping.py @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Declarative ``onex skill`` → backing-node mapping model (OMN-13097). + +Phase 4a of the skill-output-suppression slice +(``docs/plans/2026-06-12-skill-output-suppression-plan.md`` Phase 4 item 1). + +``onex skill [args]`` is the single-command dispatch surface that +replaces the 24 hand-written dispatch shims. The skill→node mapping is +DECLARATIVE DATA — it lives in ``skill_mapping.yaml`` beside this module and +is loaded into these frozen, typed models; it is NEVER hardcoded as Python +branching in the command (ticket deliverable 2). Each entry declares: + +- the backing ONEX node (resolved via the ``onex.nodes`` entry-point group, + exactly like ``onex node``), +- how the skill's CLI arguments map onto fields of the node's contract input + model (name, type, default, required), +- optional keyword classifiers (the delegate skill's ``task_type`` + auto-classification, expressed as data rather than inline logic), +- static payload fields the node requires regardless of args. + +.. versionadded:: OMN-13097 +""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, model_validator + +from omnibase_infra.cli.model_skill_arg_spec import ModelSkillArgSpec +from omnibase_infra.cli.model_skill_classifier import ModelSkillClassifier + +__all__ = ["ModelSkillMapping"] + + +class ModelSkillMapping(BaseModel): + """One ``onex skill `` → backing node mapping.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + skill_name: str = Field( + ..., + min_length=1, + description="Skill name as invoked: 'onex skill '.", + ) + node_name: str = Field( + ..., + min_length=1, + description=( + "Backing node registered under the 'onex.nodes' entry-point group." + ), + ) + result_model: str = Field( + ..., + min_length=1, + description=( + "Fully qualified name of the backing node's typed handler result " + "model — schema identity surfaced in the receipt (ticket " + "deliverable 3)." + ), + ) + event_bus: str = Field( + default="inmemory", + description="event_bus backend override for the dispatch.", + ) + timeout: int = Field( + default=300, + gt=0, + description="Max dispatch execution time in seconds.", + ) + args: tuple[ModelSkillArgSpec, ...] = Field( + default=(), + description="Declarative CLI-arg → payload-field specs.", + ) + static_payload: dict[str, JsonValue] = Field( + default_factory=dict, + description="Payload fields injected regardless of supplied args.", + ) + classifiers: tuple[ModelSkillClassifier, ...] = Field( + default=(), + description="Keyword classifiers applied to still-unset fields.", + ) + + @model_validator(mode="after") + def _validate_single_positional(self) -> ModelSkillMapping: + positionals = [a for a in self.args if a.positional] + if len(positionals) > 1: + raise ValueError( + f"skill '{self.skill_name}': at most one positional arg allowed, " + f"got {len(positionals)}" + ) + return self diff --git a/src/omnibase_infra/cli/model_skill_mapping_registry.py b/src/omnibase_infra/cli/model_skill_mapping_registry.py new file mode 100644 index 0000000000..7d76745a64 --- /dev/null +++ b/src/omnibase_infra/cli/model_skill_mapping_registry.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Declarative ``onex skill`` mapping registry (OMN-13097). + +The full skill→node registry loaded from ``skill_mapping.yaml``. +""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from omnibase_infra.cli.model_skill_mapping import ModelSkillMapping + +__all__ = ["ModelSkillMappingRegistry"] + + +class ModelSkillMappingRegistry(BaseModel): + """The full declarative skill→node registry loaded from YAML.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + skills: tuple[ModelSkillMapping, ...] = Field( + ..., + description="All declared skill mappings.", + ) + + @model_validator(mode="after") + def _validate_unique_names(self) -> ModelSkillMappingRegistry: + names = [s.skill_name for s in self.skills] + duplicates = sorted({n for n in names if names.count(n) > 1}) + if duplicates: + raise ValueError( + f"duplicate skill_name(s) in mapping registry: {', '.join(duplicates)}" + ) + return self + + def get(self, skill_name: str) -> ModelSkillMapping | None: + """Return the mapping for ``skill_name`` or None when absent.""" + for mapping in self.skills: + if mapping.skill_name == skill_name: + return mapping + return None diff --git a/src/omnibase_infra/cli/receipt_mode.py b/src/omnibase_infra/cli/receipt_mode.py new file mode 100644 index 0000000000..b6289102f6 --- /dev/null +++ b/src/omnibase_infra/cli/receipt_mode.py @@ -0,0 +1,453 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Receipt-mode execution for ``onex node`` / ``onex run`` (OMN-13094). + +Layer A of the skill-output-suppression slice +(``docs/plans/2026-06-12-skill-output-suppression-plan.md``, Phase 2 item 1): + +- ALL runtime logging routes to a run_id-suffixed capture file under the + node's state root — console handlers are never installed, which kills the + 25-50-line RuntimeLocal INFO stream at the source. +- After the run, the full capture log and the full handler result are written + to the content-addressed artifact store (omnibase_core, OMN-13093), + ``artifact.captured`` + ``tool.output.captured`` events are emitted via the + emit daemon socket, and stdout receives exactly ONE typed + :class:`~omnibase_core.models.dispatch.model_skill_result.ModelSkillResult` + JSON object carrying the FULL result. + +Failure asymmetry (capture vs telemetry): + +- Artifact write failure ⇒ the FULL output is printed instead of the receipt + (no hidden loss — parent invariant 1; no silent fallback). +- Artifact write success but event emission failure ⇒ the receipt still + prints; the event is spooled to a local outbox under the state root for + later replay. A transient emit-daemon outage never re-floods the caller + when the artifact exists. + +.. versionadded:: OMN-13094 +""" + +from __future__ import annotations + +import json +import logging +import socket +import time +import traceback +import uuid +from datetime import UTC, datetime +from pathlib import Path +from typing import cast + +import click +from pydantic import JsonValue, ValidationError + +from omnibase_core.artifacts.artifact_store import ArtifactStore +from omnibase_core.enums.enum_skill_result_status import EnumSkillResultStatus +from omnibase_core.enums.enum_workflow_result import EnumWorkflowResult +from omnibase_core.models.artifacts.model_artifact_ref import ModelArtifactRef +from omnibase_core.models.dispatch.model_skill_result import ModelSkillResult +from omnibase_infra.cli.model_receipt_runtime_summary import ModelReceiptRuntimeSummary +from omnibase_infra.runtime.runtime_local import RuntimeLocal + +__all__ = [ + "CAPTURE_DIR_NAME", + "SPOOL_DIR_NAME", + "default_emit_socket_path", + "run_receipt_mode", +] + +logger = logging.getLogger(__name__) + +# Capture logs and the emission spool live under the node's state root +# (matching the workflow_result.json convention — plan Open Question 4). +CAPTURE_DIR_NAME = "captures" +SPOOL_DIR_NAME = "emit_spool" + +# Socket timeout for emit-daemon calls. Emission is telemetry, never the +# critical path — a slow daemon must not stall the dispatch. +_EMIT_TIMEOUT_SECONDS = 2.0 + +_MAX_EMIT_RESPONSE_BYTES = 65536 + +_WORKFLOW_TO_STATUS: dict[EnumWorkflowResult, EnumSkillResultStatus] = { + EnumWorkflowResult.COMPLETED: EnumSkillResultStatus.SUCCESS, + EnumWorkflowResult.PARTIAL: EnumSkillResultStatus.PARTIAL, + EnumWorkflowResult.FAILED: EnumSkillResultStatus.FAILED, + EnumWorkflowResult.TIMEOUT: EnumSkillResultStatus.FAILED, +} + + +def default_emit_socket_path() -> Path: + """Deterministic address of the live emit daemon's Unix domain socket. + + ``~/.claude/emit.sock`` — the same default as omniclaude's emit client + wrapper. This is the daemon's published address; nothing is ever written + there by this module (emission failures spool under the node's state + root instead). Override per-invocation via ``onex node --emit-socket``. + """ + return Path.home() / ".claude" / "emit.sock" + + +def _emit_via_socket( + socket_path: Path, event_type: str, payload: dict[str, JsonValue] +) -> None: + """Send one event to the emit daemon (newline-delimited JSON protocol). + + Raises on any failure — the caller decides whether to spool. + """ + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.settimeout(_EMIT_TIMEOUT_SECONDS) + try: + sock.connect(str(socket_path)) + request = {"event_type": event_type, "payload": payload} + sock.sendall(json.dumps(request).encode("utf-8") + b"\n") + chunks: list[bytes] = [] + total = 0 + while True: + chunk = sock.recv(4096) + if not chunk: + break + total += len(chunk) + if total > _MAX_EMIT_RESPONSE_BYTES: + raise ConnectionError( + f"emit daemon response exceeded {_MAX_EMIT_RESPONSE_BYTES} bytes" + ) + chunks.append(chunk) + if b"\n" in chunk: + break + raw = b"".join(chunks).split(b"\n", 1)[0].decode("utf-8").strip() + if not raw: + raise ConnectionError("emit daemon closed connection without responding") + response: object = json.loads(raw) + if not isinstance(response, dict) or response.get("status") != "queued": + raise RuntimeError(f"emit daemon rejected {event_type}: {response!r}") + finally: + sock.close() + + +def _emit_or_spool( + event_type: str, + payload: dict[str, JsonValue], + *, + spool_dir: Path, + spool_stem: str, + socket_path: Path, +) -> None: + """Emit ``event_type`` via the daemon socket; spool locally on failure. + + Telemetry asymmetry (plan Phase 2 item 1): once the artifact exists, an + emission failure must never re-flood the caller — the event is recorded + to a local outbox file for later replay and the receipt still prints. + """ + try: + _emit_via_socket(socket_path, event_type, payload) + except (OSError, ValueError, RuntimeError) as exc: + # OSError covers socket/connection/timeout failures; ValueError covers + # malformed daemon JSON; RuntimeError covers daemon rejections. + # Spool-on-failure is the contract here — never raise past this point. + spool_dir.mkdir(parents=True, exist_ok=True) + spool_path = spool_dir / f"{event_type.replace('.', '-')}-{spool_stem}.json" + spool_record = { + "event_type": event_type, + "payload": payload, + "spooled_at_utc": datetime.now(UTC).isoformat(), + "spool_reason": f"{type(exc).__name__}: {exc}", + } + spool_path.write_text(json.dumps(spool_record, indent=2), encoding="utf-8") + logger.warning( + "receipt_mode: emit of %s failed (%s); spooled to %s", + event_type, + type(exc).__name__, + spool_path, + ) + + +def _configure_capture_logging(capture_path: Path, *, verbose: bool) -> None: + """Route ALL logging to ``capture_path``; install no console handlers. + + ``force=True`` removes any pre-existing root handlers so the runtime + INFO stream can never leak to stdout/stderr in receipt mode (F7). + """ + capture_path.parent.mkdir(parents=True, exist_ok=True) + logging.basicConfig( + level=logging.DEBUG if verbose else logging.INFO, + format="%(asctime)s %(levelname)-8s %(name)s — %(message)s", + datefmt="%H:%M:%S", + handlers=[logging.FileHandler(capture_path, encoding="utf-8")], + force=True, + ) + + +def _close_capture_logging() -> None: + """Flush and close all root handlers, then install a ``NullHandler``. + + The ``NullHandler`` prevents Python's ``lastResort`` handler from leaking + post-capture log records (e.g. spool warnings) to stderr — receipt mode + allows nothing but the receipt on stdout/stderr. Spool records carry + their own ``spool_reason``, so the dropped log line loses nothing. + """ + root = logging.getLogger() + for handler in list(root.handlers): + handler.flush() + handler.close() + root.removeHandler(handler) + root.addHandler(logging.NullHandler()) + + +def _fully_qualified_name(obj: object) -> str: + """Return ``module.QualName`` for the concrete type of ``obj``.""" + cls = type(obj) + return f"{cls.__module__}.{cls.__qualname__}" + + +def _extract_correlation_id(workflow_data: dict[str, JsonValue]) -> uuid.UUID: + """Extract the run's correlation_id from workflow state, else mint one.""" + for key in ("handler_result", "terminal_payload"): + candidate = workflow_data.get(key) + if isinstance(candidate, dict): + raw = candidate.get("correlation_id") + if isinstance(raw, str): + try: + return uuid.UUID(raw) + except ValueError: + continue + return uuid.uuid4() + + +def _load_workflow_data(state_root: Path) -> dict[str, JsonValue]: + """Read ``workflow_result.json`` if the runtime wrote one.""" + result_path = state_root / "workflow_result.json" + if not result_path.exists(): + return {} + parsed: object = json.loads(result_path.read_text(encoding="utf-8")) + if not isinstance(parsed, dict): + return {} + return cast("dict[str, JsonValue]", parsed) + + +def _print_full_output_on_capture_failure( + *, + reason: str, + capture_text: str, + workflow_data: dict[str, JsonValue], + runtime_error: str, +) -> None: + """No-hidden-loss path: artifact write failed, so print EVERYTHING. + + Parent invariant 1 / no-silent-fallback: when durable capture is not + possible, suppression is not allowed — the full output passes through. + """ + click.echo( + f"receipt mode: artifact capture failed ({reason}); " + "printing full output instead of a receipt (no hidden loss).", + err=True, + ) + if capture_text: + click.echo(capture_text, nl=False) + if runtime_error: + click.echo(runtime_error, nl=False) + if workflow_data: + click.echo(json.dumps(workflow_data, indent=2)) + + +def run_receipt_mode( + *, + node_name: str, + contract_path: Path, + input_path: Path | None, + state_root: Path, + backend_overrides: dict[str, str], + timeout: int, + verbose: bool, + emit_socket: Path, +) -> int: + """Execute the node and print exactly one ``ModelSkillResult`` JSON. + + Returns the process exit code (the runtime's exit code; 1 when the + runtime raised before producing a workflow result). + """ + run_id = uuid.uuid4() + capture_path = state_root / CAPTURE_DIR_NAME / f"{node_name}-{run_id}.log" + spool_dir = state_root / SPOOL_DIR_NAME + _configure_capture_logging(capture_path, verbose=verbose) + + started = time.monotonic() + runtime_error = "" + workflow_result: EnumWorkflowResult | None = None + exit_code = 1 + handler_result_obj: object | None = None + try: + runtime = RuntimeLocal( + workflow_path=contract_path, + state_root=state_root, + backend_overrides=backend_overrides, + input_path=input_path, + timeout=timeout, + ) + workflow_result = runtime.run() + exit_code = runtime.exit_code + handler_result_obj = runtime.handler_result + except Exception: + # RuntimeLocal.run() records execution failures itself; reaching here + # means contract load / bus construction raised. Errors are never + # hidden: the traceback goes to the capture AND inline in the result. + runtime_error = traceback.format_exc() + logger.exception("receipt_mode: runtime raised") + duration_ms = int((time.monotonic() - started) * 1000) + + _close_capture_logging() + capture_text = ( + capture_path.read_text(encoding="utf-8") if capture_path.exists() else "" + ) + workflow_data = _load_workflow_data(state_root) + correlation_id = _extract_correlation_id(workflow_data) + + if runtime_error: + status = EnumSkillResultStatus.ERROR + else: + # workflow_result is always set when runtime_error is empty. + status = _WORKFLOW_TO_STATUS[ + workflow_result + if workflow_result is not None + else EnumWorkflowResult.FAILED + ] + + # --- Layer B: durable capture (parent invariant 1) ------------------- + handler_result_json = workflow_data.get("handler_result") + artifact_payloads: list[dict[str, JsonValue]] = [] + artifact_refs: list[ModelArtifactRef] = [] + try: + store = ArtifactStore() + captures: list[tuple[bytes, str, str]] = [ + (capture_text.encode("utf-8"), "text/plain", "runtime_capture_log"), + ] + if handler_result_json is not None: + captures.append( + ( + json.dumps(handler_result_json).encode("utf-8"), + "application/json", + "handler_result", + ) + ) + for data, media_type, artifact_kind in captures: + ref = store.write_blob( + data, + media_type=media_type, + artifact_kind=artifact_kind, + source_system="onex_cli", + scope_ref=node_name, + correlation_id=str(correlation_id), + ) + artifact_refs.append(ref) + artifact_payloads.append( + { + "artifact_ref": ref.ref, + "artifact_hash": ref.hex_digest, + "artifact_size_bytes": len(data), + "artifact_media_type": media_type, + "artifact_kind": artifact_kind, + "source_system": "onex_cli", + "scope_ref": node_name, + "correlation_id": str(correlation_id), + "run_id": str(run_id), + "redaction_state": "raw", + } + ) + except (KeyError, OSError, ValueError) as exc: + # Artifact write failed ⇒ NO receipt, FULL output (invariant 1). + reason = f"{type(exc).__name__}: {exc}" + _print_full_output_on_capture_failure( + reason=reason, + capture_text=capture_text, + workflow_data=workflow_data, + runtime_error=runtime_error, + ) + return exit_code + + # --- Telemetry: emit or spool (never re-floods the caller) ----------- + socket_path = emit_socket + for index, event_payload in enumerate(artifact_payloads): + _emit_or_spool( + "artifact.captured", + event_payload, + spool_dir=spool_dir, + spool_stem=f"{run_id}-{index}", + socket_path=socket_path, + ) + _emit_or_spool( + "tool.output.captured", + { + "tool_name": "onex_node", + "node_name": node_name, + "suppression_decision": "receipt_mode", + "correlation_id": str(correlation_id), + "run_id": str(run_id), + "exit_code": exit_code, + "status": status.value, + "artifact_refs": [ref.ref for ref in artifact_refs], + "capture_log_bytes": len(capture_text.encode("utf-8")), + }, + spool_dir=spool_dir, + spool_stem=str(run_id), + socket_path=socket_path, + ) + + # --- Layer A: exactly one typed receipt on stdout --------------------- + metrics: dict[str, float] = { + "capture_log_bytes": float(len(capture_text.encode("utf-8"))), + } + receipt: ModelSkillResult[JsonValue] | ModelSkillResult[ModelReceiptRuntimeSummary] + if ( + status.is_success_like + and handler_result_json is not None + and handler_result_obj is not None + ): + receipt = ModelSkillResult[JsonValue]( + skill_name=node_name, + node_name=node_name, + status=status, + correlation_id=correlation_id, + run_id=run_id, + exit_code=exit_code, + duration_ms=duration_ms, + result=handler_result_json, + result_model=_fully_qualified_name(handler_result_obj), + metrics=metrics, + artifact_refs=artifact_refs, + ) + else: + summary = ModelReceiptRuntimeSummary( + workflow_result=( + workflow_result.value if workflow_result is not None else "error" + ), + exit_code=exit_code, + workflow=str(contract_path), + terminal_payload=workflow_data.get("terminal_payload"), + handler_result=handler_result_json, + error=runtime_error, + # Errors are never hidden: non-success inlines the FULL capture + # log; success keeps it behind the artifact ref. + capture_log="" if status.is_success_like else capture_text, + ) + receipt = ModelSkillResult[ModelReceiptRuntimeSummary]( + skill_name=node_name, + node_name=node_name, + status=status, + correlation_id=correlation_id, + run_id=run_id, + exit_code=exit_code, + duration_ms=duration_ms, + result=summary, + result_model=_fully_qualified_name(summary), + metrics=metrics, + artifact_refs=artifact_refs, + ) + + try: + click.echo(receipt.model_dump_json()) + except ValidationError as exc: # pragma: no cover - construction validates + click.echo(f"receipt mode: receipt serialization failed: {exc}", err=True) + return 1 + return exit_code diff --git a/src/omnibase_infra/cli/skill_mapping.yaml b/src/omnibase_infra/cli/skill_mapping.yaml new file mode 100644 index 0000000000..a1b9abeab8 --- /dev/null +++ b/src/omnibase_infra/cli/skill_mapping.yaml @@ -0,0 +1,255 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# Declarative `onex skill ` -> backing-node mapping (OMN-13097). +# +# Phase 4a of the skill-output-suppression slice +# (docs/plans/2026-06-12-skill-output-suppression-plan.md Phase 4 item 1). +# +# This is DATA, not code: `onex skill` loads it into ModelSkillMappingRegistry +# and uses it to build each backing node's input payload from the skill's CLI +# args, then dispatches through the proven receipt-mode path. Adding a skill is +# a YAML edit + a fixture, never a CLI code change (ticket deliverable 2). +# +# Per entry: +# skill_name – name as invoked: `onex skill ` +# node_name – backing node (onex.nodes entry-point group) +# result_model – FQN of the node's typed handler result (receipt schema +# identity; ticket deliverable 3) — verified against the +# live handler `handle()` return type on origin/dev. +# event_bus – backend override (default inmemory) +# timeout – dispatch timeout seconds (default 300) +# args – CLI arg -> payload field specs +# static_payload – payload fields injected regardless of args +# classifiers – keyword classification for unset fields (delegate) +# +# arg_type: string | integer | boolean | string_list +# boolean args are presence flags (--dry-run); string_list splits on commas. +skills: + - skill_name: aislop_sweep + node_name: node_aislop_sweep + result_model: omnimarket.nodes.node_aislop_sweep.handlers.handler_aislop_sweep.AislopSweepResult + args: + - {name: target-dirs, payload_field: target_dirs, arg_type: string_list} + - {name: checks, payload_field: checks, arg_type: string_list} + - {name: severity-threshold, payload_field: severity_threshold, arg_type: string} + - {name: dry-run, payload_field: dry_run, arg_type: boolean, default: false} + - skill_name: data_flow_sweep + node_name: node_data_flow_sweep + result_model: omnimarket.nodes.node_data_flow_sweep.handlers.handler_data_flow_sweep.DataFlowSweepResult + args: + - {name: flows, payload_field: flows, arg_type: string_list} + - {name: collect, payload_field: collect, arg_type: boolean, default: false} + - {name: dry-run, payload_field: dry_run, arg_type: boolean, default: false} + - skill_name: auto_merge + node_name: node_auto_merge_effect + result_model: omnimarket.nodes.node_auto_merge_effect.models.model_auto_merge_result.ModelAutoMergeResult + args: + - {name: pr-number, payload_field: pr_number, arg_type: integer, required: true} + - {name: repo, payload_field: repo, arg_type: string, required: true} + - {name: strategy, payload_field: strategy, arg_type: string} + - {name: delete-branch, payload_field: delete_branch, arg_type: boolean, default: false} + - {name: ticket-id, payload_field: ticket_id, arg_type: string} + - {name: gate-timeout-hours, payload_field: gate_timeout_hours, arg_type: integer} + - skill_name: build_loop + node_name: node_build_loop + result_model: omnimarket.nodes.node_build_loop.models.model_loop_state.ModelLoopState + args: + - {name: max-cycles, payload_field: max_cycles, arg_type: integer} + - {name: skip-closeout, payload_field: skip_closeout, arg_type: boolean, default: false} + - {name: dry-run, payload_field: dry_run, arg_type: boolean, default: false} + - skill_name: coderabbit_triage + node_name: node_coderabbit_triage + result_model: omnimarket.nodes.node_coderabbit_triage.handlers.handler_coderabbit_triage.ModelCoderabbitTriageResult + args: + - {name: repo, payload_field: repo, arg_type: string, required: true} + - {name: pr-number, payload_field: pr_number, arg_type: integer, required: true} + - {name: dry-run, payload_field: dry_run, arg_type: boolean, default: false} + - skill_name: compliance_sweep + node_name: node_compliance_sweep + result_model: omnimarket.nodes.node_compliance_sweep.handlers.handler_compliance_sweep.ComplianceSweepResult + args: + - {name: repos, payload_field: repos, arg_type: string_list} + - {name: checks, payload_field: checks, arg_type: string_list} + - {name: dry-run, payload_field: dry_run, arg_type: boolean, default: false} + - skill_name: coverage_sweep + node_name: node_coverage_sweep + result_model: omnimarket.nodes.node_coverage_sweep.handlers.handler_coverage_sweep.CoverageSweepResult + args: + - {name: repos, payload_field: repos, arg_type: string_list} + - {name: target-pct, payload_field: target_pct, arg_type: integer} + - {name: dry-run, payload_field: dry_run, arg_type: boolean, default: false} + - skill_name: create_ticket + node_name: node_create_ticket + result_model: omnimarket.nodes.node_create_ticket.handlers.handler_create_ticket.ModelCreateTicketResult + args: + - {name: title, payload_field: title, arg_type: string} + - {name: from-contract, payload_field: from_contract, arg_type: string} + - {name: from-plan, payload_field: from_plan, arg_type: string} + - {name: milestone, payload_field: milestone, arg_type: string} + - {name: repo, payload_field: repo, arg_type: string} + - {name: parent, payload_field: parent, arg_type: string} + - {name: blocked-by, payload_field: blocked_by, arg_type: string_list} + - {name: project, payload_field: project, arg_type: string} + - {name: team, payload_field: team, arg_type: string} + - {name: allow-arch-violation, payload_field: allow_arch_violation, arg_type: boolean, default: false} + - skill_name: database_sweep + node_name: node_database_sweep + result_model: omnimarket.nodes.node_database_sweep.handlers.handler_database_sweep.DatabaseSweepResult + args: + - {name: omni-home, payload_field: omni_home, arg_type: string} + - {name: table, payload_field: table, arg_type: string} + - {name: staleness-threshold-hours, payload_field: staleness_threshold_hours, arg_type: integer} + - {name: dry-run, payload_field: dry_run, arg_type: boolean, default: false} + - skill_name: design_to_plan + node_name: node_design_to_plan + result_model: omnimarket.nodes.node_design_to_plan.models.model_design_to_plan_phase3_launch.ModelDesignToPlanPhase3LaunchResult + args: + - {name: phase, payload_field: phase, arg_type: string} + - {name: topic, payload_field: topic, arg_type: string} + - {name: plan-path, payload_field: plan_path, arg_type: string} + - {name: no-launch, payload_field: no_launch, arg_type: boolean, default: false} + - {name: plan-only, payload_field: plan_only, arg_type: boolean, default: false} + - {name: dry-run, payload_field: dry_run, arg_type: boolean, default: false} + - skill_name: doc_freshness_sweep + node_name: node_doc_freshness_sweep + result_model: omnimarket.nodes.node_doc_freshness_sweep.handlers.handler_doc_freshness_sweep.DocFreshnessSweepResult + args: + - {name: omni-home, payload_field: omni_home, arg_type: string} + - {name: repos, payload_field: repos, arg_type: string_list} + - {name: claude-md-only, payload_field: claude_md_only, arg_type: boolean, default: false} + - {name: broken-only, payload_field: broken_only, arg_type: boolean, default: false} + - {name: dry-run, payload_field: dry_run, arg_type: boolean, default: false} + - skill_name: dod_verify + node_name: node_dod_verify + result_model: omnimarket.nodes.node_dod_verify.models.model_dod_verify_state.ModelDodVerifyState + args: + - {name: ticket-id, payload_field: ticket_id, arg_type: string, positional: true, required: true} + - {name: contract-path, payload_field: contract_path, arg_type: string} + - {name: dry-run, payload_field: dry_run, arg_type: boolean, default: false} + - skill_name: duplication_sweep + node_name: node_duplication_sweep + result_model: omnimarket.nodes.node_duplication_sweep.handlers.handler_duplication_sweep.DuplicationSweepResult + args: + - {name: omni-home, payload_field: omni_home, arg_type: string} + - {name: checks, payload_field: checks, arg_type: string_list} + - skill_name: hostile_reviewer + node_name: node_hostile_reviewer + result_model: omnimarket.nodes.node_hostile_reviewer.models.model_hostile_reviewer_completed_event.ModelHostileReviewerCompletedEvent + args: + - {name: pr-number, payload_field: pr_number, arg_type: integer} + - {name: repo, payload_field: repo, arg_type: string} + - {name: file-path, payload_field: file_path, arg_type: string} + - {name: models, payload_field: models, arg_type: string_list} + - {name: dry-run, payload_field: dry_run, arg_type: boolean, default: false} + - skill_name: linear_housekeeping + node_name: node_linear_triage + result_model: omnimarket.nodes.node_linear_triage.models.model_linear_triage_state.ModelLinearTriageResult + args: + - {name: threshold-days, payload_field: threshold_days, arg_type: integer} + - {name: flag-only, payload_field: flag_only, arg_type: boolean, default: false} + - {name: team, payload_field: team, arg_type: string} + - {name: dry-run, payload_field: dry_run, arg_type: boolean, default: false} + - skill_name: merge_sweep + node_name: node_pr_lifecycle_orchestrator + result_model: omnimarket.nodes.node_pr_lifecycle_orchestrator.handlers.handler_pr_lifecycle_orchestrator.ModelPrLifecycleResult + args: + - {name: repos, payload_field: repos, arg_type: string_list} + - {name: dry-run, payload_field: dry_run, arg_type: boolean, default: false} + - {name: inventory-only, payload_field: inventory_only, arg_type: boolean, default: false} + - {name: fix-only, payload_field: fix_only, arg_type: boolean, default: false} + - {name: merge-only, payload_field: merge_only, arg_type: boolean, default: false} + - {name: max-parallel-polish, payload_field: max_parallel_polish, arg_type: integer} + - {name: admin-fallback-threshold-minutes, payload_field: admin_fallback_threshold_minutes, arg_type: integer} + - {name: verify, payload_field: verify, arg_type: boolean, default: false} + - {name: verify-timeout-seconds, payload_field: verify_timeout_seconds, arg_type: integer} + - skill_name: plan_to_tickets + node_name: node_plan_to_tickets + result_model: omnimarket.nodes.node_plan_to_tickets.handlers.handler_plan_to_tickets.ModelPlanToTicketsResult + args: + - {name: plan-path, payload_field: plan_path, arg_type: string, positional: true, required: true} + - {name: project, payload_field: project, arg_type: string} + - {name: epic-title, payload_field: epic_title, arg_type: string} + - {name: no-create-epic, payload_field: no_create_epic, arg_type: boolean, default: false} + - {name: skip-existing, payload_field: skip_existing, arg_type: boolean, default: false} + - {name: team, payload_field: team, arg_type: string} + - {name: repo, payload_field: repo, arg_type: string} + - {name: allow-arch-violation, payload_field: allow_arch_violation, arg_type: boolean, default: false} + - {name: dry-run, payload_field: dry_run, arg_type: boolean, default: false} + - skill_name: platform_readiness + node_name: node_platform_readiness + result_model: omnimarket.nodes.node_platform_readiness.handlers.handler_platform_readiness.ModelPlatformReadinessResult + args: [] + - skill_name: pr_polish + node_name: node_pr_polish + result_model: omnimarket.nodes.node_pr_polish.models.model_pr_polish_completed_event.ModelPrPolishCompletedEvent + args: + - {name: repo, payload_field: repo, arg_type: string, required: true} + - {name: pr-number, payload_field: pr_number, arg_type: integer, required: true} + - {name: ticket-id, payload_field: ticket_id, arg_type: string} + - {name: required-clean-runs, payload_field: required_clean_runs, arg_type: integer} + - {name: max-iterations, payload_field: max_iterations, arg_type: integer} + - {name: skip-conflicts, payload_field: skip_conflicts, arg_type: boolean, default: false} + - {name: skip-pr-review, payload_field: skip_pr_review, arg_type: boolean, default: false} + - {name: skip-local-review, payload_field: skip_local_review, arg_type: boolean, default: false} + - {name: no-ci, payload_field: no_ci, arg_type: boolean, default: false} + - {name: no-push, payload_field: no_push, arg_type: boolean, default: false} + - {name: no-automerge, payload_field: no_automerge, arg_type: boolean, default: false} + - {name: dry-run, payload_field: dry_run, arg_type: boolean, default: false} + - skill_name: pr_review + node_name: node_pr_review_bot + result_model: omnimarket.nodes.node_pr_review_bot.models.models.ReviewVerdict + args: + - {name: pr-number, payload_field: pr_number, arg_type: integer, required: true} + - {name: repo, payload_field: repo, arg_type: string, required: true} + - {name: reviewer-models, payload_field: reviewer_models, arg_type: string_list} + - {name: judge-model, payload_field: judge_model, arg_type: string} + - {name: severity-threshold, payload_field: severity_threshold, arg_type: string} + - {name: max-findings-per-pr, payload_field: max_findings_per_pr, arg_type: integer} + - {name: dry-run, payload_field: dry_run, arg_type: boolean, default: false} + - skill_name: pr_review_bot + node_name: node_pr_review_bot + result_model: omnimarket.nodes.node_pr_review_bot.models.models.ReviewVerdict + args: + - {name: pr-number, payload_field: pr_number, arg_type: integer, required: true} + - {name: repo, payload_field: repo, arg_type: string, required: true} + - {name: reviewer-models, payload_field: reviewer_models, arg_type: string_list} + - {name: judge-model, payload_field: judge_model, arg_type: string} + - {name: severity-threshold, payload_field: severity_threshold, arg_type: string} + - {name: max-findings-per-pr, payload_field: max_findings_per_pr, arg_type: integer} + - {name: dry-run, payload_field: dry_run, arg_type: boolean, default: false} + - skill_name: session + node_name: node_session_orchestrator + result_model: omnimarket.nodes.node_session_orchestrator.handlers.handler_session_orchestrator.ModelSessionOrchestratorResult + args: + - {name: mode, payload_field: mode, arg_type: string} + - {name: phase, payload_field: phase, arg_type: string} + - {name: skip-health, payload_field: skip_health, arg_type: boolean, default: false} + - {name: dry-run, payload_field: dry_run, arg_type: boolean, default: false} + - skill_name: shim_audit + node_name: node_shim_scanner + result_model: omnimarket.nodes.node_shim_scanner.models.model_shim_scan_result.ModelShimScanResult + args: + - {name: paths, payload_field: paths, arg_type: string_list} + - {name: reference-date, payload_field: reference_date, arg_type: string} + - {name: warn-days-before-expiry, payload_field: warn_days_before_expiry, arg_type: integer} + - skill_name: delegate + node_name: node_delegate_skill_orchestrator + result_model: omnimarket.models.delegation.wire.model_delegate_skill_response.ModelDelegateSkillResponse + static_payload: + source: claude-code + args: + - {name: prompt, payload_field: prompt, arg_type: string, positional: true, required: true} + - {name: task-type, payload_field: task_type, arg_type: string} + - {name: max-tokens, payload_field: max_tokens, arg_type: integer, default: 2048} + classifiers: + - target_field: task_type + source_field: prompt + fallback: research + rules: + - [["test", "pytest", "unit test", "assert"], test] + - [["document", "docstring", "readme", "explain how"], document] + - [["write", "create", "implement", "build", "generate"], code_generation] + - [["refactor", "cleanup", "simplify"], refactor] + - [["review", "audit", "check"], review] + - [["reason", "think through", "decide", "compare"], reasoning] diff --git a/src/omnibase_infra/docker/catalog/cli.py b/src/omnibase_infra/docker/catalog/cli.py index 17b49337fd..fa2d39221a 100644 --- a/src/omnibase_infra/docker/catalog/cli.py +++ b/src/omnibase_infra/docker/catalog/cli.py @@ -17,9 +17,11 @@ from __future__ import annotations +import datetime import os import subprocess import sys +import tomllib from pathlib import Path import yaml @@ -27,6 +29,9 @@ from omnibase_infra.docker.catalog.generator import generate_compose from omnibase_infra.docker.catalog.resolver import CatalogResolver from omnibase_infra.docker.catalog.validator import validate_env +from omnibase_infra.docker.catalog.validator_healthcheck_start_period import ( + validate_migration_gate_start_period, +) _OMNIBASE_ENV = Path.home() / ".omnibase" / ".env" @@ -183,6 +188,66 @@ def _current_git_sha() -> str: return "unknown" +def _runtime_version() -> str: + """Return the runtime package version stamped as the OCI image version. + + Read from the repo ``pyproject.toml`` so ``org.opencontainers.image.version`` + reflects the real package version rather than the Dockerfile placeholder + default (``0.1.0``). Fails fast if the version cannot be resolved so a build + never silently stamps a placeholder identity (OMN-12965). + """ + pyproject = _REPO_ROOT / "pyproject.toml" + try: + data = tomllib.loads(pyproject.read_text(encoding="utf-8")) + version = str(data["project"]["version"]).strip() + except (FileNotFoundError, KeyError, TypeError, tomllib.TOMLDecodeError) as exc: + raise RuntimeError( + f"Could not resolve RUNTIME_VERSION from {pyproject}; " + "refusing to build an image with placeholder identity (OMN-12965)." + ) from exc + if not version: + raise RuntimeError( + f"Empty RUNTIME_VERSION in {pyproject}; " + "refusing to build an image with placeholder identity (OMN-12965)." + ) + return version + + +def _image_identity_build_args() -> list[str]: + """Return the OCI image-identity ``--build-arg`` pairs for a runtime build. + + Stamps the full quad consumed by the runtime-stage OCI labels and provenance + manifest: ``GIT_SHA`` (busts the COPY src/ layer + builder-stage revision + label), ``VCS_REF`` (runtime-stage ``org.opencontainers.image.revision``), + ``RUNTIME_VERSION`` (``org.opencontainers.image.version``), and ``BUILD_DATE`` + (``org.opencontainers.image.created``). + + Fails fast when the git SHA is unresolved: a blank/``unknown`` revision + produces a blank-identity image that degrades every proof packet (the runtime + SHA + image digest are required citations in accepted evidence). This is the + enforcement point for OMN-12965 — never stamp a placeholder identity. + """ + git_sha = _current_git_sha() + if not git_sha or git_sha == "unknown": + raise RuntimeError( + "Cannot resolve git revision for the runtime image build. A blank or " + "'unknown' org.opencontainers.image.revision degrades every proof " + "packet (OMN-12965). Run the build from a clean git checkout of " + f"{_REPO_ROOT}." + ) + build_date = datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + return [ + "--build-arg", + f"GIT_SHA={git_sha}", + "--build-arg", + f"VCS_REF={git_sha}", + "--build-arg", + f"RUNTIME_VERSION={_runtime_version()}", + "--build-arg", + f"BUILD_DATE={build_date}", + ] + + def cmd_seed(_args: list[str]) -> int: """Seed Infisical with config keys from contracts.""" return _run_seed() @@ -236,7 +301,6 @@ def cmd_up(args: list[str]) -> int: # Build if requested (OMN-7214) if force_build: print("Rebuilding images (--build)...") - git_sha = _current_git_sha() build_proc = subprocess.run( [ "docker", @@ -244,8 +308,7 @@ def cmd_up(args: list[str]) -> int: "-f", _DEFAULT_OUTPUT, "build", - "--build-arg", - f"GIT_SHA={git_sha}", + *_image_identity_build_args(), ], cwd=str(_REPO_ROOT), check=False, @@ -339,6 +402,12 @@ def cmd_validate_runtime(args: list[str]) -> int: " (required_env wins — consider removing from operational_defaults)" ) + # Migration-completion gates must keep start_period above the floor so a + # still-applying gate is reported `starting`, not UNHEALTHY (OMN-12973). + start_period_result = validate_migration_gate_start_period(resolved.manifests) + for violation in start_period_result.violations: + errors.append(violation.message()) + if warnings: for w in warnings: print(f" WARN: {w}", file=sys.stderr) diff --git a/src/omnibase_infra/docker/catalog/validator_healthcheck_start_period.py b/src/omnibase_infra/docker/catalog/validator_healthcheck_start_period.py new file mode 100644 index 0000000000..ffb5a27326 --- /dev/null +++ b/src/omnibase_infra/docker/catalog/validator_healthcheck_start_period.py @@ -0,0 +1,155 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Start-period floor validator for migration-completion gates (OMN-12973). + +A *migration-completion gate* is a catalog service that: + +1. declares a healthcheck whose pass condition is "migrations are complete", and +2. waits on one or more sibling services via + ``service_completed_successfully`` (the migration runners) before its own + health can pass. + +The canonical example is ``migration-gate``: a long-running sentinel +(``while true; sleep 3600``) whose healthcheck polls +``db_metadata.migrations_complete``. It is *not* a one-shot — it stays alive so +downstream runtime services can gate on it via ``service_healthy``. + +Failure mode this validator prevents (observed in prod 2026-06-11, OMN-12973): +a too-short ``start_period_s`` lets the container leave Docker's ``starting`` +grace window before the migrations it polls for have actually completed. Past +the grace window, the still-failing healthcheck is reported ``UNHEALTHY`` until +migrations finish (~2 min on a cold prod volume), even though nothing is wrong — +the gate is doing exactly its job. ``start_period_s`` must therefore cover the +real migration window so a still-applying gate stays in ``health: starting``. + +This is a forward-only ratchet: it asserts a floor, never an exact value. +Raising the floor is a deliberate config decision; lowering a gate's +``start_period_s`` below the floor is a regression and fails CI + pre-commit. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from omnibase_infra.docker.catalog.enum_depends_on_condition import ( + EnumDependsOnCondition, +) +from omnibase_infra.docker.catalog.manifest_schema import CatalogManifest, HealthCheck + +# Floor (seconds) for a migration-completion gate's healthcheck start_period. +# Derived from the observed prod migration window (OMN-12973): the gate started +# at 09:35:22 and intelligence-migration finished at 09:37:18 — ~116s. 120s is +# the enforced floor; the migration-gate manifest ships 180s for margin. +MIGRATION_GATE_START_PERIOD_FLOOR_S = 120 + +# Substring that identifies a healthcheck whose pass-condition is "migrations +# are complete". The canonical migration sentinel polls +# ``check_migrations_complete.sh``; its presence in the healthcheck test is the +# discriminating signal that the service's health *is* migration completion (as +# opposed to a normal app service that merely depends on a migration runner +# finishing — e.g. intelligence-api, whose healthcheck is an HTTP liveness probe). +_MIGRATION_COMPLETION_PROBE_MARKER = "check_migrations_complete" + + +def _healthcheck_polls_migration_completion(healthcheck: HealthCheck) -> bool: + """True if the healthcheck command's pass-condition is migration completion. + + ``HealthCheck.test`` is either a shell string (CMD-SHELL form) or an argv + list (CMD form); both are joined into one string before scanning for the + migration-completion probe marker. + """ + test = healthcheck.test + command = test if isinstance(test, str) else " ".join(test) + return _MIGRATION_COMPLETION_PROBE_MARKER in command + + +def is_migration_completion_gate(manifest: CatalogManifest) -> bool: + """True if ``manifest`` is a sentinel whose own health *is* migration completion. + + The discriminating signature has two parts, both required: + + 1. a healthcheck whose pass-condition is "migrations are complete" (its + command invokes the migration-completion probe), and + 2. at least one dependency with + ``condition == service_completed_successfully`` (it waits on the + migration runner one-shots). + + Part (2) alone is NOT sufficient: ordinary application services (e.g. + ``intelligence-api``) also depend on a migration runner completing, but + their healthcheck is a liveness probe with a legitimately short + ``start_period``. Only the sentinel — whose healthcheck literally polls + migration completion — must hold ``starting`` for the full migration window, + so only it is subject to the start_period floor. + """ + if manifest.healthcheck is None: + return False + if not _healthcheck_polls_migration_completion(manifest.healthcheck): + return False + return any( + dep.condition is EnumDependsOnCondition.SERVICE_COMPLETED_SUCCESSFULLY + for dep in manifest.depends_on + ) + + +@dataclass(frozen=True) # internal-dataclass-ok: docker-catalog-internal +class StartPeriodViolation: + """A single migration-gate service whose start_period is below the floor.""" + + service: str + start_period_s: int + floor_s: int + + def message(self) -> str: + return ( + f"migration-completion gate '{self.service}' has " + f"healthcheck.start_period_s={self.start_period_s}, below the " + f"required floor of {self.floor_s}s. A start_period shorter than the " + f"real migration window causes the gate to be reported UNHEALTHY " + f"while migrations are still applying (OMN-12973). Raise " + f"start_period_s to >= {self.floor_s}." + ) + + +@dataclass(frozen=True) # internal-dataclass-ok: docker-catalog-internal +class StartPeriodValidationResult: + """Result of validating migration-gate start_period floors.""" + + ok: bool + violations: list[StartPeriodViolation] = field(default_factory=list) + + def report(self) -> str: + if self.ok: + return "all migration-completion gates satisfy the start_period floor" + return "\n".join(v.message() for v in self.violations) + + +def validate_migration_gate_start_period( + manifests: dict[str, CatalogManifest], + floor_s: int = MIGRATION_GATE_START_PERIOD_FLOOR_S, +) -> StartPeriodValidationResult: + """Assert every migration-completion gate meets the start_period floor. + + Args: + manifests: catalog manifests keyed by service name. + floor_s: minimum allowed ``start_period_s`` for a migration gate. + + Returns: + A result with ``ok=False`` and one violation per offending gate. + """ + violations: list[StartPeriodViolation] = [] + for name in sorted(manifests): + manifest = manifests[name] + if not is_migration_completion_gate(manifest): + continue + # healthcheck is non-None here (guaranteed by is_migration_completion_gate) + assert manifest.healthcheck is not None + start_period_s = manifest.healthcheck.start_period_s + if start_period_s < floor_s: + violations.append( + StartPeriodViolation( + service=name, + start_period_s=start_period_s, + floor_s=floor_s, + ) + ) + return StartPeriodValidationResult(ok=len(violations) == 0, violations=violations) diff --git a/src/omnibase_infra/enums/generated/enum_omnibase_infra_topic.py b/src/omnibase_infra/enums/generated/enum_omnibase_infra_topic.py index e9581dcef5..1394c95a23 100644 --- a/src/omnibase_infra/enums/generated/enum_omnibase_infra_topic.py +++ b/src/omnibase_infra/enums/generated/enum_omnibase_infra_topic.py @@ -55,6 +55,7 @@ class EnumOmnibaseInfraTopic(str, Enum): EVT_DELEGATION_FAILED_V1 = "onex.evt.omnibase-infra.delegation-failed.v1" # onex.evt.omnibase-infra.delegation-failed.v1 EVT_EVAL_COMPLETED_V1 = "onex.evt.omnibase-infra.eval-completed.v1" # onex.evt.omnibase-infra.eval-completed.v1 EVT_EVENT_FORWARDED_V1 = "onex.evt.omnibase-infra.event-forwarded.v1" # onex.evt.omnibase-infra.event-forwarded.v1 + EVT_GATEWAY_HEARTBEAT_V1 = "onex.evt.omnibase-infra.gateway-heartbeat.v1" # onex.evt.omnibase-infra.gateway-heartbeat.v1 EVT_GMAIL_INTENT_RECEIVED_V1 = "onex.evt.omnibase-infra.gmail-intent-received.v1" # onex.evt.omnibase-infra.gmail-intent-received.v1 EVT_INFERENCE_RESPONSE_V1 = "onex.evt.omnibase-infra.inference-response.v1" # onex.evt.omnibase-infra.inference-response.v1 EVT_LLM_CALL_COMPLETED_V1 = "onex.evt.omnibase-infra.llm-call-completed.v1" # onex.evt.omnibase-infra.llm-call-completed.v1 diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/__init__.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/__init__.py new file mode 100644 index 0000000000..90d781e22c --- /dev/null +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/__init__.py @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Tenant gateway bus forwarder effect node.""" + +from .node import NodeBusForwarderEffect + +__all__ = ["NodeBusForwarderEffect"] diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/contract.yaml b/src/omnibase_infra/nodes/node_bus_forwarder_effect/contract.yaml new file mode 100644 index 0000000000..e8291f00e2 --- /dev/null +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/contract.yaml @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +name: "node_bus_forwarder_effect" +contract_name: "node_bus_forwarder_effect" +node_name: "node_bus_forwarder_effect" +node_type: "EFFECT_GENERIC" +contract_version: + major: 0 + minor: 1 + patch: 0 +node_version: + major: 0 + minor: 1 + patch: 0 +description: > + Trust-boundary effect node that mirrors contract-declared tenant gateway + topics between a local runtime bus and the hosted cloud Kafka edge. The node + owns all publishes; handlers only validate and return transformed envelopes. + +runtime_profiles: + - effects + +input_model: + name: "ModelGatewayEnvelope" + module: "omnibase_infra.nodes.node_bus_forwarder_effect.models.model_gateway_envelope" + description: "Canonical gateway envelope with tenant binding and topic transform fields." +output_model: + name: "ModelGatewayEnvelope" + module: "omnibase_infra.nodes.node_bus_forwarder_effect.models.model_gateway_envelope" + description: "Validated gateway envelope after prefix add/strip transformation." + +config: + name: "ModelGatewayForwarderConfig" + module: "omnibase_infra.nodes.node_bus_forwarder_effect.models.model_gateway_forwarder_config" + description: "Frozen gateway forwarder configuration resolved from contract and secret refs." + gateway_forwarder: + cloud_leg: + transport: "kafka" + broker_provider_id: "22222222-2222-2222-2222-222222222222" + cloud_broker_ref: "gateway.cloud.kafka.broker" + cloud_auth_ref: "gateway.cloud.kafka.oauth" + acl_provisioner_ref: "gateway.cloud.kafka.authorization" + client_id_ref: "gateway.cloud.kafka.oauth.client_id" + client_secret_api_key_ref: "gateway.cloud.kafka.oauth.client_secret" + local_leg: + supported_flavors: + - "containerized" + - "lightweight" + mirror_topics: + inbound: + - "onex.cmd.omnibase-infra.delegation-inference-request.v1" + - "onex.evt.omnibase-infra.gateway-heartbeat.v1" + outbound: + - "onex.evt.omnibase-infra.inference-response.v1" + - "onex.evt.omnibase-infra.gateway-heartbeat.v1" + liveness: + heartbeat_interval_seconds: 15 + max_silence_window_seconds: 60 + lag_threshold_messages: 500 + lag_threshold_seconds: 120 + drain_deadline_seconds: 30 + dedupe: + retention_hours: 24 + +event_bus: + subscribe_topics: + - "onex.cmd.omnibase-infra.delegation-inference-request.v1" + - "onex.evt.omnibase-infra.inference-response.v1" + - "onex.evt.omnibase-infra.gateway-heartbeat.v1" + publish_topics: + - "onex.cmd.omnibase-infra.delegation-inference-request.v1" + - "onex.evt.omnibase-infra.inference-response.v1" + - "onex.evt.omnibase-infra.gateway-heartbeat.v1" + +handler_routing: + routing_strategy: "operation_match" + handlers: + - operation: "gateway.forward_outbound" + handler: + name: "HandlerForwardOutbound" + module: "omnibase_infra.nodes.node_bus_forwarder_effect.handlers.handler_forward_outbound" + handler_type: "COMPUTE" + description: "Validate local bare topic and add the tenant cloud wire prefix." + - operation: "gateway.consume_inbound" + handler: + name: "HandlerConsumeInbound" + module: "omnibase_infra.nodes.node_bus_forwarder_effect.handlers.handler_consume_inbound" + handler_type: "COMPUTE" + description: "Validate tenant-prefixed cloud topic and strip to a bare local topic." + +capabilities: + - name: "tenant_prefix_transform" + description: "Adds and strips tenant wire prefixes with envelope validation." + - name: "contract_declared_mirror_topics_only" + description: "Rejects any canonical topic not declared in mirror topic sets." + - name: "node_owned_publish" + description: "Handlers never publish; the effect node owns both bus legs." + +metadata: + description: "Hybrid gateway local runtime bus forwarder" + author: "OmniNode Team" + created: "2026-06-10" + tags: + - gateway + - event-bus + - tenant-isolation + - trust-boundary diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/handlers/__init__.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/handlers/__init__.py new file mode 100644 index 0000000000..089542d045 --- /dev/null +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/handlers/__init__.py @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Pure transform handlers for the gateway forwarder.""" + +from .handler_consume_inbound import HandlerConsumeInbound +from .handler_forward_outbound import HandlerForwardOutbound + +__all__ = ["HandlerConsumeInbound", "HandlerForwardOutbound"] diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/handlers/handler_consume_inbound.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/handlers/handler_consume_inbound.py new file mode 100644 index 0000000000..22ec48ae6c --- /dev/null +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/handlers/handler_consume_inbound.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Inbound gateway transform handler.""" + +from __future__ import annotations + +from omnibase_infra.enums import EnumHandlerType, EnumHandlerTypeCategory +from omnibase_infra.errors import ProtocolConfigurationError +from omnibase_infra.nodes.node_bus_forwarder_effect.models import ( + ModelGatewayEnvelope, + ModelGatewayForwarderConfig, +) +from omnibase_infra.nodes.node_bus_forwarder_effect.services.service_gateway_topic_transform import ( + prefix_topic, + strip_topic_prefix, +) + + +class HandlerConsumeInbound: + """Validate cloud wire topics and strip to a bare local contract topic.""" + + def __init__(self, config: ModelGatewayForwarderConfig | None = None) -> None: + self._config = config + + @property + def handler_type(self) -> EnumHandlerType: + return EnumHandlerType.COMPUTE_HANDLER + + @property + def handler_category(self) -> EnumHandlerTypeCategory: + return EnumHandlerTypeCategory.COMPUTE + + def consume_inbound(self, envelope: ModelGatewayEnvelope) -> ModelGatewayEnvelope: + """Return an inbound envelope with a validated bare canonical topic.""" + config = self._require_config() + identity = config.tenant_identity + if envelope.tenant_id != identity.tenant_id: + raise ValueError("envelope tenant_id does not match attached tenant") + if envelope.tenant_slug != identity.tenant_slug: + raise ValueError("envelope tenant_slug does not match attached tenant") + + canonical_topic = strip_topic_prefix(identity.tenant_slug, envelope.wire_topic) + if canonical_topic != envelope.canonical_topic: + raise ValueError("wire_topic and canonical_topic do not match") + if canonical_topic not in config.mirror_topics.inbound: + raise ValueError("canonical_topic is not declared for inbound mirroring") + expected_wire_topic = prefix_topic(identity.tenant_slug, canonical_topic) + if envelope.wire_topic != expected_wire_topic: + raise ValueError("wire_topic does not match canonical tenant prefix") + + return envelope.model_copy( + update={ + "source_topic": envelope.wire_topic, + "canonical_topic": canonical_topic, + } + ) + + def _require_config(self) -> ModelGatewayForwarderConfig: + if self._config is None: + raise ProtocolConfigurationError( + "HandlerConsumeInbound requires ModelGatewayForwarderConfig" + ) + return self._config diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/handlers/handler_forward_outbound.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/handlers/handler_forward_outbound.py new file mode 100644 index 0000000000..2ee628adb3 --- /dev/null +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/handlers/handler_forward_outbound.py @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Outbound gateway transform handler.""" + +from __future__ import annotations + +from omnibase_infra.enums import EnumHandlerType, EnumHandlerTypeCategory +from omnibase_infra.errors import ProtocolConfigurationError +from omnibase_infra.nodes.node_bus_forwarder_effect.models import ( + ModelGatewayEnvelope, + ModelGatewayForwarderConfig, +) +from omnibase_infra.nodes.node_bus_forwarder_effect.services.service_gateway_topic_transform import ( + prefix_topic, +) + + +class HandlerForwardOutbound: + """Validate local bare topics and add the tenant cloud wire prefix.""" + + def __init__(self, config: ModelGatewayForwarderConfig | None = None) -> None: + self._config = config + + @property + def handler_type(self) -> EnumHandlerType: + return EnumHandlerType.COMPUTE_HANDLER + + @property + def handler_category(self) -> EnumHandlerTypeCategory: + return EnumHandlerTypeCategory.COMPUTE + + def forward_outbound(self, envelope: ModelGatewayEnvelope) -> ModelGatewayEnvelope: + """Return an outbound envelope with a validated tenant wire topic.""" + config = self._require_config() + identity = config.tenant_identity + if envelope.tenant_id != identity.tenant_id: + raise ValueError("envelope tenant_id does not match attached tenant") + if envelope.tenant_slug != identity.tenant_slug: + raise ValueError("envelope tenant_slug does not match attached tenant") + if envelope.canonical_topic not in config.mirror_topics.outbound: + raise ValueError("canonical_topic is not declared for outbound mirroring") + + expected_wire_topic = prefix_topic( + identity.tenant_slug, + envelope.canonical_topic, + ) + return envelope.model_copy( + update={ + "source_topic": envelope.canonical_topic, + "wire_topic": expected_wire_topic, + } + ) + + def _require_config(self) -> ModelGatewayForwarderConfig: + if self._config is None: + raise ProtocolConfigurationError( + "HandlerForwardOutbound requires ModelGatewayForwarderConfig" + ) + return self._config diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/__init__.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/__init__.py new file mode 100644 index 0000000000..6f0fc6e1ab --- /dev/null +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/__init__.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Models for the tenant gateway bus forwarder.""" + +from .model_gateway_cloud_bus_config import ModelGatewayCloudBusConfig +from .model_gateway_envelope import ModelGatewayEnvelope +from .model_gateway_forwarder_config import ModelGatewayForwarderConfig +from .model_gateway_mirror_topics import ModelGatewayMirrorTopics +from .model_gateway_tenant_identity import ModelGatewayTenantIdentity + +__all__ = [ + "ModelGatewayCloudBusConfig", + "ModelGatewayEnvelope", + "ModelGatewayForwarderConfig", + "ModelGatewayMirrorTopics", + "ModelGatewayTenantIdentity", +] diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_cloud_bus_config.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_cloud_bus_config.py new file mode 100644 index 0000000000..dca70b3fd5 --- /dev/null +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_cloud_bus_config.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Cloud bus config model for the gateway forwarder.""" + +from __future__ import annotations + +from typing import Literal +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class ModelGatewayCloudBusConfig(BaseModel): + """Provider-neutral cloud Kafka leg config resolved from contract refs.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + broker_provider_id: UUID + cloud_broker_ref: str = Field(..., min_length=1) + cloud_auth_ref: str = Field(..., min_length=1) + acl_provisioner_ref: str = Field(..., min_length=1) + client_id_ref: str = Field(..., min_length=1) + client_secret_api_key_ref: str = Field(..., min_length=1) + security_protocol: Literal["SASL_SSL"] = "SASL_SSL" + sasl_mechanism: Literal["OAUTHBEARER"] = "OAUTHBEARER" + + @field_validator( + "cloud_broker_ref", + "cloud_auth_ref", + "acl_provisioner_ref", + "client_id_ref", + "client_secret_api_key_ref", + ) + @classmethod + def _validate_contract_ref(cls, value: str) -> str: + ref = value.strip() + if not ref: + raise ValueError("gateway cloud bus refs must not be empty") + if ref.startswith("KAFKA_"): + raise ValueError( + "gateway cloud bus config must use contract refs, not KAFKA_* env" + ) + return ref diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_envelope.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_envelope.py new file mode 100644 index 0000000000..78558a67f8 --- /dev/null +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_envelope.py @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Canonical gateway envelope model.""" + +from __future__ import annotations + +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class ModelGatewayEnvelope(BaseModel): + """Tenant-bound envelope used at the gateway transform boundary.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + tenant_id: UUID = Field(..., description="Immutable tenant identity.") + tenant_slug: str = Field(..., description="Recorded human-facing tenant slug.") + envelope_id: UUID = Field(..., description="Unique envelope id used for dedupe.") + correlation_id: UUID = Field(..., description="Per-tenant request correlation id.") + causation_id: str | None = Field(default=None, description="Parent envelope id.") + event_type: str = Field(..., description="Raw routing key preserved across hops.") + source_topic: str = Field(..., description="Topic this envelope was consumed from.") + wire_topic: str = Field(..., description="Tenant-prefixed cloud topic.") + canonical_topic: str = Field(..., description="Bare contract-declared topic.") + payload: dict[str, object] = Field(default_factory=dict) + + @field_validator( + "tenant_slug", + "event_type", + "source_topic", + "canonical_topic", + ) + @classmethod + def _non_empty(cls, value: str) -> str: + if not value or not value.strip(): + raise ValueError("gateway envelope field must not be empty") + return value.strip() diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_forwarder_config.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_forwarder_config.py new file mode 100644 index 0000000000..4bd0ad8707 --- /dev/null +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_forwarder_config.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Typed config for the tenant gateway bus forwarder.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from omnibase_infra.nodes.node_bus_forwarder_effect.models.model_gateway_cloud_bus_config import ( + ModelGatewayCloudBusConfig, +) +from omnibase_infra.nodes.node_bus_forwarder_effect.models.model_gateway_mirror_topics import ( + ModelGatewayMirrorTopics, +) +from omnibase_infra.nodes.node_bus_forwarder_effect.models.model_gateway_tenant_identity import ( + ModelGatewayTenantIdentity, +) + + +class ModelGatewayForwarderConfig(BaseModel): + """Complete forwarder config for one attached tenant edge.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + tenant_identity: ModelGatewayTenantIdentity + cloud_bus: ModelGatewayCloudBusConfig + local_transport_flavor: Literal["containerized", "lightweight"] + mirror_topics: ModelGatewayMirrorTopics + heartbeat_interval_seconds: int = Field(default=15, ge=1) + max_silence_window_seconds: int = Field(default=60, ge=1) + lag_threshold_messages: int = Field(default=500, ge=1) + lag_threshold_seconds: int = Field(default=120, ge=1) + drain_deadline_seconds: int = Field(default=30, ge=1) + dedupe_retention_hours: int = Field(default=24, ge=1) + + @model_validator(mode="after") + def _validate_liveness_windows(self) -> ModelGatewayForwarderConfig: + if self.max_silence_window_seconds <= self.heartbeat_interval_seconds: + raise ValueError( + "max_silence_window_seconds must exceed heartbeat_interval_seconds" + ) + return self diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_mirror_topics.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_mirror_topics.py new file mode 100644 index 0000000000..b84963686a --- /dev/null +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_mirror_topics.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Mirror topic config model for the gateway forwarder.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from omnibase_infra.nodes.node_bus_forwarder_effect.services.service_gateway_topic_transform import ( + validate_canonical_topic, +) + + +class ModelGatewayMirrorTopics(BaseModel): + """Bare contract-declared topics mirrored across the gateway.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + inbound: tuple[str, ...] = Field(..., min_length=1) + outbound: tuple[str, ...] = Field(..., min_length=1) + + @field_validator("inbound", "outbound") + @classmethod + def _validate_topics(cls, topics: tuple[str, ...]) -> tuple[str, ...]: + for topic in topics: + validate_canonical_topic(topic) + return topics diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_tenant_identity.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_tenant_identity.py new file mode 100644 index 0000000000..a7d1c90de9 --- /dev/null +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/models/model_gateway_tenant_identity.py @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Tenant identity model for the gateway forwarder.""" + +from __future__ import annotations + +import re +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, field_validator + +from omnibase_infra.nodes.node_bus_forwarder_effect.services.service_gateway_topic_transform import ( + RESERVED_TENANT_SLUGS, +) + +_TENANT_SLUG_RE = re.compile(r"^[a-z][a-z0-9-]{1,61}[a-z0-9]$") + + +class ModelGatewayTenantIdentity(BaseModel): + """Immutable tenant identity used to bind the gateway session.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + tenant_id: UUID + tenant_slug: str + principal_id: UUID + + @field_validator("tenant_slug") + @classmethod + def _validate_tenant_slug(cls, value: str) -> str: + slug = value.strip() + if slug in RESERVED_TENANT_SLUGS: + raise ValueError(f"tenant_slug is reserved: {slug}") + if not _TENANT_SLUG_RE.match(slug) or "--" in slug: + raise ValueError("tenant_slug must be DNS-compatible lowercase slug") + return slug diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/node.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/node.py new file mode 100644 index 0000000000..bf3441c1f4 --- /dev/null +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/node.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Bus forwarder effect node. + +The long-running process owns publishing on both bus legs. Handlers under this +node only validate and return transformed envelopes. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from omnibase_core.nodes.node_effect import NodeEffect + +if TYPE_CHECKING: + from omnibase_core.models.container import ModelONEXContainer + + +class NodeBusForwarderEffect(NodeEffect): + """Contract-driven effect node for tenant gateway bus forwarding.""" + + def __init__(self, container: ModelONEXContainer) -> None: + """Initialize with container dependency injection.""" + super().__init__(container) + + +__all__ = ["NodeBusForwarderEffect"] diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/__init__.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/__init__.py new file mode 100644 index 0000000000..579ba5414a --- /dev/null +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Services for gateway bus forwarding and topic transforms.""" diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/service_gateway_forwarder.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/service_gateway_forwarder.py new file mode 100644 index 0000000000..98b2b729ab --- /dev/null +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/service_gateway_forwarder.py @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Executable bus-to-bus gateway forwarder service.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from typing import Protocol + +from omnibase_infra.nodes.node_bus_forwarder_effect.handlers import ( + HandlerConsumeInbound, + HandlerForwardOutbound, +) +from omnibase_infra.nodes.node_bus_forwarder_effect.models import ( + ModelGatewayEnvelope, + ModelGatewayForwarderConfig, +) +from omnibase_infra.nodes.node_bus_forwarder_effect.services.service_gateway_topic_transform import ( + prefix_topic, +) + + +class ProtocolGatewayBus(Protocol): + """Structural subset shared by EventBusKafka, EventBusInmemory, and tests.""" + + async def publish( + self, + topic: str, + key: bytes | None, + value: bytes, + headers: object | None = None, + ) -> None: + """Publish bytes to a topic.""" + + async def subscribe( + self, + topic: str, + node_identity: object | None = None, + on_message: Callable[[object], Awaitable[None]] | None = None, + *, + group_id: str | None = None, + **kwargs: object, + ) -> Callable[[], Awaitable[None]]: + """Subscribe to a topic and return an async unsubscribe callback.""" + + +class ServiceGatewayForwarder: + """Subscribe to mirrored topics on both legs and republish transformed envelopes.""" + + def __init__( + self, + *, + config: ModelGatewayForwarderConfig, + local_bus: ProtocolGatewayBus, + cloud_bus: ProtocolGatewayBus, + ) -> None: + self._config = config + self._local_bus = local_bus + self._cloud_bus = cloud_bus + self._outbound_handler = HandlerForwardOutbound(config) + self._inbound_handler = HandlerConsumeInbound(config) + self._unsubscribe_callbacks: list[Callable[[], Awaitable[None]]] = [] + self._started = False + + async def start(self) -> None: + """Start subscriptions on both bus legs.""" + if self._started: + return + + for topic in self._config.mirror_topics.outbound: + unsubscribe = await self._local_bus.subscribe( + topic=topic, + group_id=self._group_id("outbound"), + on_message=self._forward_outbound_message, + ) + self._unsubscribe_callbacks.append(unsubscribe) + + tenant_slug = self._config.tenant_identity.tenant_slug + for topic in self._config.mirror_topics.inbound: + unsubscribe = await self._cloud_bus.subscribe( + topic=prefix_topic(tenant_slug, topic), + group_id=self._group_id("inbound"), + on_message=self._consume_inbound_message, + ) + self._unsubscribe_callbacks.append(unsubscribe) + + self._started = True + + async def stop(self) -> None: + """Stop all active subscriptions.""" + callbacks = list(reversed(self._unsubscribe_callbacks)) + self._unsubscribe_callbacks.clear() + self._started = False + for unsubscribe in callbacks: + await unsubscribe() + + async def _forward_outbound_message(self, message: object) -> None: + envelope = self._decode_message(message) + transformed = self._outbound_handler.forward_outbound(envelope) + await self._cloud_bus.publish( + topic=transformed.wire_topic, + key=getattr(message, "key", None), + value=self._encode_envelope(transformed), + headers=getattr(message, "headers", None), + ) + + async def _consume_inbound_message(self, message: object) -> None: + envelope = self._decode_message(message) + transformed = self._inbound_handler.consume_inbound(envelope) + await self._local_bus.publish( + topic=transformed.canonical_topic, + key=getattr(message, "key", None), + value=self._encode_envelope(transformed), + headers=getattr(message, "headers", None), + ) + + def _group_id(self, direction: str) -> str: + identity = self._config.tenant_identity + return f"tenant-{identity.tenant_slug}-gateway-forwarder-{direction}" + + @staticmethod + def _decode_message(message: object) -> ModelGatewayEnvelope: + value = getattr(message, "value", message) + if isinstance(value, str): + value = value.encode("utf-8") + if not isinstance(value, bytes): + raise TypeError("gateway bus message value must be bytes or string") + return ModelGatewayEnvelope.model_validate_json(value) + + @staticmethod + def _encode_envelope(envelope: ModelGatewayEnvelope) -> bytes: + return envelope.model_dump_json().encode("utf-8") diff --git a/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/service_gateway_topic_transform.py b/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/service_gateway_topic_transform.py new file mode 100644 index 0000000000..9b68cc1200 --- /dev/null +++ b/src/omnibase_infra/nodes/node_bus_forwarder_effect/services/service_gateway_topic_transform.py @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Tenant-prefix transform helpers for the gateway trust boundary.""" + +from __future__ import annotations + +import re + +from omnibase_core.validation import validate_topic_suffix + +RESERVED_TENANT_SLUGS = frozenset({"", "system"}) +_TENANT_SLUG_RE = re.compile(r"^[a-z][a-z0-9-]{1,61}[a-z0-9]$") + + +def validate_tenant_slug(tenant_slug: str) -> str: + """Validate a non-reserved tenant slug for gateway wire prefixes.""" + slug = tenant_slug.strip() if tenant_slug else tenant_slug + if slug in RESERVED_TENANT_SLUGS: + raise ValueError(f"tenant_slug is reserved: {slug!r}") + if not slug or not _TENANT_SLUG_RE.match(slug) or "--" in slug: + raise ValueError("tenant_slug must be DNS-compatible lowercase slug") + return slug + + +def validate_canonical_topic(canonical_topic: str) -> str: + """Validate a bare ONEX contract topic, never a tenant-prefixed wire topic.""" + topic = canonical_topic.strip() if canonical_topic else canonical_topic + if not topic: + raise ValueError("canonical_topic must not be empty") + if topic.startswith("tenant-"): + raise ValueError( + "canonical_topic must be bare and must not carry tenant prefix" + ) + result = validate_topic_suffix(topic) + if not result.is_valid: + raise ValueError(f"invalid canonical gateway topic: {result.error}") + return topic + + +def prefix_topic(tenant_slug: str, canonical_topic: str) -> str: + """Return the tenant-prefixed cloud wire topic.""" + slug = validate_tenant_slug(tenant_slug) + topic = validate_canonical_topic(canonical_topic) + return f"tenant-{slug}.{topic}" + + +def strip_topic_prefix(tenant_slug: str, wire_topic: str) -> str: + """Strip and validate the tenant prefix from a cloud wire topic.""" + slug = validate_tenant_slug(tenant_slug) + prefix = f"tenant-{slug}." + if not wire_topic.startswith(prefix): + raise ValueError("wire_topic does not match attached tenant prefix") + canonical_topic = wire_topic[len(prefix) :] + return validate_canonical_topic(canonical_topic) diff --git a/src/omnibase_infra/runtime/auto_wiring/handler_wiring.py b/src/omnibase_infra/runtime/auto_wiring/handler_wiring.py index f70ce222ec..e5361fb7dd 100644 --- a/src/omnibase_infra/runtime/auto_wiring/handler_wiring.py +++ b/src/omnibase_infra/runtime/auto_wiring/handler_wiring.py @@ -99,6 +99,9 @@ from omnibase_infra.protocols.protocol_pattern_b_broker_transport import ( ProtocolPatternBBrokerTransport, ) + from omnibase_infra.runtime.service_terminal_event_consumer import ( + TerminalEventConsumer, + ) logger = logging.getLogger(__name__) @@ -1541,6 +1544,7 @@ def _should_skip_sync_container_resolution(handler_cls: type) -> bool: { "event_bus", "event_publisher", + "event_consumer", "dispatch_port", "provisioner", "drain_proof_gate", @@ -1598,6 +1602,43 @@ def _log_publish_failure(done: asyncio.Task[None]) -> None: return _publish +def _make_sync_event_consumer( + *, + event_bus: object, + handler_name: str, +) -> TerminalEventConsumer: + """Materialize the blocking terminal-event consumer for request/response handlers. + + Mirror of ``_make_sync_event_publisher`` for the consume leg. Some EFFECT + handlers (e.g. ``HandlerContextRoiRunner``, OMN-13005) publish a command and + then block on the correlated terminal event, reading result fields back from + its payload. They declare an injectable ``event_consumer``. + + The returned object is directly callable with the legacy single-call shape + ``(terminal_topic, correlation_id, timeout_seconds) -> dict | None`` and also + exposes ``.open(topic) -> TerminalConsumerSession`` for the two-phase + (subscribe-before-publish) protocol introduced in OMN-13012: the handler + positions the consumer (assign + seek_to_end) BEFORE publishing its command, + then waits AFTER, so a terminal emitted in the publish→wait gap is not seeked + past. + + Without this injection the handler falls back to its own no-op default that + returns ``None`` immediately — never honoring the timeout, so every result + row is a degenerate generation-failure even though the terminal event arrives + moments later. The concrete consumer runs the correlate-and-wait loop on an + isolated event loop in a worker thread so blocking does not deadlock the + runtime dispatch loop that delivers the awaited terminal. + """ + from omnibase_infra.runtime.service_terminal_event_consumer import ( + make_terminal_event_consumer, + ) + + return make_terminal_event_consumer( + event_bus=event_bus, + handler_name=handler_name, + ) + + def _contracts_root_for_runtime_dependencies() -> Path: raw = os.environ.get("ONEX_CONTRACTS_DIR") if raw: @@ -1681,10 +1722,12 @@ def _materialize_known_handler_dependencies( requires_delegation_port = _handler_requires_delegation_dispatch_port(handler_cls) required_params = _required_handler_init_params(handler_cls) requires_event_publisher = "event_publisher" in constructor_params + requires_event_consumer = "event_consumer" in constructor_params if ( not required_params and not requires_delegation_port and not requires_event_publisher + and not requires_event_consumer ): return materialized_explicit_dependencies available = { @@ -1701,10 +1744,16 @@ def _materialize_known_handler_dependencies( event_bus=event_bus, handler_name=handler_name, ) + if requires_event_consumer and event_bus is not None: + available["event_consumer"] = _make_sync_event_consumer( + event_bus=event_bus, + handler_name=handler_name, + ) if required_params.issubset(_TOPIC_MIGRATION_EXECUTOR_DEPS): available.update(_build_topic_migration_executor_dependencies()) if not ( requires_event_publisher + or requires_event_consumer or required_params.intersection( {"container", "ownership_query", "dispatch_port"} ) @@ -1739,6 +1788,8 @@ def _materialize_known_handler_dependencies( handler_dependencies.setdefault("dispatch_port", available["dispatch_port"]) if requires_event_publisher and "event_publisher" in available: handler_dependencies.setdefault("event_publisher", available["event_publisher"]) + if requires_event_consumer and "event_consumer" in available: + handler_dependencies.setdefault("event_consumer", available["event_consumer"]) merged[handler_name] = handler_dependencies return merged diff --git a/src/omnibase_infra/runtime/config_discovery/config_prefetcher.py b/src/omnibase_infra/runtime/config_discovery/config_prefetcher.py index 060d80455c..a9142dc51b 100644 --- a/src/omnibase_infra/runtime/config_discovery/config_prefetcher.py +++ b/src/omnibase_infra/runtime/config_discovery/config_prefetcher.py @@ -19,6 +19,21 @@ is unavailable, prefetch fails loudly. When not enforced, missing values are logged as warnings and skipped. +Precedence Rules (OMN-13070): + On **controlled lanes** (``infisical_required=True``): + 1. Fetched/overlay config (Infisical) **always wins** over ambient env. + 2. When Infisical returns ``None`` for a key, ambient env is used as a + **declared bootstrap fallback** — but only if the key is present, and + a ``INFO``-level provenance log line is emitted identifying that the + value came from ambient env rather than the controlled store. + 3. A key missing from both Infisical and ambient env is reported as an + error (not silently dropped). + + On **uncontrolled lanes** (``infisical_required=False``, the default): + - Keys already present in the process environment are skipped; the + ambient value is used directly (legacy behaviour — local dev + without Infisical continues to work). + .. versionadded:: 0.10.0 Created as part of OMN-2287. @@ -30,6 +45,11 @@ ``BaseModel`` (``ConfigDict(frozen=True, extra="forbid")``). The ``missing`` field type changed from ``list[str]`` to ``tuple[str, ...]`` to satisfy immutability requirements. + +.. versionchanged:: 0.10.3 + OMN-13070: On controlled lanes (``infisical_required=True``) fetched config + wins over ambient env. Ambient env is now only a declared bootstrap + fallback on controlled lanes, with a provenance log line. """ from __future__ import annotations @@ -169,15 +189,95 @@ def _fetch_key( ) return None + def _resolve_key( + self, + key: str, + spec: ModelTransportConfigSpec, + *, + is_env_dep: bool = False, + ) -> tuple[str, SecretStr | None, str | None]: + """Resolve a single config key and return the outcome to the caller. + + Returns a 3-tuple ``(outcome, value, error)`` where: + - ``outcome`` is one of ``"resolved"``, ``"missing"``, or ``"error"`` + - ``value`` is the ``SecretStr`` when ``outcome == "resolved"`` + - ``error`` is the error message string when ``outcome == "error"`` + + On controlled lanes (``infisical_required=True``), Infisical is + always consulted first. Ambient env is used only as a declared + bootstrap fallback when Infisical returns ``None``, and a provenance + ``INFO`` log line is emitted in that case. + + On uncontrolled lanes (``infisical_required=False``), a key already + present in the process environment is used directly (legacy behaviour + — local dev without Infisical continues to work unchanged). + + Args: + key: The config key name. + spec: The transport config spec (provides the Infisical folder). + is_env_dep: ``True`` when the key comes from an explicit + ``dependencies[]`` declaration rather than a transport spec. + + Returns: + ``(outcome, value, error)`` 3-tuple; callers record into accumulators. + """ + if self._infisical_required: + # Controlled lane: Infisical/overlay WINS over ambient env. + # Always attempt the fetch; fall back to ambient env only when + # Infisical returns None, with an explicit provenance log line. + value = self._fetch_key(key, spec) + if value is not None: + return ("resolved", value, None) + if key in os.environ: + # Declared bootstrap fallback — log provenance explicitly. + logger.info( + "Key %s not found in Infisical; using ambient env as " + "bootstrap fallback (controlled lane) — " + "provenance: process environment", + key, + ) + return ("resolved", SecretStr(os.environ[key]), None) + # Missing from both Infisical and ambient env on a controlled + # lane: record as error, not silently dropped. + if is_env_dep or spec.required: + return ( + "error", + None, + f"Required key {key} not found at" + f" {spec.infisical_folder} and absent from ambient env", + ) + return ("missing", None, None) + + # Uncontrolled lane: ambient env takes precedence (legacy). + if key in os.environ: + logger.debug( + "Key %s already in environment, skipping prefetch", + key, + ) + return ("resolved", SecretStr(os.environ[key]), None) + + value = self._fetch_key(key, spec) + if value is not None: + return ("resolved", value, None) + return ("missing", None, None) + def prefetch( self, requirements: ModelConfigRequirements, ) -> ModelPrefetchResult: """Prefetch all configuration values for the given requirements. - Builds transport specs from the requirements' transport types, - then fetches each key via the handler. Keys already present in - the process environment are skipped (env always takes precedence). + Builds transport specs from the requirements' transport types, then + fetches each key via the handler. + + Precedence on controlled lanes (``infisical_required=True``): + Infisical/overlay config **wins** over ambient env. Ambient env is + used only as a declared bootstrap fallback when Infisical returns + ``None``, and a provenance ``INFO`` log line is emitted. + + Precedence on uncontrolled lanes (``infisical_required=False``): + Keys already present in the process environment are skipped; the + ambient value is used directly (legacy behaviour for local dev). Args: requirements: Config requirements extracted from contracts. @@ -216,33 +316,20 @@ def prefetch( env_keys.append(req.key) logger.info( - "Prefetching config: %d transport specs, %d env keys", + "Prefetching config: %d transport specs, %d env keys (controlled_lane=%s)", len(specs), len(env_keys), + self._infisical_required, ) # Fetch transport-based keys for spec in specs: for key in spec.keys: - # Skip if already in environment (env overrides Infisical). - # Use ``key in os.environ`` (not ``os.environ.get(key)``) so - # that intentionally empty values are respected and not - # overwritten by Infisical. - if key in os.environ: - logger.debug( - "Key %s already in environment, skipping prefetch", - key, - ) - resolved[key] = SecretStr(os.environ[key]) - continue - - value = self._fetch_key(key, spec) - if value is not None: + outcome, value, error = self._resolve_key(key, spec) + if outcome == "resolved" and value is not None: resolved[key] = value - elif self._infisical_required and spec.required: - errors[key] = ( - f"Required key {key} not found at {spec.infisical_folder}" - ) + elif outcome == "error" and error is not None: + errors[key] = error else: missing.append(key) @@ -257,18 +344,13 @@ def prefetch( keys=tuple(env_keys), ) for key in env_keys: - if key in os.environ: - resolved[key] = SecretStr(os.environ[key]) - continue - - value = self._fetch_key(key, env_spec) - if value is not None: + outcome, value, error = self._resolve_key( + key, env_spec, is_env_dep=True + ) + if outcome == "resolved" and value is not None: resolved[key] = value - elif self._infisical_required: - errors[key] = ( - f"Required env dependency key {key} not found at" - f" {env_spec.infisical_folder}" - ) + elif outcome == "error" and error is not None: + errors[key] = error else: missing.append(key) @@ -316,25 +398,45 @@ def prefetch_for_contracts( def apply_to_environment(self, result: ModelPrefetchResult) -> int: """Apply prefetched values to the process environment. - Only sets keys that are NOT already in the environment (environment - variables always take precedence over Infisical values). + On controlled lanes (``infisical_required=True``), fetched config + **always wins**: existing ambient env values are overwritten so that + stale shell or compose state cannot survive the prefetch boundary. + + On uncontrolled lanes (``infisical_required=False``), only keys that + are NOT already present in the environment are set (legacy behaviour + — local dev without Infisical continues to work unchanged). Args: result: The prefetch result to apply. Returns: - Number of keys actually set in the environment. + Number of keys actually set (or overwritten) in the environment. """ applied = 0 for key, value in result.resolved.items(): - if key not in os.environ: + if self._infisical_required: + # Controlled lane: fetched config wins — overwrite ambient env. + existing = os.environ.get(key) + os.environ[key] = value.get_secret_value() + applied += 1 + if existing is not None: + logger.info( + "Applied prefetched key %s to environment " + "(controlled lane — overwrote ambient env value)", + key, + ) + else: + logger.debug("Applied prefetched key %s to environment", key) + # Uncontrolled lane: ambient env takes precedence (legacy). + elif key not in os.environ: os.environ[key] = value.get_secret_value() applied += 1 logger.debug("Applied prefetched key %s to environment", key) logger.info( - "Applied %d/%d prefetched keys to environment", + "Applied %d/%d prefetched keys to environment (controlled_lane=%s)", applied, len(result.resolved), + self._infisical_required, ) return applied diff --git a/src/omnibase_infra/runtime/config_provenance.py b/src/omnibase_infra/runtime/config_provenance.py new file mode 100644 index 0000000000..4ea40238a4 --- /dev/null +++ b/src/omnibase_infra/runtime/config_provenance.py @@ -0,0 +1,166 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Config provenance for runtime-rendered contracts (OMN-12958). + +The Bifrost delegation contract is rendered once at container startup and written +to a Docker named volume at ``/app/data/delegation/bifrost_delegation.yaml``. The +volume survives ``docker compose build``/recreate, so the deployed (volume) config +can silently diverge from the packaged source shipped in the image. Two competing +authorities — the packaged source and the volume copy — is the defect this module +addresses. + +This module computes a deterministic provenance record (path + sha256 of both the +deployed volume contract and the packaged source) and classifies drift between +them. The record is: + +* logged at runtime startup (path + sha), +* exposed via the runtime health endpoint + (``health/health_config_provenance.py``), +* written next to the deployed contract as a sidecar JSON so the sweep + (omnimarket ``node_volume_config_drift_sweep``) and proof packets can read it + without re-deriving the source path. + +This module is **read-only** with respect to the contract content itself: it never +rewrites the deployed contract. The re-seed (overwriting a drifted volume copy from +packaged source) is an operator deploy-procedure action, not a runtime side effect. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +from pydantic import BaseModel, ConfigDict, Field + +# Sidecar file written next to the rendered contract. Named with a leading dot so +# it never collides with a contract-discovery glob. +PROVENANCE_SIDECAR_NAME = ".config_provenance.json" + + +def compute_sha256(path: Path) -> str | None: + """Return the hex sha256 of ``path`` content, or ``None`` when absent. + + Hashes raw bytes (not parsed YAML) so the digest is byte-exact and stable + across machines. Returns ``None`` for a missing file rather than raising, + because an absent deployed contract is itself a valid provenance state that + the caller classifies. + """ + if not path.exists(): + return None + digest = hashlib.sha256() + digest.update(path.read_bytes()) + return digest.hexdigest() + + +class ModelConfigProvenance(BaseModel): + """Provenance of a runtime-rendered config relative to its packaged source. + + ``deployed`` is the volume copy actually loaded by the runtime; ``source`` is + the packaged contract shipped in the image. ``has_drifted`` is ``True`` only + when both digests are known and differ — an absent deployed copy (first boot) + or absent source is reported via the digests being ``None``, not as drift. + """ + + model_config = ConfigDict(strict=True, frozen=True, extra="forbid") + + config_name: str = Field( + ..., + description="Logical config identifier (e.g. 'bifrost_delegation')", + min_length=1, + ) + deployed_path: str = Field( + ..., + description="Absolute path of the deployed (volume) contract", + min_length=1, + ) + deployed_sha256: str | None = Field( + default=None, + description="sha256 of the deployed contract bytes, or None if absent", + ) + source_path: str = Field( + ..., + description="Absolute path of the packaged source contract", + min_length=1, + ) + source_sha256: str | None = Field( + default=None, + description="sha256 of the packaged source contract bytes, or None if absent", + ) + + @property + def deployed_present(self) -> bool: + """Whether the deployed (volume) contract exists.""" + return self.deployed_sha256 is not None + + @property + def source_present(self) -> bool: + """Whether the packaged source contract exists.""" + return self.source_sha256 is not None + + @property + def has_drifted(self) -> bool: + """Whether the deployed copy diverges from packaged source. + + ``True`` only when both digests are known and differ. Unknown digests + (missing file) are not drift — the caller inspects ``*_present``. + """ + if self.deployed_sha256 is None or self.source_sha256 is None: + return False + return self.deployed_sha256 != self.source_sha256 + + def provenance_line(self) -> str: + """Single-line, log-friendly provenance summary (path + sha + drift).""" + deployed = self.deployed_sha256 or "absent" + source = self.source_sha256 or "absent" + drift = "DRIFT" if self.has_drifted else "in-sync" + return ( + f"config_provenance config={self.config_name} status={drift} " + f"deployed_path={self.deployed_path} deployed_sha256={deployed} " + f"source_path={self.source_path} source_sha256={source}" + ) + + +def build_config_provenance( + *, + config_name: str, + deployed_path: Path, + source_path: Path, +) -> ModelConfigProvenance: + """Compute provenance for a deployed contract against its packaged source.""" + return ModelConfigProvenance( + config_name=config_name, + deployed_path=str(deployed_path), + deployed_sha256=compute_sha256(deployed_path), + source_path=str(source_path), + source_sha256=compute_sha256(source_path), + ) + + +def write_provenance_sidecar( + provenance: ModelConfigProvenance, + *, + deployed_path: Path, +) -> Path: + """Write the provenance record as a JSON sidecar next to the deployed contract. + + The sidecar lets the drift sweep and proof packets read provenance without + re-resolving the packaged source path (which depends on import-time package + layout inside the container). Returns the sidecar path. + """ + sidecar = deployed_path.parent / PROVENANCE_SIDECAR_NAME + sidecar.parent.mkdir(parents=True, exist_ok=True) + sidecar.write_text( + json.dumps(provenance.model_dump(mode="json"), indent=2, sort_keys=True), + encoding="utf-8", + ) + return sidecar + + +__all__: list[str] = [ + "PROVENANCE_SIDECAR_NAME", + "ModelConfigProvenance", + "build_config_provenance", + "compute_sha256", + "write_provenance_sidecar", +] diff --git a/src/omnibase_infra/runtime/health/health_config_provenance.py b/src/omnibase_infra/runtime/health/health_config_provenance.py new file mode 100644 index 0000000000..d22d8e2ab5 --- /dev/null +++ b/src/omnibase_infra/runtime/health/health_config_provenance.py @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Health check for runtime config provenance (OMN-12958). + +Promotes config provenance (deployed-volume sha vs packaged-source sha) to a +health-check component so volume/config drift pages an operator and surfaces in +proof packets instead of silently degrading delegation routing. + +* deployed contract absent -> unhealthy (runtime has no config to load) +* deployed sha != source sha -> degraded (drift: re-seed required) +* source absent -> degraded (cannot prove provenance) +* deployed sha == source sha -> healthy +""" + +from __future__ import annotations + +from omnibase_core.types import JsonType +from omnibase_infra.runtime.config_provenance import ModelConfigProvenance +from omnibase_infra.runtime.models.model_component_health import ( + ModelComponentHealth, +) + +_COMPONENT_NAME = "config_provenance" + + +def check_config_provenance_health( + provenance: ModelConfigProvenance, +) -> ModelComponentHealth: + """Classify config provenance into a component-health status. + + Args: + provenance: Computed provenance for a deployed contract. + + Returns: + ModelComponentHealth carrying the path + sha details so the health + endpoint and proof packets expose provenance, not just a status bit. + """ + details: dict[str, JsonType] = { + "config_name": provenance.config_name, + "deployed_path": provenance.deployed_path, + "deployed_sha256": provenance.deployed_sha256, + "source_path": provenance.source_path, + "source_sha256": provenance.source_sha256, + "has_drifted": provenance.has_drifted, + } + + if not provenance.deployed_present: + return ModelComponentHealth.unhealthy( + name=_COMPONENT_NAME, + error=( + f"deployed config absent at {provenance.deployed_path} " + f"({provenance.config_name})" + ), + details=details, + ) + + if not provenance.source_present: + return ModelComponentHealth.degraded( + name=_COMPONENT_NAME, + error=( + f"packaged source absent at {provenance.source_path}; " + "cannot prove config provenance" + ), + details=details, + ) + + if provenance.has_drifted: + return ModelComponentHealth.degraded( + name=_COMPONENT_NAME, + error=( + f"volume config drifted from packaged source for " + f"{provenance.config_name}: deployed_sha256=" + f"{provenance.deployed_sha256} source_sha256=" + f"{provenance.source_sha256}; re-seed required" + ), + details=details, + ) + + return ModelComponentHealth.healthy( + name=_COMPONENT_NAME, + details=details, + ) + + +__all__: list[str] = ["check_config_provenance_health"] diff --git a/src/omnibase_infra/runtime/models/model_bifrost_lane_overlay.py b/src/omnibase_infra/runtime/models/model_bifrost_lane_overlay.py new file mode 100644 index 0000000000..afb3227806 --- /dev/null +++ b/src/omnibase_infra/runtime/models/model_bifrost_lane_overlay.py @@ -0,0 +1,101 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Typed model for the Bifrost lane overlay YAML (OMN-12864). + +The lane overlay commits the four BIFROST_LOCAL_* endpoint URL bindings that +were previously only available via ephemeral shell exports on .201. Storing +them as a committed typed YAML file under docker/lane-overlays/ makes the +deployment authority auditable, diff-able, and CI-checked. + +Each endpoint_url is the COMPLETE final URL including the /v1/chat/completions +path (OMN-12815). The overlay file is rendered to a dotenv sidecar +(docker/lane-overlays/.bifrost.env) consumed by the compose stack at +runtime. Any mismatch between the YAML source and rendered env sidecar fails +CI via test_bifrost_lane_overlay_env_in_sync. + +Validation enforces OMN-12815: every URL must end in /v1/chat/completions or +/chat/completions to catch bare-base misconfiguration early. +""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +class ModelBifrostLaneOverlay(BaseModel): + """Typed deployment bindings for Bifrost local backend endpoint URLs. + + All four endpoint_url values are COMPLETE final URLs (OMN-12815): + they are posted VERBATIM at every call site — no in-code construction. + """ + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + lane: str = Field( + min_length=1, + description="Lane identifier (e.g. 'dev', 'stability-test', 'prod').", + ) + coder_endpoint_url: str = Field( + min_length=1, + description=( + "COMPLETE chat-completions URL for the local coder backend " + "(vLLM on .201:8000). Posted VERBATIM — no path appended." + ), + ) + reasoner_endpoint_url: str = Field( + min_length=1, + description=( + "COMPLETE chat-completions URL for the local reasoner backend " + "(vLLM on .201:8001). Posted VERBATIM — no path appended." + ), + ) + embedding_endpoint_url: str = Field( + min_length=1, + description=( + "COMPLETE chat-completions URL for the local embedding backend " + "(.201:8100). Posted VERBATIM — no path appended." + ), + ) + ds4_flash_endpoint_url: str = Field( + min_length=1, + description=( + "COMPLETE chat-completions URL for the DS-V4-Flash backend " + "(.200:8101). Posted VERBATIM — no path appended." + ), + ) + + @model_validator(mode="after") + def _validate_complete_endpoint_urls(self) -> ModelBifrostLaneOverlay: + """Reject bare-base URLs that lack the /chat/completions path.""" + fields = { + "coder_endpoint_url": self.coder_endpoint_url, + "reasoner_endpoint_url": self.reasoner_endpoint_url, + "embedding_endpoint_url": self.embedding_endpoint_url, + "ds4_flash_endpoint_url": self.ds4_flash_endpoint_url, + } + errors: list[str] = [] + for field_name, url in fields.items(): + stripped = url.rstrip("/") + if not (stripped.endswith(("/v1/chat/completions", "/chat/completions"))): + errors.append( + f"{field_name}: endpoint_url must end in /v1/chat/completions " + f"or /chat/completions (OMN-12815), got {url!r}" + ) + if errors: + msg = "Bifrost lane overlay endpoint_url completeness errors: " + "; ".join( + errors + ) + raise ValueError(msg) + return self + + def as_env_dict(self) -> dict[str, str]: + """Return the env-var mapping consumed by the compose stack.""" + return { + "BIFROST_LOCAL_CODER_ENDPOINT_URL": self.coder_endpoint_url, + "BIFROST_LOCAL_REASONER_ENDPOINT_URL": self.reasoner_endpoint_url, + "BIFROST_LOCAL_EMBEDDING_ENDPOINT_URL": self.embedding_endpoint_url, + "BIFROST_LOCAL_DS_V4_FLASH_ENDPOINT_URL": self.ds4_flash_endpoint_url, + } + + +__all__: list[str] = ["ModelBifrostLaneOverlay"] diff --git a/src/omnibase_infra/runtime/models/model_runtime_policy_contract.py b/src/omnibase_infra/runtime/models/model_runtime_policy_contract.py index 24490a156a..aa6487e8f4 100644 --- a/src/omnibase_infra/runtime/models/model_runtime_policy_contract.py +++ b/src/omnibase_infra/runtime/models/model_runtime_policy_contract.py @@ -24,6 +24,9 @@ class ModelRuntimePolicyContract(BaseModel): version: int = Field(ge=1) active_runtime_packages: tuple[str, ...] = Field(min_length=1) llm_cloud_endpoint_host_allowlist: tuple[str, ...] = Field(default=()) + bifrost_vertex_gemini_endpoint_url: str = Field(min_length=1) + google_cloud_project: str = Field(min_length=1) + google_cloud_location: str = Field(min_length=1) omnimemory_memgraph_port: int = Field(ge=1, le=65535) auxiliary_services_omnimemory_enabled: bool profiles: dict[RuntimeProfileName, ModelRuntimeProfilePolicy] diff --git a/src/omnibase_infra/runtime/models/model_runtime_process_policy.py b/src/omnibase_infra/runtime/models/model_runtime_process_policy.py index 815602b8c4..9b263d83b4 100644 --- a/src/omnibase_infra/runtime/models/model_runtime_process_policy.py +++ b/src/omnibase_infra/runtime/models/model_runtime_process_policy.py @@ -23,6 +23,14 @@ class ModelRuntimeProcessPolicy(BaseModel): omnimemory_enabled: bool omnimemory_memgraph_host: str = "" publish_introspection: bool = False + # OMN-12990: contract-declared replica count for this runtime process. + # The base compose worker deploy default is ${WORKER_REPLICAS:-0}, which + # silently scales the worker to zero on a plain compose recreate. Each lane + # pins this value via {PROFILE}_WORKER_REPLICAS (rendered into + # runtime-policy.env) and the lane overrides reference it fail-fast, so a + # recreate that omits the policy env fails loudly instead of dropping the + # worker with no signal. + replicas: int = Field(default=1, ge=1) @field_validator("capabilities") @classmethod diff --git a/src/omnibase_infra/runtime/render_bifrost_delegation_contract.py b/src/omnibase_infra/runtime/render_bifrost_delegation_contract.py index 3a83300591..7aad96cc19 100644 --- a/src/omnibase_infra/runtime/render_bifrost_delegation_contract.py +++ b/src/omnibase_infra/runtime/render_bifrost_delegation_contract.py @@ -7,6 +7,31 @@ backend named by each backend's ``base_url_env`` field. This module materializes the deployed contract once at container startup so delegation code reads a contract artifact, not scattered environment variables. + +OMN-12864 — committed overlay authority + BIFROST_LOCAL_*_ENDPOINT_URL values are now committed deployment bindings + in docker/lane-overlays/dev.bifrost.yaml (and per-lane siblings). They are + no longer ambient shell exports. A clean-shell compose up picks them up via + the lane overlay env_file. + +OMN-12814 — fail-loud loader + ``render_bifrost_delegation_contract`` raises ``ProtocolConfigurationError`` + on any error that previously produced a silent empty result: + - FileNotFoundError on missing source contract + - yaml.YAMLError on malformed source YAML + - ValidationError on schema violations + - zero populated endpoints after rendering + There is no lru_cache on the render result. Every restart re-renders from + the packaged source (OMN-12945), so a stale cache can never pin a broken + result across a deploy. + +OMN-12945 — re-seed from packaged source on deploy + The ``force_reseed`` parameter (and the ``BIFROST_FORCE_RESEED`` env flag) + bypasses the stale-volume early-return path and always rebuilds the volume + copy from the packaged bifrost_delegation.yaml merged with the committed + lane overlay env-var endpoints. The entrypoint sets ``BIFROST_FORCE_RESEED`` + so every container restart re-seeds, eliminating the named-volume drift + defect (OMN-12945). """ from __future__ import annotations @@ -24,6 +49,10 @@ import yaml from omnibase_infra.errors import ProtocolConfigurationError +from omnibase_infra.runtime.config_provenance import ( + build_config_provenance, + write_provenance_sidecar, +) _LEGACY_SOURCE_PATH = Path("/app/src/omnibase_infra/configs/bifrost_delegation.yaml") _DEFAULT_TARGET_PATH = Path("/app/data/delegation/bifrost_delegation.yaml") @@ -494,14 +523,54 @@ def render_bifrost_delegation_contract( environ: Mapping[str, str] | None = None, verify_endpoints: bool | None = None, endpoint_probe: EndpointProbe | None = None, + force_reseed: bool | None = None, ) -> Path | None: - """Render the deployed Bifrost delegation contract and return its path.""" + """Render the deployed Bifrost delegation contract and return its path. + + Args: + source_path: Path to the packaged bifrost_delegation.yaml source. + Defaults to the omnimarket-bundled copy. + target_path: Path to write the rendered contract on the runtime volume. + Defaults to /app/data/delegation/bifrost_delegation.yaml. + Pass ``None`` to accept the env-configured target; returns None + when BIFROST_CONTRACT_PATH resolves to an empty path (services + that deliberately skip rendering, e.g. projection-api). + environ: Mapping used for env-var reads. Defaults to os.environ. + verify_endpoints: When True, probe each populated endpoint via + GET /v1/models and reject backends whose model is not listed. + Defaults to the BIFROST_VERIFY_ENDPOINTS env flag. + endpoint_probe: Probe callable used for endpoint verification. Defaults + to _probe_openai_model_endpoint. + force_reseed: When True (or when BIFROST_FORCE_RESEED=1 is set), always + re-render from the packaged source, ignoring any existing volume + content (OMN-12945). This is the correct mode on container restart: + the volume copy is never an authority on a fresh boot. + + Returns: + Path to the rendered contract, or None when rendering is explicitly + disabled for this service. + + Raises: + ProtocolConfigurationError: On any rendering failure — missing source, + malformed YAML, schema validation failure, or zero populated + endpoints (OMN-12814: fail-loud, never returns a silent empty result). + FileNotFoundError: If the source contract cannot be found. + ValueError: If the source YAML fails schema validation. + """ env = environ if environ is not None else os.environ should_verify = ( _env_flag(env.get("BIFROST_VERIFY_ENDPOINTS"), default=False) if verify_endpoints is None else verify_endpoints ) + # OMN-12945: BIFROST_FORCE_RESEED bypasses the stale-volume early-return path. + # The entrypoint sets this flag so every container restart re-seeds from the + # packaged source. Callers may also pass force_reseed=True explicitly. + should_force_reseed = ( + _env_flag(env.get("BIFROST_FORCE_RESEED"), default=False) + if force_reseed is None + else force_reseed + ) probe = endpoint_probe or _probe_openai_model_endpoint _source_env = env.get("BIFROST_SOURCE_CONTRACT_PATH", "").strip() source = source_path or (Path(_source_env) if _source_env else _DEFAULT_SOURCE_PATH) @@ -509,7 +578,15 @@ def render_bifrost_delegation_contract( if target is None: return None - if not should_verify and _has_populated_endpoint(target): + # Cache-hit path: skip re-rendering when the volume already contains a + # valid, up-to-date contract AND we are not forcing a re-seed. + # OMN-12945: force_reseed/BIFROST_FORCE_RESEED bypasses this entirely — + # the entrypoint sets it on every restart to eliminate named-volume drift. + if ( + not should_force_reseed + and not should_verify + and _has_populated_endpoint(target) + ): source_data = _load_yaml(source) if source.exists() else None data = _load_yaml(target) _validate_bifrost_delegation_data( @@ -528,6 +605,9 @@ def render_bifrost_delegation_contract( _strip_render_hint_fields(target) return target + # Re-seed path: always render from the packaged source. + # OMN-12814: every error below raises ProtocolConfigurationError — no silent + # empty-result fallback. data = _load_render_source( should_verify=should_verify, target=target, source=source ) @@ -546,8 +626,11 @@ def render_bifrost_delegation_contract( if populated == 0: raise ProtocolConfigurationError( "Bifrost delegation contract rendered with no populated endpoint_url " - "values. Add base_url_env fields to the source contract or provide a " - "pre-rendered contract via BIFROST_CONTRACT_PATH." + "values. Set the four BIFROST_LOCAL_*_ENDPOINT_URL vars from the " + "committed lane overlay (docker/lane-overlays/.bifrost.env) " + "or provide a pre-rendered contract at BIFROST_CONTRACT_PATH. " + "OMN-12864: these values must come from the committed overlay, " + "not ephemeral shell exports." ) # Strip render-only hint fields before writing the deployed contract — @@ -566,6 +649,31 @@ def render_bifrost_delegation_contract( return target +def _emit_config_provenance(rendered: Path, *, env: Mapping[str, str]) -> None: + """Compute, log, and persist config provenance for the rendered contract. + + Provenance compares the deployed (volume) contract that the runtime actually + loaded against the packaged source resolved from the same rules the renderer + uses. The single-line summary is logged to stdout so it appears in container + startup logs; the sidecar JSON lets the drift sweep and proof packets read + provenance without re-resolving the packaged source path (OMN-12958). + """ + source_env = env.get("BIFROST_SOURCE_CONTRACT_PATH", "").strip() + source = Path(source_env) if source_env else _DEFAULT_SOURCE_PATH + provenance = build_config_provenance( + config_name="bifrost_delegation", + deployed_path=rendered, + source_path=source, + ) + sys.stdout.write(f"[entrypoint] {provenance.provenance_line()}\n") + if provenance.has_drifted: + sys.stdout.write( + "[entrypoint] WARNING: deployed Bifrost delegation config drifted " + "from packaged source; re-seed required (OMN-12958)\n" + ) + write_provenance_sidecar(provenance, deployed_path=rendered) + + def main() -> int: rendered = render_bifrost_delegation_contract() if rendered is None: @@ -575,6 +683,7 @@ def main() -> int: ) return 0 sys.stdout.write(f"[entrypoint] Bifrost delegation contract ready: {rendered}\n") + _emit_config_provenance(rendered, env=os.environ) return 0 diff --git a/src/omnibase_infra/runtime/runtime_local.py b/src/omnibase_infra/runtime/runtime_local.py index e6ad606535..f39d86cd99 100644 --- a/src/omnibase_infra/runtime/runtime_local.py +++ b/src/omnibase_infra/runtime/runtime_local.py @@ -627,16 +627,26 @@ def _validate_routing( handlers = RuntimeLocal._as_workflow_maps(routing.get("handlers", [])) errors: list[str] = [] input_topic_set = set(subscribe_topics) + strategy = str(routing.get("routing_strategy", "")) # Per-entry field and topic validation for i, entry in enumerate(handlers): prefix = f"handlers[{i}]" em = RuntimeLocal._as_workflow_map(entry.get("event_model", {})) hd = RuntimeLocal._as_workflow_map(entry.get("handler", {})) - if not em.get("name"): - errors.append(f"{prefix}.event_model.name is missing") - if not em.get("module"): - errors.append(f"{prefix}.event_model.module is missing") + if strategy == "payload_type_match": + # payload_type_match routes by model type — event_model is required. + if not em.get("name"): + errors.append(f"{prefix}.event_model.name is missing") + if not em.get("module"): + errors.append(f"{prefix}.event_model.module is missing") + else: + # operation_match (and any future non-payload strategies) route by + # the `operation` field — event_model is not used and must not be + # required here. Require `operation` instead. + op = entry.get("operation") + if not op: + errors.append(f"{prefix}.operation is missing") if not hd.get("name"): errors.append(f"{prefix}.handler.name is missing") if not hd.get("module"): @@ -801,18 +811,26 @@ def _fail_callback() -> None: entry.handler_module, entry.handler_class ) - # Import the input model class - try: - em_mod = importlib.import_module(entry.event_model_module) - input_model_cls = getattr(em_mod, entry.event_model_class) - except (ImportError, AttributeError) as exc: - msg = ( - f"Failed to resolve event model " - f"{entry.event_model_module}.{entry.event_model_class}: {exc}" - ) - logger.exception("RuntimeLocal: %s", msg) - self._result = EnumWorkflowResult.FAILED - return + # Resolve the input model class. operation_match entries route by the + # `operation` field and declare no event_model, so event_model_module is + # empty — skip the import and forward the raw payload (OMN-13141). Only + # payload_type_match entries carry a typed event model to import. + input_model_type: type[BaseModel] | None = None + if entry.event_model_module: + try: + em_mod = importlib.import_module(entry.event_model_module) + input_model_type = cast( + "type[BaseModel]", + getattr(em_mod, entry.event_model_class), + ) + except (ImportError, AttributeError) as exc: + msg = ( + f"Failed to resolve event model " + f"{entry.event_model_module}.{entry.event_model_class}: {exc}" + ) + logger.exception("RuntimeLocal: %s", msg) + self._result = EnumWorkflowResult.FAILED + return def _make_fail_cb(name: str) -> Callable[[], None]: def _cb() -> None: @@ -821,7 +839,6 @@ def _cb() -> None: return _cb - input_model_type: type[BaseModel] = cast("type[BaseModel]", input_model_cls) adapter = LocalRuntimeBusAdapter( handler=cast("ProtocolLocalRuntimeCallableTarget", handler_instance), handler_name=entry.handler_name, @@ -1487,5 +1504,25 @@ def exit_code(self) -> int: """CLI exit code corresponding to the current result.""" return _exit_code_for(self._result) + @property + def last_error(self) -> str | None: + """Human-readable description of the failure that ended the run, if any. + + Set when a wired handler fails or a routing/boot error aborts the run; + ``None`` on a clean completion. Consumed by the ``onex node`` CLI to + populate the ``SkillRoutingError`` envelope on non-zero exit (OMN-8724). + """ + return self._last_error + + @property + def handler_result(self) -> object | None: + """The terminal handler's result object, if the run produced one. + + Receipt mode (OMN-13094) uses the concrete type of this object as + the receipt's ``result_model`` schema identity; the JSON-serialized + form lives in ``workflow_result.json`` under ``handler_result``. + """ + return self._handler_result + # Prevent resource leaks — reserved for future bus/container teardown. del_alias = None # placeholder for __del__ if needed diff --git a/src/omnibase_infra/runtime/runtime_local_adapter.py b/src/omnibase_infra/runtime/runtime_local_adapter.py index 711e8bff22..fe815dfd91 100644 --- a/src/omnibase_infra/runtime/runtime_local_adapter.py +++ b/src/omnibase_infra/runtime/runtime_local_adapter.py @@ -46,13 +46,16 @@ def __init__( self, handler: ProtocolLocalRuntimeCallableTarget, handler_name: str, - input_model_cls: type[BaseModel], + input_model_cls: type[BaseModel] | None, output_topic: str | None, bus: ProtocolLocalRuntimeBus, on_error: Callable[[], None] | None = None, ) -> None: self.handler = handler self.handler_name = handler_name + # None when the routing entry declares no payload model — operation_match + # entries route by `operation`, not a typed event model (OMN-13141). The + # raw decoded dict is forwarded to the handler in that case. self.input_model_cls = input_model_cls self.output_topic = output_topic self.bus = bus @@ -71,7 +74,14 @@ async def on_message(self, msg: ProtocolLocalRuntimeMessage) -> None: correlation_id = ( correlation_value if isinstance(correlation_value, str) else None ) - input_model = self.input_model_cls(**payload_dict) + # operation_match entries declare no payload model (OMN-13141): forward + # the raw decoded dict. payload_type_match validates against the model. + # Typed `object` (model or dict) — helpers narrow with isinstance. + input_payload: object = ( + self.input_model_cls(**payload_dict) + if self.input_model_cls is not None + else payload_dict + ) except Exception: # fallback-ok: local runtime adapter records handler failure and continues shutdown logger.exception( "LocalRuntimeBusAdapter: deserialization failed for %s (correlation_id=%s)", @@ -91,7 +101,7 @@ async def on_message(self, msg: ProtocolLocalRuntimeMessage) -> None: start = time.monotonic() try: handle_method = self.handler.handle - maybe_result = _invoke_handle_method(handle_method, input_model) + maybe_result = _invoke_handle_method(handle_method, input_payload) if inspect.isawaitable(maybe_result): awaitable_result: Awaitable[object] = cast( "Awaitable[object]", maybe_result @@ -157,17 +167,50 @@ async def on_message(self, msg: ProtocolLocalRuntimeMessage) -> None: def _invoke_handle_method( handle_method: Callable[..., object], - input_model: BaseModel, + input_payload: object, ) -> object: - """Invoke a local-runtime handler using its declared calling convention.""" + """Invoke a local-runtime handler using its declared calling convention. + + ``input_payload`` is a validated model for payload_type_match entries, or the + raw decoded dict for operation_match entries that declare no payload model + (OMN-13141). Keyword-fanout uses ``model_dump`` for models and the dict + directly; single-model-parameter handlers receive the object itself. + + When the handler's sole positional parameter is annotated with a concrete + ``BaseModel`` subclass but ``input_payload`` is still the raw decoded dict + (the operation_match case — no contract-declared event model to validate + against upstream), the dict is validated into that annotated type here + before the call (OMN-8724). Without this coercion a typed single-parameter + handler such as ``handle(self, request: GoldenChainSweepRequest)`` receives + a bare ``dict`` and crashes on the first attribute access. This is the + systemic fix for the dict-not-typed dispatch-boundary class (same family as + OMN-13141 / the savings_estimation ``.get()`` bug). + """ + kwargs: dict[str, object] = ( + input_payload.model_dump(mode="json") + if isinstance(input_payload, BaseModel) + else input_payload + if isinstance(input_payload, dict) + else {} + ) + # eval_str=True resolves PEP 563 string annotations (``from __future__ import + # annotations`` is used by every node handler, so without this the parameter + # annotation is the literal string "GoldenChainSweepRequest" and the + # BaseModel-subclass check below never matches — the OMN-8724 root cause). try: - signature = inspect.signature(handle_method) - except (TypeError, ValueError): - return handle_method(input_model) + signature = inspect.signature(handle_method, eval_str=True) + except (TypeError, ValueError, NameError): + # NameError: an annotation referenced a name not importable in the + # handler module globals. Fall back to unevaluated annotations rather + # than aborting dispatch. + try: + signature = inspect.signature(handle_method) + except (TypeError, ValueError): + return handle_method(input_payload) parameters = tuple(signature.parameters.values()) if any(param.kind is inspect.Parameter.VAR_KEYWORD for param in parameters): - return handle_method(**input_model.model_dump(mode="json")) + return handle_method(**kwargs) positional_parameters = tuple( param @@ -181,26 +224,59 @@ def _invoke_handle_method( if len(positional_parameters) == 0: return handle_method() - if len(positional_parameters) == 1 and _parameter_expects_model( - positional_parameters[0], - type(input_model), - ): - return handle_method(input_model) + if len(positional_parameters) == 1: + model_type = _coercion_target_model_type( + positional_parameters[0], + input_payload, + ) + if model_type is not None: + # Annotation is a concrete BaseModel subclass and the payload is a + # raw dict: validate the dict into the declared type before calling. + return handle_method(model_type.model_validate(input_payload)) + if _parameter_expects_model(positional_parameters[0], input_payload): + return handle_method(input_payload) - return handle_method(**input_model.model_dump(mode="json")) + return handle_method(**kwargs) + + +def _coercion_target_model_type( + parameter: inspect.Parameter, + input_payload: object, +) -> type[BaseModel] | None: + """Return the annotated ``BaseModel`` subclass to coerce a raw dict payload into. + + Returns ``None`` unless ``input_payload`` is a raw ``dict`` and the + parameter's annotation is a concrete ``BaseModel`` subclass. This is the only + coercion trigger (OMN-8724): a real annotation, not a parameter-name + heuristic, drives it — so a model-typed single-parameter handler reached via + ``operation_match`` (no upstream event-model validation) gets a validated + model instead of the raw dict that previously crashed it. + """ + if not isinstance(input_payload, dict): + return None + annotation = parameter.annotation + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + return annotation + return None def _parameter_expects_model( parameter: inspect.Parameter, - input_model_cls: type[BaseModel], + input_payload: object, ) -> bool: - """Return True when a single handle parameter expects the request model.""" + """Return True when a single handle parameter should receive the payload object. + + Only reached after dict→model coercion has been considered + (``_coercion_target_model_type``), so this governs the already-a-model and + name-heuristic pass-through paths that pre-date OMN-8724. + """ if parameter.name in {"request", "payload", "event", "input_model"}: return True annotation = parameter.annotation return ( - isinstance(annotation, type) + isinstance(input_payload, BaseModel) + and isinstance(annotation, type) and issubclass(annotation, BaseModel) - and issubclass(input_model_cls, annotation) + and issubclass(type(input_payload), annotation) ) diff --git a/src/omnibase_infra/runtime/runtime_profile.py b/src/omnibase_infra/runtime/runtime_profile.py index a5f11fe493..668589bfab 100644 --- a/src/omnibase_infra/runtime/runtime_profile.py +++ b/src/omnibase_infra/runtime/runtime_profile.py @@ -110,6 +110,16 @@ class ModelRuntimeProfile(BaseModel): } +# The infra runtime owns each profile's *behaviour* (prefetch policy); the +# canonical *name set* lives in omnibase_core (OMN-12957) so contract validation +# can enforce ``runtime_profiles`` membership without a core->infra dependency. +# The two must stay in lockstep: a name core blesses that infra cannot boot — or +# an infra profile core does not know about — is a silent-orphan hazard. The +# parity guard is the test ``test_profiles_match_core_registry`` (a hard import- +# time raise here would crash the runtime kernel on any core/infra version skew, +# so the invariant is enforced at test/CI time instead of at import). + + def load_runtime_profile(profile_name: str | None = None) -> ModelRuntimeProfile: """Return the ``ModelRuntimeProfile`` for *profile_name*. diff --git a/src/omnibase_infra/runtime/service_kernel.py b/src/omnibase_infra/runtime/service_kernel.py index 8dc313780b..4d646d0585 100644 --- a/src/omnibase_infra/runtime/service_kernel.py +++ b/src/omnibase_infra/runtime/service_kernel.py @@ -98,6 +98,7 @@ from omnibase_infra.event_bus.event_bus_inmemory import EventBusInmemory from omnibase_infra.event_bus.event_bus_kafka import EventBusKafka from omnibase_infra.event_bus.models.config import ModelKafkaEventBusConfig +from omnibase_infra.event_bus.models.model_event_message import ModelEventMessage from omnibase_infra.models import ModelNodeIdentity from omnibase_infra.models.health.model_llm_endpoint_health_config import ( ModelLlmEndpointHealthConfig, @@ -1484,6 +1485,7 @@ async def _baselines_loop() -> None: ) from omnibase_infra.services.observability.savings_estimation.consumer import ( ServiceSavingsEstimator, + decode_event_message, ) from omnibase_infra.topics import topic_keys from omnibase_infra.topics.service_topic_registry import ( @@ -1510,30 +1512,25 @@ async def _baselines_loop() -> None: async def _savings_consumer_loop() -> None: """Consume input events and produce savings estimates.""" - import json as _json - _poll_interval = 2.0 for _input_topic in _savings_input_topics: try: async def _savings_on_message( - msg: object, - *, - _topic: str = _input_topic, + message: ModelEventMessage, ) -> None: - _savings_estimator.ingest_event( - _topic, - # Why: Runtime wiring validates and narrows this payload shape before use. - _json.loads(msg) # type: ignore[arg-type] - if isinstance(msg, (str, bytes)) - else msg, - ) + # The event bus delivers a typed ModelEventMessage, + # not a raw dict/str. decode_event_message reads the + # JSON payload off the typed `.value` field — it + # must never call `.get()` on the message, which + # has no such method (OMN-13149). + _topic, _payload = decode_event_message(message) + _savings_estimator.ingest_event(_topic, _payload) await event_bus.subscribe( _input_topic, - # Why: Runtime wiring validates and narrows this payload shape before use. - on_message=_savings_on_message, # type: ignore[arg-type] + on_message=_savings_on_message, group_id=f"savings-estimator.{_input_topic}", ) except Exception: # noqa: BLE001 @@ -3231,10 +3228,6 @@ async def _wiring_health_loop() -> None: version="v1", ) - from omnibase_infra.event_bus.models.model_event_message import ( - ModelEventMessage, - ) - async def _triage_on_message( message: ModelEventMessage, ) -> None: diff --git a/src/omnibase_infra/runtime/service_pattern_b_broker.py b/src/omnibase_infra/runtime/service_pattern_b_broker.py index 9bae67238b..a4701eed52 100644 --- a/src/omnibase_infra/runtime/service_pattern_b_broker.py +++ b/src/omnibase_infra/runtime/service_pattern_b_broker.py @@ -7,6 +7,7 @@ import asyncio import inspect +import json import logging from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass @@ -36,6 +37,20 @@ _LOGGER = logging.getLogger(__name__) +_DIRECT_TERMINAL_ASSIGN_TIMEOUT_CAP_SECONDS = 30.0 +_DIRECT_TERMINAL_METADATA_POLL_INTERVAL_SECONDS = 0.05 +_DIRECT_TERMINAL_SESSION_TIMEOUT_MS = 45000 +_DIRECT_TERMINAL_HEARTBEAT_INTERVAL_MS = 15000 +_DIRECT_TERMINAL_MAX_POLL_INTERVAL_MS = 300000 + +# Bounded grace for a reply topic that EXISTS but is slow to surface partition +# metadata. A topic that has never been produced to (e.g. the FAILED terminal +# topic in a run where nothing fails) legitimately advertises no partitions; it +# must NOT burn the whole assign cap on every trial (OMN-13118 battery wedge). +# After this grace window of forced refreshes the broker's answer is taken as +# authoritative: assign whatever partitions exist (possibly none) and return. +_DIRECT_TERMINAL_PARTITIONLESS_GRACE_SECONDS = 2.0 + def _broker_group_id(command_topic: str) -> str: normalized = command_topic.replace(".", "-") @@ -68,6 +83,384 @@ class TerminalPayload: topic: str +@dataclass( + frozen=True, slots=True +) # internal-dataclass-ok: module-internal Kafka boundary handle +class DirectTerminalConsumer: + consumer: AIOKafkaConsumer + + +def _direct_terminal_bootstrap_servers(event_bus: object) -> str: + servers = getattr(event_bus, "_bootstrap_servers", None) + if not isinstance(servers, str) or not servers: + raise RuntimeError( + "terminal-event consumer: runtime event_bus exposes no string " + "_bootstrap_servers; cannot build a Kafka correlate consumer." + ) + return servers + + +def _direct_terminal_auth_kwargs(event_bus: object) -> dict[str, object]: + build_auth_kwargs = getattr(event_bus, "_build_auth_kwargs", None) + if not callable(build_auth_kwargs): + return {} + auth = cast("Callable[[], Mapping[str, object] | None]", build_auth_kwargs)() + return dict(auth or {}) + + +def _direct_terminal_client_version_kwargs(event_bus: object) -> dict[str, object]: + config = getattr(event_bus, "config", SimpleNamespace()) + api_version = getattr(config, "api_version", None) + if api_version is None: + return {} + try: + parameters = inspect.signature(AIOKafkaConsumer.__init__).parameters + except (TypeError, ValueError): + return {} + if "api_version" not in parameters: + return {} + return {"api_version": api_version} + + +def _extract_direct_terminal_correlation_id(payload: dict[str, object]) -> str | None: + value = payload.get("correlation_id") + if value is None: + return None + return str(value) + + +def _build_direct_terminal_consumer(event_bus: object) -> AIOKafkaConsumer: + config = getattr(event_bus, "config", SimpleNamespace()) + return AIOKafkaConsumer( + bootstrap_servers=_direct_terminal_bootstrap_servers(event_bus), + group_id=None, + enable_auto_commit=False, + auto_offset_reset="latest", + session_timeout_ms=getattr( + config, + "session_timeout_ms", + _DIRECT_TERMINAL_SESSION_TIMEOUT_MS, + ), + heartbeat_interval_ms=getattr( + config, + "heartbeat_interval_ms", + _DIRECT_TERMINAL_HEARTBEAT_INTERVAL_MS, + ), + max_poll_interval_ms=getattr( + config, + "max_poll_interval_ms", + _DIRECT_TERMINAL_MAX_POLL_INTERVAL_MS, + ), + retry_backoff_ms=getattr(config, "reconnect_backoff_ms", 2000), + **_direct_terminal_client_version_kwargs(event_bus), + **_direct_terminal_auth_kwargs(event_bus), + ) + + +async def _await_metadata_op(result: object) -> None: + """Await a client metadata operation result if it is awaitable. + + ``AIOKafkaClient.set_topics`` / ``force_metadata_update`` return a future + that resolves when the broker metadata fetch completes. Awaiting it is what + makes partition metadata actually surface; ignoring it (the prior bug) left + the assign loop spinning on a stale, empty partition view. Test fakes return + a plain ``bool``, so only await when the result is awaitable. + """ + if inspect.isawaitable(result): + await result + + +async def _force_terminal_topic_metadata( + consumer: AIOKafkaConsumer, + terminal_topic: str, +) -> None: + """Track the reply topic and force a fresh broker metadata fetch. + + ``set_topics`` only forces a metadata refresh when the requested topic set + DIFFERS from the currently-tracked set (aiokafka 0.13.0 + ``AIOKafkaClient.set_topics``). Re-calling ``set_topics([same_topic])`` each + loop iteration therefore took the no-refresh branch after the first call, so + a consumer whose first metadata fetch had not yet surfaced partitions never + re-fetched and burned the whole assign cap, raising a bare ``TimeoutError`` + (the empty-message ``wait failed`` seen live in OMN-13012). We register the + topic once, await that fetch, and on each subsequent miss force a refresh via + ``force_metadata_update`` (which always fetches) rather than the no-op + ``set_topics`` repeat. + """ + client = getattr(consumer, "_client", None) + if client is None: + return + set_topics = getattr(client, "set_topics", None) + if callable(set_topics): + await _await_metadata_op(set_topics([terminal_topic])) + + +async def _refresh_terminal_topic_metadata(consumer: AIOKafkaConsumer) -> None: + """Force a broker metadata fetch for the already-tracked reply topic.""" + client = getattr(consumer, "_client", None) + if client is None: + return + force_metadata_update = getattr(client, "force_metadata_update", None) + if callable(force_metadata_update): + await _await_metadata_op(force_metadata_update()) + + +async def _assign_direct_terminal_partitions( + consumer: AIOKafkaConsumer, + terminal_topic: str, + assign_cap_seconds: float, +) -> None: + """Track the reply topic, wait briefly for its metadata, then assign. + + The first ``set_topics`` registers the topic and forces the initial metadata + fetch (awaited). If partitions are still not visible (slow broker metadata), + each subsequent iteration forces a fresh fetch via ``force_metadata_update`` + — NOT a repeat ``set_topics([same_topic])``, which aiokafka treats as a + no-op and never re-fetches (the OMN-13012 consume-leg wedge: only one of the + two ephemeral terminal consumers ever surfaced partitions). + + Partition-less reply topics are a VALID steady state (OMN-13118) + -------------------------------------------------------------------- + A terminal topic that has never been produced to — the FAILED reply topic in + a run where every generation passes — legitimately advertises no partitions + (Redpanda auto-creates topics on first produce). Blocking the FULL + ``assign_cap_seconds`` (30s live) on it and then RAISING was the load- + dependent battery wedge: the runner opens one ephemeral consumer per terminal + topic BEFORE publishing, serially, so EVERY trial stalled ~30s on the + partition-less FAILED open before it could correlate the COMPLETED terminal — + a 160-trial battery needed >80 min and never completed. + + The fix: give a bounded grace window for a topic that EXISTS but is slow to + surface metadata; after it, take the broker's answer as authoritative and + assign whatever partitions exist — possibly none. A consumer with an empty + assignment is positioned (``seek_to_end`` is a no-op) and ``poll`` treats it + as "no terminal will arrive here" (returns None on its own timeout) without + blocking the open phase. The COMPLETED-topic session, opened separately and + assigned its real partition, wins the cross-topic race and correlates fast. + """ + loop = asyncio.get_running_loop() + grace = min(_DIRECT_TERMINAL_PARTITIONLESS_GRACE_SECONDS, assign_cap_seconds) + deadline = loop.time() + grace + await _force_terminal_topic_metadata(consumer, terminal_topic) + while True: + partitions = consumer.partitions_for_topic(terminal_topic) or set() + if partitions: + consumer.assign([TopicPartition(terminal_topic, p) for p in partitions]) + return + if loop.time() >= deadline: + # The broker has had the grace window to advertise partitions and + # surfaced none: the topic has not been produced to yet. Assign the + # empty set and return promptly rather than burning the cap. A real + # terminal on this topic would have created its partition before the + # correlated event was published, so a partition-less topic at this + # point carries no terminal for this correlation. + consumer.assign([]) + return + await asyncio.sleep(_DIRECT_TERMINAL_METADATA_POLL_INTERVAL_SECONDS) + await _refresh_terminal_topic_metadata(consumer) + + +async def _pin_direct_terminal_end_offsets( + consumer: AIOKafkaConsumer, + *, + timeout_seconds: float, +) -> None: + """Resolve the current end offsets SYNCHRONOUSLY and pin them via ``seek``. + + Why not ``seek_to_end`` (OMN-13118 strike-three) + ------------------------------------------------ + ``AIOKafkaConsumer.seek_to_end`` is LAZY: it calls + ``Fetcher.request_offset_reset(partitions, LATEST)``, which only marks the + partition "awaiting reset" and registers a position future. The actual LATEST + offset is resolved by a ``ListOffsets`` round-trip that happens on the FIRST + ``getone`` — which is AFTER the caller has published its command. With + generation completing in ~1s (dispatch loop freed, OMN-13010), the correlated + terminal lands in the gap between this open() and the first poll, so the + lazily-resolved LATEST position is the high-water-mark AFTER the record and + the poll starts reading PAST it. The terminal is never read, the trial never + correlates, and the experiment matrix re-fires the same cell forever — the + consume-leg wedge that survived PR #1969 (set_topics no-op) and PR #1970 + (partition-less assign-cap) because neither addressed the seek timing. + + ``end_offsets`` is a SYNCHRONOUS broker round-trip that returns the HWM + observed NOW (without changing the consumer position); ``seek`` then pins each + partition's read offset to that value deterministically. The resume position + is therefore fixed at open() time, BEFORE the publish, so any terminal emitted + at-or-after this offset is delivered to the subsequent poll. This is the real + subscribe-before-publish guarantee the two-phase protocol promised. + + An empty assignment (partition-less reply topic, OMN-13118) has no offsets to + resolve, so this is a no-op for that leg. + + ``end_offsets`` is a broker ``ListOffsets`` round-trip that aiokafka documents + as able to block indefinitely. It is bounded by ``timeout_seconds`` (the same + cap as ``start`` and partition assignment) so a stalled broker fails fast here + instead of hanging the pre-publish positioning and reintroducing a wait wedge + in a new spot. + """ + assignment = consumer.assignment() + if not assignment: + return + end_offsets = await asyncio.wait_for( + consumer.end_offsets(list(assignment)), + timeout=timeout_seconds, + ) + for topic_partition, offset in end_offsets.items(): + consumer.seek(topic_partition, offset) + + +async def open_direct_terminal_consumer( + *, + event_bus: object, + terminal_topic: str, +) -> DirectTerminalConsumer: + """Start, assign, and pin an ephemeral terminal consumer to the current end. + + The read position is captured SYNCHRONOUSLY via ``end_offsets`` + ``seek`` (not + the lazy ``seek_to_end``) so it is fixed at open() time, before the caller + publishes — the subscribe-before-publish guarantee that closes the + seek-past-the-terminal race (OMN-13118). + """ + consumer = _build_direct_terminal_consumer(event_bus) + try: + await asyncio.wait_for( + consumer.start(), + timeout=_DIRECT_TERMINAL_ASSIGN_TIMEOUT_CAP_SECONDS, + ) + await _assign_direct_terminal_partitions( + consumer, + terminal_topic, + _DIRECT_TERMINAL_ASSIGN_TIMEOUT_CAP_SECONDS, + ) + await _pin_direct_terminal_end_offsets( + consumer, + timeout_seconds=_DIRECT_TERMINAL_ASSIGN_TIMEOUT_CAP_SECONDS, + ) + except BaseException: + await close_direct_terminal_consumer( + DirectTerminalConsumer(consumer), + terminal_topic=terminal_topic, + log_prefix="terminal-event consumer open cleanup", + ) + raise + return DirectTerminalConsumer(consumer) + + +async def poll_direct_terminal_consumer( + *, + handle: DirectTerminalConsumer, + terminal_topic: str, + correlation_id: str, + timeout_seconds: float, +) -> dict[str, object] | None: + """Poll a positioned terminal consumer for the correlated terminal payload.""" + loop = asyncio.get_running_loop() + try: + # A partition-less reply topic assigns the empty set (OMN-13118): the + # broker never advertised a partition because the topic has not been + # produced to. ``getone()`` on an unassigned consumer raises + # ``IllegalStateError`` immediately; instead, treat this leg as "no + # terminal will arrive here" — sleep out the caller's timeout so the + # cross-topic race lets the COMPLETED-topic consumer win — and return + # None on genuine timeout. The losing leg is unblocked early when its + # session is closed by the race winner. + if not handle.consumer.assignment(): + try: + await asyncio.sleep(timeout_seconds) + except asyncio.CancelledError: + pass + return None + deadline = loop.time() + timeout_seconds + while True: + remaining = deadline - loop.time() + if remaining <= 0: + return None + try: + message = await asyncio.wait_for( + handle.consumer.getone(), + timeout=remaining, + ) + except TimeoutError: + return None + if message.value is None: + continue + try: + body: dict[str, object] = json.loads(message.value.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError): + continue + if _extract_direct_terminal_correlation_id(body) == correlation_id: + return body + finally: + await close_direct_terminal_consumer(handle, terminal_topic=terminal_topic) + + +async def _poll_terminal_without_close( + *, + handle: DirectTerminalConsumer, + terminal_topic: str, + correlation_id: str, + timeout_seconds: float, +) -> dict[str, object] | None: + """Poll one positioned consumer for the correlated terminal, WITHOUT closing. + + ``poll_direct_terminal_consumer`` stops its consumer in a ``finally`` because + its single-topic callers own one consumer for the whole open→wait→close + lifecycle. The Pattern B broker races MULTIPLE independent consumers and tears + them ALL down together after the race resolves (so the losing leg is unblocked + by ``close`` rather than waiting out its own timeout). This thin wrapper runs + the same correlate-and-wait body without the inner close so the broker keeps a + single teardown point. + """ + loop = asyncio.get_running_loop() + if not handle.consumer.assignment(): + # Partition-less reply topic (OMN-13118): nothing will arrive here. Sleep + # out the caller timeout so the race lets a real-partition topic win; the + # broker cancels this task the moment the winner correlates. + try: + await asyncio.sleep(timeout_seconds) + except asyncio.CancelledError: + pass + return None + deadline = loop.time() + timeout_seconds + while True: + remaining = deadline - loop.time() + if remaining <= 0: + return None + try: + message = await asyncio.wait_for( + handle.consumer.getone(), + timeout=remaining, + ) + except TimeoutError: + return None + if message.value is None: + continue + try: + body: dict[str, object] = json.loads(message.value.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError): + continue + if _extract_direct_terminal_correlation_id(body) == correlation_id: + return body + + +async def close_direct_terminal_consumer( + handle: DirectTerminalConsumer, + *, + terminal_topic: str, + log_prefix: str = "terminal-event consumer", +) -> None: + """Stop a direct terminal consumer, logging cleanup failures at the boundary.""" + try: + await handle.consumer.stop() + except Exception as exc: # noqa: BLE001 — boundary: best-effort cleanup + _LOGGER.warning( + "%s: failed to stop consumer for %s: %s", + log_prefix, + terminal_topic, + sanitize_error_message(exc), + ) + + class RuntimePatternBBroker: """Long-lived broker that normalizes worker terminal events for external CLIs.""" @@ -244,111 +637,117 @@ async def _dispatch_and_wait_with_direct_kafka_consumer( command: ModelDispatchBusCommand, route: ModelRuntimeLocalIngressRoute, ) -> TerminalPayload: + """Wait for the correlated terminal with ONE consumer PER terminal topic. + + Independent consumers, not one shared subscription (OMN-13128/13118 strike-5) + ---------------------------------------------------------------------------- + The generation consumer emits exactly one terminal per command — + ``completed`` (``contract_passed=True``) or ``failed`` + (``contract_passed=False``) — on DIFFERENT topics, so the broker must wait + on BOTH within the single per-command timeout. The prior shape assigned + BOTH topics' partitions to a SINGLE ephemeral ``group_id=None`` consumer. + A single aiokafka consumer holds ONE manual subscription + (``ManualSubscription`` over its assigned partitions); the live battery + diagnosis (HALT_K10_WEDGE_PERSISTS, strikes 3+4) showed that consumer's + COMPLETED-topic delivery window collapsing before it could surface the + correlated record, so the trial never correlated and the K>=10 matrix + re-fired cell 1 forever. + + The fix gives EACH terminal topic its OWN independent + ``open_direct_terminal_consumer`` — each started, assigned, and + offset-pinned (``end_offsets()`` + ``seek()``, #1971) at open() BEFORE the + worker command is published (subscribe-before-publish). The two waits then + run CONCURRENTLY via ``asyncio.wait(..., FIRST_COMPLETED)``; the first + correlated terminal wins and the losing wait is cancelled. No consumer ever + holds two subscriptions, so there is no subscription to tear down out from + under the COMPLETED delivery. + + The #1970 partition-less no-op and the #1971 synchronous offset pin are + preserved: they live in ``open_direct_terminal_consumer`` / + ``poll_direct_terminal_consumer``, which each consumer goes through. A + never-produced FAILED topic assigns the empty set, ``poll`` sleeps out its + timeout returning ``None``, and the COMPLETED consumer wins the race fast. + """ terminal_topics = _terminal_topics(route) if not terminal_topics: raise RuntimeError(f"Route '{command.command_name}' has no terminal event") - kafka_event_bus = self._kafka_event_bus() - config = getattr(kafka_event_bus, "config", SimpleNamespace()) - consumer = AIOKafkaConsumer( - bootstrap_servers=self._kafka_bootstrap_servers(), - group_id=None, - auto_offset_reset="latest", - enable_auto_commit=False, - session_timeout_ms=getattr(config, "session_timeout_ms", 45000), - heartbeat_interval_ms=getattr(config, "heartbeat_interval_ms", 15000), - max_poll_interval_ms=getattr(config, "max_poll_interval_ms", 300000), - retry_backoff_ms=getattr(config, "reconnect_backoff_ms", 2000), - **self._direct_kafka_client_version_kwargs(AIOKafkaConsumer), - **self._direct_kafka_auth_kwargs(), - ) - + event_bus = self._event_bus + handles: dict[str, DirectTerminalConsumer] = {} try: - await asyncio.wait_for( - consumer.start(), timeout=min(30, command.timeout_seconds) - ) - await self._assign_terminal_topic_partitions( - consumer, - terminal_topics, - command.timeout_seconds, - ) - await consumer.seek_to_end(*consumer.assignment()) + # Open one independent consumer per terminal topic, each positioned at + # the current end BEFORE we publish. Open serially so a partition-less + # FAILED topic's bounded grace does not race the COMPLETED open; both + # are positioned before any command is on the wire. + for terminal_topic in terminal_topics: + handles[terminal_topic] = await open_direct_terminal_consumer( + event_bus=event_bus, + terminal_topic=terminal_topic, + ) + await self._publish_worker_command(command, route) - deadline = asyncio.get_running_loop().time() + command.timeout_seconds - while True: - remaining = deadline - asyncio.get_running_loop().time() - if remaining <= 0: - raise TimeoutError - message = await asyncio.wait_for(consumer.getone(), timeout=remaining) - terminal_envelope = ModelEventEnvelope[object].model_validate_json( - message.value - ) - if terminal_envelope.correlation_id == command.correlation_id: - topic = getattr(message, "topic", None) - if not isinstance(topic, str) or not topic: - topic = str( - terminal_envelope.event_type or route.terminal_event - ) - return TerminalPayload( - payload=terminal_envelope.payload, - topic=topic, - ) + return await self._race_correlated_terminals( + handles, + correlation_id=str(command.correlation_id), + timeout_seconds=float(command.timeout_seconds), + ) finally: - try: - await consumer.stop() - except Exception as exc: # noqa: BLE001 - _LOGGER.warning( - "Failed to stop Pattern B terminal Kafka consumer: %s", - sanitize_error_message(exc), + for terminal_topic, handle in handles.items(): + await close_direct_terminal_consumer( + handle, + terminal_topic=terminal_topic, + log_prefix="Pattern B terminal Kafka consumer", ) - async def _assign_terminal_topic_partitions( + async def _race_correlated_terminals( self, - consumer: AIOKafkaConsumer, - topics: tuple[str, ...], - timeout_seconds: int, - ) -> None: - deadline = asyncio.get_running_loop().time() + min(30, timeout_seconds) - expected_topics = set(topics) - while True: - await self._refresh_terminal_topic_metadata(consumer, topics) - partitions: set[TopicPartition] = set() - ready_topics: set[str] = set() - for topic in topics: - topic_partitions = consumer.partitions_for_topic(topic) or set() - if topic_partitions: - ready_topics.add(topic) - partitions.update( - TopicPartition(topic, partition) for partition in topic_partitions + handles: Mapping[str, DirectTerminalConsumer], + *, + correlation_id: str, + timeout_seconds: float, + ) -> TerminalPayload: + """Poll every terminal consumer concurrently; first correlated wins. + + Each topic's poll runs as its own task. ``asyncio.wait(FIRST_COMPLETED)`` + returns as soon as one task finishes; a task that found the correlated + record yields the payload, a task that timed out yields ``None`` (and we + keep waiting on the rest until one correlates or all are exhausted). The + losing tasks are cancelled on the way out so the winner returns + immediately without joining a full-timeout sleep on the partition-less leg. + """ + tasks: dict[asyncio.Task[dict[str, object] | None], str] = {} + for terminal_topic, handle in handles.items(): + task = asyncio.ensure_future( + _poll_terminal_without_close( + handle=handle, + terminal_topic=terminal_topic, + correlation_id=correlation_id, + timeout_seconds=timeout_seconds, ) - if ready_topics == expected_topics: - consumer.assign(partitions) - return - if asyncio.get_running_loop().time() >= deadline: - raise TimeoutError - await asyncio.sleep(0.05) - - async def _refresh_terminal_topic_metadata( - self, - consumer: AIOKafkaConsumer, - topics: tuple[str, ...], - ) -> None: - client = getattr(consumer, "_client", None) - set_topics = getattr(client, "set_topics", None) - if callable(set_topics): - set_topics(list(topics)) - return - - add_topic = getattr(client, "add_topic", None) - if callable(add_topic): - for topic in topics: - add_topic(topic) - return + ) + tasks[task] = terminal_topic - topics_method = getattr(type(consumer), "topics", None) - if callable(topics_method): - await topics_method(consumer) + pending = set(tasks) + try: + while pending: + done, pending = await asyncio.wait( + pending, return_when=asyncio.FIRST_COMPLETED + ) + for task in done: + body = task.result() + if body is None: + continue + return TerminalPayload( + payload=body.get("payload", body), + topic=tasks[task], + ) + raise TimeoutError + finally: + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) def _supports_direct_kafka_terminal_consumer(self) -> bool: return ( @@ -357,44 +756,6 @@ def _supports_direct_kafka_terminal_consumer(self) -> bool: and hasattr(self._event_bus, "config") ) - def _direct_kafka_auth_kwargs(self) -> dict[str, object]: - build_auth_kwargs = getattr( # noqa: B009 - self._kafka_event_bus(), - "_build_auth_kwargs", - ) - auth_kwargs = cast( - "Callable[[], Mapping[str, object] | None]", - build_auth_kwargs, - )() - return dict(auth_kwargs or {}) - - def _direct_kafka_client_version_kwargs( - self, client_cls: type[object] - ) -> dict[str, object]: - config = getattr(self._kafka_event_bus(), "config", SimpleNamespace()) - api_version = getattr(config, "api_version", None) - if api_version is None: - return {} - try: - parameters = inspect.signature(client_cls.__init__).parameters - except (TypeError, ValueError): - return {} - if "api_version" not in parameters: - return {} - return {"api_version": api_version} - - def _kafka_bootstrap_servers(self) -> str: - bootstrap_servers = getattr( # noqa: B009 - self._kafka_event_bus(), - "_bootstrap_servers", - ) - if not isinstance(bootstrap_servers, str): - raise RuntimeError("Kafka event bus bootstrap servers are not configured") - return bootstrap_servers - - def _kafka_event_bus(self) -> object: - return self._event_bus - async def _publish_worker_command( self, command: ModelDispatchBusCommand, diff --git a/src/omnibase_infra/runtime/service_terminal_event_consumer.py b/src/omnibase_infra/runtime/service_terminal_event_consumer.py new file mode 100644 index 0000000000..99defc05d6 --- /dev/null +++ b/src/omnibase_infra/runtime/service_terminal_event_consumer.py @@ -0,0 +1,550 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Long-lived terminal-event correlator for auto-wired request/response EFFECT handlers. + +Some EFFECT handlers drive a request/response interaction over the bus: they +publish a command, then block until the correlated terminal event arrives on a +reply topic (correlating by ``correlation_id``), and read result fields back +from that terminal payload. ``HandlerContextRoiRunner`` is the motivating case +(OMN-13005): per (task x arm x trial) it publishes ``node-generation-requested.v1`` +and must wait for the correlated terminal — ``node-generation-completed.v1`` OR +``node-generation-failed.v1`` (OMN-13038) — before recording an attempt-reduction +row. + +Long-lived correlator (OMN-13118 Tier B redesign) +-------------------------------------------------- +The earlier implementation built a brand-new OS thread + asyncio event loop + +``AIOKafkaConsumer`` + ``AIOKafkaClient`` (full bootstrap / metadata fetch / +coordination) for EVERY (trial x terminal-topic) — up to 80 churns for a 40-cell +run, 320+ for the 160-battery — and tore each down right after, racing a ~1s +generation. Five offset/subscription patches (#1969 set_topics no-op, #1970 +partition-less assign, #1971 seek-pin, #1972 independent consumers) tuned that +per-trial-ephemeral substrate and all failed the live K>=10 battery gate with the +identical wedge signature. The defect class is the ephemerality itself: a +brand-new client per trial races its own bootstrap and teardown against the fast +terminal, so the COMPLETED leg silently never delivers while only the empty-set +FAILED leg times out at the 30s assign cap. (See +``docs/evidence/2026-06-12-weekend-pass/experiments/exp1/B1_M1_M2_PROBE_RESULT.md``.) + +This module owns ONE runtime-lifetime consumer subscribed ONCE to BOTH terminal +topics, running a SINGLE poll loop on a SINGLE thread+loop for the whole batch. +It demuxes every terminal to the waiting trial by ``correlation_id`` via a +``correlation_id -> Future`` registry: a trial registers its ``correlation_id`` +BEFORE publishing its command (preserving the subscribe-before-publish guarantee +#1971/OMN-13012 established at the consumer level), then awaits its future with +the trial timeout. There is no per-trial consumer / assign / pin / thread / loop / +client to race — the entire timing-bug class is structurally removed, not +re-tuned. This mirrors the already-working long-lived ``node_generation_consumer`` +and ``RuntimePatternBBroker.start()`` command subscription. + +Architecture (ARCH-002 — "runtime owns all Kafka plumbing"): + - Nodes/handlers declare the consume requirement in their contract and call the + injected ``event_consumer``; they never touch the Kafka client. + - ``TerminalEventConsumer`` is the contract-declared consume dependency + materialized by the runtime auto-wiring (``handler_wiring.py``) through + ``make_terminal_event_consumer``. Its ``.open(topic)`` returns a + ``TerminalConsumerSession`` bound to the shared long-lived correlator; + ``session.wait(correlation_id, timeout)`` registers the cid and blocks on the + correlated future. The handler keeps the same two-phase + open-before-publish / wait-after-publish protocol. + +Loop isolation (why a worker thread): + The auto-wired dispatch callback invokes a sync handler ``handle()`` directly + on the runtime's running event loop. A blocking Kafka correlate cannot run on + that same loop (it would deadlock, and ``asyncio.run()`` raises inside a + running loop). The correlator therefore owns ONE dedicated event loop on ONE + daemon thread for the whole process lifetime; the sync ``wait`` blocks the + calling thread on a future resolved by the correlator's poll loop on that loop. + The runtime dispatch loop stays free to deliver the very terminals the + correlator reads. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import threading +from collections.abc import Coroutine, Iterable +from concurrent.futures import Future +from types import TracebackType +from typing import cast + +from aiokafka import AIOKafkaConsumer, TopicPartition + +from omnibase_infra.runtime.service_pattern_b_broker import ( + _build_direct_terminal_consumer, + _extract_direct_terminal_correlation_id, + _refresh_terminal_topic_metadata, +) +from omnibase_infra.utils.util_error_sanitization import sanitize_error_message + +logger = logging.getLogger(__name__) + +# (terminal_topic, correlation_id, timeout_seconds) -> deserialized payload | None +EventConsumer = object + +# Bound the one-time subscription/partition-assignment phase so a broker that +# never surfaces a terminal topic fails fast instead of hanging the first open. +_SUBSCRIBE_TIMEOUT_CAP_SECONDS = 30.0 +_METADATA_POLL_INTERVAL_SECONDS = 0.05 + +# Grace beyond a submitted coroutine's own timeout so a clean shutdown completes +# before the calling thread gives up on the worker loop. +_LOOP_SUBMIT_GRACE_SECONDS = 5.0 + +# Per-iteration poll wait: short so the loop notices cancellation/closing +# promptly while still batching broker fetches. +_POLL_GETONE_TIMEOUT_SECONDS = 1.0 + +# Cap the unmatched-terminal registry so a long-running batch that reads +# terminals for correlation_ids no live trial is waiting on (late deliveries, +# other runs) cannot grow without bound. Mirrors the bounded ``on_terminal`` +# guard in ``RuntimePatternBBroker``. +_MAX_PENDING_UNMATCHED = 4096 + + +class LongLivedTerminalCorrelator: + """ONE consumer, subscribed once to all terminal topics, demuxing by cid. + + Owns a single event loop on a single daemon thread for its whole lifetime. + Lazily starts ONE ``AIOKafkaConsumer`` on first use, assigns the partitions + of every requested terminal topic, pins each to its current end offset, and + runs a single poll loop that resolves the per-correlation futures as + terminals arrive. Adding a terminal topic re-resolves the assignment to the + union and pins the new partitions to their current end — done ONCE per topic, + not per trial. + + Thread-safety: all Kafka state lives on the worker loop. Public methods + (``register``, ``ensure_topic``, ``wait``) submit coroutines to that loop and + block the calling thread on the result, so they are safe to call from the + sync handler thread. + """ + + def __init__(self, *, event_bus: object, handler_name: str) -> None: + self._event_bus = event_bus + self._handler_name = handler_name + self._loop = asyncio.new_event_loop() + self._closed = False + self._consumer: AIOKafkaConsumer | None = None + self._subscribed_topics: set[str] = set() + # correlation_id -> future resolved with the terminal payload (or None). + self._pending: dict[str, asyncio.Future[dict[str, object] | None]] = {} + self._poll_task: asyncio.Task[None] | None = None + self._start_lock = asyncio.Lock() + self._thread = threading.Thread( + target=self._run_loop, + name=f"terminal-correlator-{handler_name}", + daemon=True, + ) + self._thread.start() + + # -- worker-loop plumbing ------------------------------------------------ + + def _run_loop(self) -> None: + asyncio.set_event_loop(self._loop) + self._loop.run_forever() + + def _submit( + self, + coro: Coroutine[object, object, object], + *, + timeout: float, + ) -> object: + future: Future[object] = asyncio.run_coroutine_threadsafe(coro, self._loop) + return future.result(timeout=timeout) + + # -- one-time consumer lifecycle (on the worker loop) -------------------- + + async def _ensure_consumer_started(self) -> None: + if self._consumer is not None: + return + consumer = _build_direct_terminal_consumer(self._event_bus) + await asyncio.wait_for(consumer.start(), timeout=_SUBSCRIBE_TIMEOUT_CAP_SECONDS) + self._consumer = consumer + self._poll_task = self._loop.create_task(self._poll_forever()) + + async def _ensure_topic_assigned(self, terminal_topic: str) -> None: + """Add a terminal topic to the single consumer's assignment ONCE. + + Resolves the partitions for ``terminal_topic`` (a bounded metadata grace + for a topic that exists but is slow to surface; an empty set for a reply + topic never produced to, e.g. the FAILED topic in an all-pass run is a + valid steady state), unions them into the existing assignment, and pins + the NEW partitions to their current end offset so a terminal published + after this call is delivered. Idempotent per topic. + """ + async with self._start_lock: + await self._ensure_consumer_started() + if terminal_topic in self._subscribed_topics: + return + consumer = self._consumer + assert consumer is not None + new_partitions = await self._resolve_partitions(consumer, terminal_topic) + existing = set(consumer.assignment()) + union = existing.union(new_partitions) + if union: + consumer.assign(list(union)) + # Pin only the newly added partitions to their current end so the + # already-positioned topics keep their continuously-advancing cursor + # (one consumer, one monotonic position per partition). + if new_partitions: + end_offsets = await asyncio.wait_for( + consumer.end_offsets(list(new_partitions)), + timeout=_SUBSCRIBE_TIMEOUT_CAP_SECONDS, + ) + for topic_partition, offset in end_offsets.items(): + consumer.seek(topic_partition, offset) + self._subscribed_topics.add(terminal_topic) + + async def _resolve_partitions( + self, consumer: AIOKafkaConsumer, terminal_topic: str + ) -> list[TopicPartition]: + loop = asyncio.get_running_loop() + deadline = loop.time() + _SUBSCRIBE_TIMEOUT_CAP_SECONDS + client = getattr(consumer, "_client", None) + set_topics = getattr(client, "set_topics", None) if client else None + if callable(set_topics): + result = set_topics([*self._subscribed_topics, terminal_topic]) + if asyncio.iscoroutine(result) or asyncio.isfuture(result): + await result + await _refresh_terminal_topic_metadata(consumer) + while True: + partitions = consumer.partitions_for_topic(terminal_topic) or set() + if partitions: + return [TopicPartition(terminal_topic, p) for p in partitions] + if loop.time() >= deadline: + # A reply topic with no partitions has never been produced to — + # a valid steady state (OMN-13118). No terminal for any live + # correlation can be on it, so contribute no partitions; the + # other terminal topic carries the correlated record. + return [] + await asyncio.sleep(_METADATA_POLL_INTERVAL_SECONDS) + await _refresh_terminal_topic_metadata(consumer) + + # -- single poll loop: demux terminals to waiting trials ----------------- + + async def _poll_forever(self) -> None: + consumer = self._consumer + assert consumer is not None + while not self._closed: + try: + if not consumer.assignment(): + await asyncio.sleep(_METADATA_POLL_INTERVAL_SECONDS) + continue + message = await asyncio.wait_for( + consumer.getone(), timeout=_POLL_GETONE_TIMEOUT_SECONDS + ) + except TimeoutError: + continue + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 — poll boundary: log + continue + logger.warning( + "terminal-event correlator: poll error for %s: %s", + self._handler_name, + sanitize_error_message(exc), + ) + await asyncio.sleep(_METADATA_POLL_INTERVAL_SECONDS) + continue + self._deliver(message) + + def _deliver(self, message: object) -> None: + value = getattr(message, "value", None) + if value is None: + return + try: + body: dict[str, object] = json.loads(value.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError, AttributeError): + return + correlation_id = _extract_direct_terminal_correlation_id(body) + if correlation_id is None: + return + # OMN-13118 (P_RESOLVE_BEFORE_AWAIT fix): resolve the future IN PLACE; do + # NOT pop on match. The always-on poll loop typically delivers the + # terminal before the runner's wait() attaches (publish -> generation -> + # wait latency, ~28s observed live), so popping on match discarded the + # resolved result and the late awaiter found nothing -> 120s timeout -> + # zero rows. Leaving the resolved future in the registry lets a late + # _wait read it; _wait pops on exit. register() (pre-publish) guarantees + # the future exists when this matches. Idempotent under a duplicate + # delivery via the future.done() guard. + future = self._pending.get(correlation_id) + if future is None: + # Unmatched terminal (late, or a correlation no live trial awaits). + # Drop it — the correlator is fire-and-forget for unawaited cids. + return + if not future.done(): + future.set_result(body) + + # -- public API (called from the sync handler thread) -------------------- + + def ensure_topic(self, terminal_topic: str) -> None: + """Subscribe the single consumer to ``terminal_topic`` (once, blocking).""" + if self._closed: + return + self._submit( + self._ensure_topic_assigned(terminal_topic), + timeout=_SUBSCRIBE_TIMEOUT_CAP_SECONDS + _LOOP_SUBMIT_GRACE_SECONDS, + ) + + def register(self, correlation_id: str) -> None: + """Register a correlation_id's future BEFORE the caller publishes. + + Subscribe-before-publish at the correlation level: the future exists in + the registry before the command is on the wire, so a terminal that lands + immediately after publish is delivered to it by the always-running poll + loop (no per-trial open/assign/pin to race). + """ + if self._closed: + return + self._submit(self._register(correlation_id), timeout=_LOOP_SUBMIT_GRACE_SECONDS) + + async def _register(self, correlation_id: str) -> None: + if len(self._pending) >= _MAX_PENDING_UNMATCHED: + # Bounded registry: evict the oldest unresolved future rather than + # grow without bound. A live trial re-registers on its next wait. + stale_cid, stale_future = next(iter(self._pending.items())) + self._pending.pop(stale_cid, None) + if not stale_future.done(): + stale_future.set_result(None) + if correlation_id not in self._pending: + self._pending[correlation_id] = self._loop.create_future() + + def wait( + self, correlation_id: str, timeout_seconds: float + ) -> dict[str, object] | None: + """Block up to ``timeout_seconds`` for the correlated terminal. + + The future was created by ``register`` (pre-publish). Blocks the calling + thread on it via the worker loop, returning the matched payload or + ``None`` on timeout. On timeout the registry entry is dropped so a late + delivery is not mismatched to a future trial. + """ + if self._closed: + return None + try: + result = self._submit( + self._wait(correlation_id, timeout_seconds), + timeout=timeout_seconds + _LOOP_SUBMIT_GRACE_SECONDS, + ) + except BaseException as exc: # noqa: BLE001 — re-raised as None to caller + logger.warning( + "terminal-event correlator: wait failed for %s (cid=%s): %s", + self._handler_name, + correlation_id, + sanitize_error_message(exc) + if isinstance(exc, Exception) + else type(exc).__name__, + ) + return None + return cast("dict[str, object] | None", result) + + async def _wait( + self, correlation_id: str, timeout_seconds: float + ) -> dict[str, object] | None: + future = self._pending.get(correlation_id) + if future is None: + # Defensive: caller skipped register(); create it now (race-prone but + # never worse than the legacy single-call path). + future = self._loop.create_future() + self._pending[correlation_id] = future + try: + return await asyncio.wait_for( + asyncio.shield(future), timeout=timeout_seconds + ) + except TimeoutError: + return None + finally: + # OMN-13118 (P_RESOLVE_BEFORE_AWAIT fix): the awaiter owns removal now + # that _deliver resolves the future in place (no pop-on-match). Pop on + # BOTH success and timeout so a resolved-and-read future does not + # linger in the registry. Idempotent if two waiters share the cid. + self._pending.pop(correlation_id, None) + + def close(self) -> None: + """Stop the poll loop, the consumer, and the worker loop. Idempotent.""" + if self._closed: + return + self._closed = True + try: + stop_future: Future[object] = asyncio.run_coroutine_threadsafe( + self._shutdown(), self._loop + ) + stop_future.result(timeout=_LOOP_SUBMIT_GRACE_SECONDS) + except Exception as exc: # noqa: BLE001 — boundary: best-effort cleanup + logger.warning( + "terminal-event correlator: shutdown raced for %s: %s", + self._handler_name, + sanitize_error_message(exc), + ) + self._loop.call_soon_threadsafe(self._loop.stop) + self._thread.join(timeout=_LOOP_SUBMIT_GRACE_SECONDS) + try: + self._loop.close() + except Exception as exc: # noqa: BLE001 — boundary: best-effort cleanup + logger.debug( + "terminal-event correlator: loop close raced for %s: %s", + self._handler_name, + exc, + ) + + async def _shutdown(self) -> None: + if self._poll_task is not None: + self._poll_task.cancel() + try: + await self._poll_task + except asyncio.CancelledError: + pass + except Exception: # noqa: BLE001 — boundary: best-effort cleanup + pass + for future in self._pending.values(): + if not future.done(): + future.set_result(None) + self._pending.clear() + if self._consumer is not None: + try: + await self._consumer.stop() + except Exception as exc: # noqa: BLE001 — boundary: best-effort cleanup + logger.warning( + "terminal-event correlator: failed to stop consumer for %s: %s", + self._handler_name, + sanitize_error_message(exc), + ) + self._consumer = None + + +class TerminalConsumerSession: + """Two-phase handle over ONE terminal topic, backed by the shared correlator. + + ``open`` ensures the shared long-lived consumer is subscribed to the topic + (a no-op after the first trial) and registers the caller's correlation_id + BEFORE publish. ``wait`` blocks on the correlated future the always-running + poll loop resolves. No per-trial thread / loop / consumer / assign / pin. + + Usage (subscribe-before-publish):: + + session = consumer.open(terminal_topic) # ensure topic subscribed + session.register(correlation_id) # register BEFORE publish + publisher(command_topic, payload) # publish AFTER positioning + result = session.wait(correlation_id, timeout) + """ + + def __init__( + self, + *, + correlator: LongLivedTerminalCorrelator, + terminal_topic: str, + ) -> None: + self._correlator = correlator + self._terminal_topic = terminal_topic + self._registered: set[str] = set() + + def open(self) -> TerminalConsumerSession: + """Ensure the shared consumer is subscribed to this terminal topic.""" + self._correlator.ensure_topic(self._terminal_topic) + return self + + def wait( + self, correlation_id: str, timeout_seconds: float + ) -> dict[str, object] | None: + """Register the cid (if not yet) and block for the correlated terminal.""" + self.register(correlation_id) + return self._correlator.wait(correlation_id, timeout_seconds) + + def register(self, correlation_id: str) -> None: + """Register the cid BEFORE publish (explicit subscribe-before-publish).""" + if correlation_id not in self._registered: + self._correlator.register(correlation_id) + self._registered.add(correlation_id) + + def close(self) -> None: + """No-op: the shared correlator outlives any single session. + + The per-trial session owns no resources of its own — the consumer, + thread, and loop are the correlator's and live for the whole batch. + Closing a session must NOT tear the shared correlator down (that is the + teardown-races-fetch class the redesign eliminates). + """ + return + + def __enter__(self) -> TerminalConsumerSession: + return self.open() + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + self.close() + + +class TerminalEventConsumer: + """Callable injected as ``event_consumer`` — backed by ONE long-lived correlator. + + ``.open(topic)`` returns a :class:`TerminalConsumerSession` for + subscribe-before-publish callers. Calling the instance directly preserves the + legacy ``(topic, cid, timeout) -> dict | None`` shape (open + register + wait + in one step). Both route through the single shared correlator — there is no + per-call consumer, thread, or loop. + """ + + def __init__(self, *, event_bus: object, handler_name: str) -> None: + self._event_bus = event_bus + self._handler_name = handler_name + self._correlator = LongLivedTerminalCorrelator( + event_bus=event_bus, handler_name=handler_name + ) + + def open(self, terminal_topic: str) -> TerminalConsumerSession: + """Open a session bound to the shared correlator (ensures topic at open).""" + session = TerminalConsumerSession( + correlator=self._correlator, + terminal_topic=terminal_topic, + ) + return session.open() + + def __call__( + self, terminal_topic: str, correlation_id: str, timeout_seconds: float + ) -> dict[str, object] | None: + """Single-call form: ensure topic + register + wait in one step.""" + session = self.open(terminal_topic) + session.register(correlation_id) + return session.wait(correlation_id, timeout_seconds) + + def topics(self) -> Iterable[str]: + return tuple(self._correlator._subscribed_topics) + + def close(self) -> None: + """Tear down the shared correlator (runtime shutdown / test cleanup).""" + self._correlator.close() + + +def make_terminal_event_consumer( + *, + event_bus: object, + handler_name: str, +) -> TerminalEventConsumer: + """Build the ``event_consumer`` injected into request/response handlers. + + The returned object is directly callable with the legacy single-call shape + ``(terminal_topic, correlation_id, timeout_seconds) -> dict | None`` and also + exposes ``.open(topic) -> TerminalConsumerSession`` for the two-phase + (subscribe-before-publish) protocol. Both forms route through ONE long-lived + correlator (one consumer subscribed once to all terminal topics, single poll + loop, demux by correlation_id) running on a dedicated event loop in a worker + thread, so blocking does not deadlock the runtime dispatch loop that delivers + the awaited terminals — and there is no per-trial consumer/thread/loop to + race (OMN-13118 Tier B). + """ + return TerminalEventConsumer(event_bus=event_bus, handler_name=handler_name) + + +__all__: list[str] = [ + "EventConsumer", + "LongLivedTerminalCorrelator", + "TerminalConsumerSession", + "TerminalEventConsumer", + "make_terminal_event_consumer", +] diff --git a/src/omnibase_infra/services/observability/savings_estimation/consumer.py b/src/omnibase_infra/services/observability/savings_estimation/consumer.py index 6d2d822e98..846b31c6d6 100644 --- a/src/omnibase_infra/services/observability/savings_estimation/consumer.py +++ b/src/omnibase_infra/services/observability/savings_estimation/consumer.py @@ -32,6 +32,7 @@ from __future__ import annotations +import json import logging import time from collections import OrderedDict @@ -39,6 +40,7 @@ from enum import StrEnum from uuid import UUID +from omnibase_infra.event_bus.models.model_event_message import ModelEventMessage from omnibase_infra.nodes.node_savings_estimation_compute.handlers.handler_savings_estimation import ( HandlerSavingsEstimation, ) @@ -456,6 +458,41 @@ def _ingest_dispatch_eval(buf: SessionBuffer, payload: dict[str, object]) -> Non buf.outcome_received_at = time.monotonic() +def decode_event_message(message: ModelEventMessage) -> tuple[str, dict[str, object]]: + """Decode a typed event-bus message into a (topic, payload) pair. + + Both the Kafka and in-memory event buses deliver a + :class:`ModelEventMessage` to a consumer's ``on_message`` callback — never a + raw ``dict`` or ``str``. The message body is the JSON payload carried in the + typed ``value`` field (bytes). This decodes that field directly off the + typed model. It does NOT call ``.get()`` on the message, which has no such + method — the pre-fix wiring did, raising + ``AttributeError: 'ModelEventMessage' object has no attribute 'get'`` + (OMN-13149). + + Args: + message: The typed event-bus message delivered by the consumer callback. + ``message.topic`` is the correlation topic and ``message.value`` is + the JSON-encoded payload. + + Returns: + A ``(topic, payload)`` pair ready for + :meth:`ServiceSavingsEstimator.ingest_event`. + + Raises: + TypeError: If the decoded payload is not a JSON object. The savings + correlation logic requires a mapping keyed by ``session_id``; a + non-object payload is a contract violation and fails fast. + """ + payload = json.loads(message.value) + if not isinstance(payload, dict): + raise TypeError( + "savings estimation payload must be a JSON object, " + f"got {type(payload).__name__} on topic {message.topic!r}" + ) + return message.topic, payload + + class ServiceSavingsEstimator: """Session-event correlator producing savings-estimated.v1 events. @@ -737,4 +774,4 @@ def _mark_finalized(self, session_id: str) -> None: self._finalized[session_id] = True -__all__: list[str] = ["ServiceSavingsEstimator"] +__all__: list[str] = ["ServiceSavingsEstimator", "decode_event_message"] diff --git a/src/omnibase_infra/validation/validation_exemptions.yaml b/src/omnibase_infra/validation/validation_exemptions.yaml index 3b124febb9..6d0468d6a6 100644 --- a/src/omnibase_infra/validation/validation_exemptions.yaml +++ b/src/omnibase_infra/validation/validation_exemptions.yaml @@ -324,6 +324,51 @@ pattern_exemptions: documentation: - OMN-11546 concrete-node audit ticket: OMN-11570 + - file_pattern: 'cli_delegate\.py' + method_pattern: "Function 'delegate_command'" + violation_pattern: 'has \d+ parameters' + reason: > + Click injects command arguments and options directly into the command callback (same contract as run_node_by_name). The callback immediately forwards those values to run_delegate; replacing the signature with an intermediate model would obscure the Click contract without reducing runtime complexity. + + documentation: + - OMN-13096 onex delegate single-command rewrite + ticket: OMN-13096 + # ========================================================================== + # onex skill subcommand Exemptions (OMN-13097) + # ========================================================================== + # Same Click-callback pattern as run_node_by_name above: Click injects the + # skill name plus dispatch options directly into the command callback, which + # immediately delegates to run_receipt_mode. An intermediate options model + # would obscure the Click contract without reducing complexity. + - file_pattern: 'cli_skill\.py' + method_pattern: "Function 'run_skill_by_name'" + violation_pattern: 'has \d+ parameters' + reason: > + Click injects command arguments and options directly into the command callback (identical pattern to cli_node.py run_node_by_name). The body normalizes them and delegates to run_receipt_mode; an intermediate model would obscure the Click contract without reducing complexity. + + documentation: + - docs/plans/2026-06-12-skill-output-suppression-plan.md (Phase 4) + ticket: OMN-13097 + # skill_name / node_name in the declarative skill->node mapping are literal + # identifiers (the skill name as invoked, the onex.nodes entry-point name), + # not foreign-key references to entities with separate ID + display_name. No + # such entity exists in this domain — the mapping IS the registry. + - file_pattern: 'model_skill_mapping\.py' + violation_pattern: "Field 'skill_name' might reference an entity" + reason: > + skill_name is the literal skill identifier as invoked ('onex skill '), not a foreign key to a skill entity. It is the registry key itself; no separate skill entity with ID + display_name exists. + + documentation: + - docs/plans/2026-06-12-skill-output-suppression-plan.md (Phase 4) + ticket: OMN-13097 + - file_pattern: 'model_skill_mapping\.py' + violation_pattern: "Field 'node_name' might reference an entity" + reason: > + node_name is the onex.nodes entry-point identifier resolved exactly like 'onex node ', used to locate the packaged contract. It is a routing identifier, not a foreign key to a node entity with ID + display_name. + + documentation: + - docs/plans/2026-06-12-skill-output-suppression-plan.md (Phase 4) + ticket: OMN-13097 # ========================================================================== # KafkaContractCache Exemptions (OMN-1989) # ========================================================================== @@ -727,6 +772,19 @@ pattern_exemptions: - CLAUDE.md (Registry Naming Conventions) ticket: OMN-1139 # ========================================================================== + # ModelConfigProvenance config_name Exemption (OMN-12958) + # ========================================================================== + # config_name is the logical config identifier string (e.g., "bifrost_delegation"), + # not an entity reference requiring ID + display_name. + - file_pattern: 'config_provenance\.py' + violation_pattern: "Field 'config_name' might reference an entity" + reason: > + config_name is a logical config identifier string (e.g., "bifrost_delegation") used to label which runtime-rendered contract a provenance record describes. It is not an entity reference and does not require the ID + display_name pattern. + + documentation: + - CLAUDE.md (Registry Naming Conventions) + ticket: OMN-12958 + # ========================================================================== # ModelIntrospectionConfig node_name Exemption (omnibase-core 0.14.0 upgrade) # ========================================================================== # node_name is a canonical identifier for consumer group derivation, not an entity reference. diff --git a/tests/ci/test_bifrost_lane_overlay.py b/tests/ci/test_bifrost_lane_overlay.py new file mode 100644 index 0000000000..055fb4147a --- /dev/null +++ b/tests/ci/test_bifrost_lane_overlay.py @@ -0,0 +1,160 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""CI gate: Bifrost lane overlay env sidecar is in sync with the YAML source. + +OMN-12864: docker/lane-overlays/dev.bifrost.env must be the rendered form of +docker/lane-overlays/dev.bifrost.yaml. This test fails when the sidecar is +stale, preventing a deploy that would silently use wrong endpoint bindings. + +Re-render with: + uv run python scripts/render_bifrost_lane_overlay_env.py +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +OVERLAY_YAML = ROOT / "docker" / "lane-overlays" / "dev.bifrost.yaml" +OVERLAY_ENV = ROOT / "docker" / "lane-overlays" / "dev.bifrost.env" +RENDER_SCRIPT = ROOT / "scripts" / "render_bifrost_lane_overlay_env.py" + +pytestmark = pytest.mark.unit + +# Insert src/ into path so imports work without an installed package. +sys.path.insert(0, str(ROOT / "src")) + +from omnibase_infra.runtime.models.model_bifrost_lane_overlay import ( + ModelBifrostLaneOverlay, +) + + +def _load_overlay() -> ModelBifrostLaneOverlay: + raw = yaml.safe_load(OVERLAY_YAML.read_text(encoding="utf-8")) + return ModelBifrostLaneOverlay.model_validate(raw) + + +def _parse_env_file(path: Path) -> dict[str, str]: + env: dict[str, str] = {} + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + env[key.strip()] = value.strip() + return env + + +def test_overlay_yaml_loads_and_validates() -> None: + """The dev.bifrost.yaml is valid and parses without errors.""" + overlay = _load_overlay() + assert overlay.lane == "dev" + assert overlay.coder_endpoint_url.endswith("/v1/chat/completions") + assert overlay.reasoner_endpoint_url.endswith("/v1/chat/completions") + assert overlay.embedding_endpoint_url.endswith("/v1/chat/completions") + assert overlay.ds4_flash_endpoint_url.endswith("/v1/chat/completions") + + +def test_overlay_env_sidecar_exists() -> None: + """The rendered dev.bifrost.env sidecar must be committed alongside the YAML.""" + assert OVERLAY_ENV.exists(), ( + f"Missing rendered sidecar: {OVERLAY_ENV}\n" + "Re-render with: uv run python scripts/render_bifrost_lane_overlay_env.py" + ) + + +def test_overlay_env_sidecar_in_sync_with_yaml() -> None: + """The dev.bifrost.env sidecar must match what the YAML would render. + + Fails when the YAML source was edited without re-rendering the sidecar. + Fix: uv run python scripts/render_bifrost_lane_overlay_env.py + """ + overlay = _load_overlay() + expected_env = overlay.as_env_dict() + + actual_env = _parse_env_file(OVERLAY_ENV) + + for key, expected_value in expected_env.items(): + assert key in actual_env, ( + f"Missing key {key!r} in {OVERLAY_ENV}. " + "Re-render: uv run python scripts/render_bifrost_lane_overlay_env.py" + ) + assert actual_env[key] == expected_value, ( + f"{key}: env sidecar has {actual_env[key]!r}, YAML source wants {expected_value!r}. " + "Re-render: uv run python scripts/render_bifrost_lane_overlay_env.py" + ) + + +def test_overlay_env_has_all_four_bifrost_keys() -> None: + """The sidecar must declare all four BIFROST_LOCAL_* keys.""" + env = _parse_env_file(OVERLAY_ENV) + required_keys = { + "BIFROST_LOCAL_CODER_ENDPOINT_URL", + "BIFROST_LOCAL_REASONER_ENDPOINT_URL", + "BIFROST_LOCAL_EMBEDDING_ENDPOINT_URL", + "BIFROST_LOCAL_DS_V4_FLASH_ENDPOINT_URL", + } + missing = required_keys - set(env.keys()) + assert not missing, ( + f"Missing keys in {OVERLAY_ENV}: {sorted(missing)}. " + "Re-render: uv run python scripts/render_bifrost_lane_overlay_env.py" + ) + + +def test_render_script_exists() -> None: + """The render script must exist so operators can regenerate the sidecar.""" + assert RENDER_SCRIPT.exists(), f"Missing render script: {RENDER_SCRIPT}" + + +COMPOSE_INFRA = ROOT / "docker" / "docker-compose.infra.yml" +_RUNTIME_SERVICES_NEEDING_BIFROST = frozenset( + {"omninode-runtime", "runtime-effects", "runtime-worker"} +) + + +def test_compose_env_file_is_service_level_not_top_level() -> None: + """OMN-12864: dev.bifrost.env must be wired as a service-level env_file, not top-level. + + Docker Compose v2 schema rejects 'env_file' as an additional property at the + top level (outside of 'services'). The committed sidecar must be wired + per-service so compose config renders cleanly in CI. + """ + import yaml + + text = COMPOSE_INFRA.read_text(encoding="utf-8") + data = yaml.safe_load(text) + + # Top-level env_file is schema-invalid in compose v2 — must be absent. + assert "env_file" not in data, ( + "docker-compose.infra.yml has a top-level 'env_file' key which is schema-invalid " + "in Docker Compose v2. Move it to service-level env_file blocks." + ) + + +def test_runtime_services_have_bifrost_env_file() -> None: + """OMN-12864: Runtime services that render Bifrost must declare the committed env_file. + + This ensures BIFROST_LOCAL_* vars flow into the three runtime containers from + the committed overlay, not from ephemeral shell exports. + """ + import yaml + + data = yaml.safe_load(COMPOSE_INFRA.read_text(encoding="utf-8")) + services = data.get("services", {}) + + for svc in _RUNTIME_SERVICES_NEEDING_BIFROST: + assert svc in services, f"Service {svc!r} not found in compose file" + svc_cfg = services[svc] + env_files = svc_cfg.get("env_file", []) + assert isinstance(env_files, list) and env_files, ( + f"Service {svc!r} has no env_file block — " + "BIFROST_LOCAL_* overlay won't reach this container" + ) + paths = [(e["path"] if isinstance(e, dict) else e) for e in env_files] + assert any("dev.bifrost.env" in p for p in paths), ( + f"Service {svc!r} env_file does not include dev.bifrost.env. Found: {paths}" + ) diff --git a/tests/ci/test_dispatcher_route_coverage_gate.py b/tests/ci/test_dispatcher_route_coverage_gate.py new file mode 100644 index 0000000000..8be4fc2ef2 --- /dev/null +++ b/tests/ci/test_dispatcher_route_coverage_gate.py @@ -0,0 +1,474 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""CI gate tests: dispatcher route coverage (OMN-12858, OMN-12879, OMN-12880). + +Verifies the check_dispatcher_route_coverage gate logic using synthetic +in-memory contracts via the gate's check_route_coverage() API. + +No Kafka, no DB, no real handler imports — pure static contract analysis. + +Test cases: + RED — contract subscribes a command topic but declares no handler_routing + or runtime_dispatch => gate FAILS with the unrouted topic in failures + GREEN — contract subscribes a command topic and declares handler_routing + => gate PASSES (0 failures) + COMPAT — compatibility_publish_topics are sender-side only; the gate MUST + NOT flag them as unrouted (OMN-12880) + CHANGED — when --changed-contracts mode is active, only the specified + contracts are checked; unchanged contracts with gaps are skipped (OMN-12879) + ALLOWLIST — known pre-existing violations in _ALLOWLISTED_CMD_TOPICS pass + without failing the gate + +These tests prove the gate would have caught: + 1. The June 9 DLQ regression: node_generation_consumer subscribed + onex.cmd.omnimarket.node-generation-requested.v1 but auto-wiring produced + zero dispatchers after a sole-handler revert — messages went to DLQ silently. + 2. The June 12 DEL-01 live finding: delegate-skill consumed-but-undispatched + on dev (onex.cmd.omnimarket.delegate-skill.v1 with no matching route). + +[OMN-12858, OMN-12879, OMN-12880] +""" + +from __future__ import annotations + +import sys +import textwrap +from pathlib import Path + +import pytest +import yaml + +# --------------------------------------------------------------------------- +# Locate the gate script so tests can import its public check_route_coverage() +# API without needing the package installed. +# --------------------------------------------------------------------------- +_SCRIPTS_DIR = Path(__file__).resolve().parent.parent.parent / "scripts" +if str(_SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPTS_DIR)) + +from check_dispatcher_route_coverage import ( # type: ignore[import-not-found] + RouteCoverageReport, + check_route_coverage, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _write_contract(tmp_path: Path, node_name: str, contract_yaml: str) -> Path: + """Write a synthetic contract.yaml for a node and return its directory.""" + node_dir = tmp_path / node_name + node_dir.mkdir(parents=True, exist_ok=True) + (node_dir / "contract.yaml").write_text(textwrap.dedent(contract_yaml)) + return node_dir + + +# --------------------------------------------------------------------------- +# RED: unrouted command topic — gate FAILS +# --------------------------------------------------------------------------- +@pytest.mark.unit +def test_unrouted_command_topic_detected(tmp_path: Path) -> None: + """Gate FAILS when a subscribed command topic has no handler_routing. + + Shape of the June 9 DLQ regression and the June 12 DEL-01 live finding: + the contract subscribes a command topic but has no handler_routing or + runtime_dispatch, so messages go to DLQ silently at runtime. + """ + _write_contract( + tmp_path, + "node_synthetic_unrouted", + """ + name: node_synthetic_unrouted + node_type: EFFECT_GENERIC + contract_version: {major: 1, minor: 0, patch: 0} + event_bus: + subscribe_topics: + - "onex.cmd.omnimarket.delegate-skill.v1" + """, + ) + + report: RouteCoverageReport = check_route_coverage([tmp_path]) + + assert not report.passed, "Gate should FAIL for unrouted command topic" + assert len(report.failures) == 1 + assert report.failures[0].topic == "onex.cmd.omnimarket.delegate-skill.v1" + assert report.failures[0].contract_name == "node_synthetic_unrouted" + + +@pytest.mark.unit +def test_june9_regression_shape_detected(tmp_path: Path) -> None: + """Gate catches the exact June 9 DLQ regression shape. + + node_generation_consumer subscribed .node-generation-requested.v1 but had + no dispatcher route after a handler revert. + """ + _write_contract( + tmp_path, + "node_generation_consumer", + """ + name: node_generation_consumer + node_type: EFFECT_GENERIC + contract_version: {major: 1, minor: 0, patch: 0} + event_bus: + subscribe_topics: + - "onex.cmd.omnimarket.node-generation-requested.v1" + """, + ) + + report: RouteCoverageReport = check_route_coverage([tmp_path]) + + assert not report.passed + assert any( + f.topic == "onex.cmd.omnimarket.node-generation-requested.v1" + for f in report.failures + ), "June 9 regression topic must appear in failures" + + +# --------------------------------------------------------------------------- +# GREEN: routed command topic — gate PASSES +# --------------------------------------------------------------------------- +@pytest.mark.unit +def test_routed_via_handler_routing_passes(tmp_path: Path) -> None: + """Gate PASSES when handler_routing is declared on the contract.""" + _write_contract( + tmp_path, + "node_delegate_skill", + """ + name: node_delegate_skill + node_type: EFFECT_GENERIC + contract_version: {major: 1, minor: 0, patch: 0} + event_bus: + subscribe_topics: + - "onex.cmd.omnimarket.delegate-skill.v1" + handler_routing: + routing_strategy: operation_match + handlers: + - event_type: omnimarket.delegate-skill + handler: + name: HandlerDelegateSkill + module: omnimarket.nodes.node_delegate_skill.handlers.handler_delegate_skill + """, + ) + + report: RouteCoverageReport = check_route_coverage([tmp_path]) + + assert report.passed, f"Gate should PASS; failures={report.failures}" + assert len(report.failures) == 0 + + +@pytest.mark.unit +def test_routed_via_runtime_dispatch_passes(tmp_path: Path) -> None: + """Gate PASSES when runtime_dispatch is declared on the contract.""" + _write_contract( + tmp_path, + "node_pattern_b", + """ + name: node_pattern_b + node_type: EFFECT_GENERIC + contract_version: {major: 1, minor: 0, patch: 0} + event_bus: + subscribe_topics: + - "onex.cmd.omnibase-infra.invocation.v1" + runtime_dispatch: + strategy: pattern_b + broker_class: RuntimePatternBBroker + """, + ) + + report: RouteCoverageReport = check_route_coverage([tmp_path]) + + assert report.passed, ( + f"Gate should PASS with runtime_dispatch; failures={report.failures}" + ) + + +# --------------------------------------------------------------------------- +# COMPAT: compatibility_publish_topics are sender-side only (OMN-12880) +# --------------------------------------------------------------------------- +@pytest.mark.unit +def test_compat_publish_topic_not_flagged(tmp_path: Path) -> None: + """OMN-12880: compatibility_publish_topics MUST NOT be treated as gaps. + + A contract that only declares a compatibility_publish_topic (sender-side) + must not cause the gate to report a missing dispatcher route. + """ + _write_contract( + tmp_path, + "node_compat_publisher", + """ + name: node_compat_publisher + node_type: EFFECT_GENERIC + contract_version: {major: 1, minor: 0, patch: 0} + compatibility_publish_topics: + - "onex.cmd.omniclaude.task-delegated.v1" + """, + ) + + report: RouteCoverageReport = check_route_coverage([tmp_path]) + + assert report.passed, ( + "Gate must NOT flag compatibility_publish_topics as gaps; " + f"failures={report.failures}" + ) + + +@pytest.mark.unit +def test_event_topic_not_flagged(tmp_path: Path) -> None: + """Event (evt) topics are not command topics and must be ignored by the gate.""" + _write_contract( + tmp_path, + "node_event_only", + """ + name: node_event_only + node_type: REDUCER_GENERIC + contract_version: {major: 1, minor: 0, patch: 0} + event_bus: + subscribe_topics: + - "onex.evt.omnimarket.node-generation-completed.v1" + """, + ) + + report: RouteCoverageReport = check_route_coverage([tmp_path]) + + assert report.passed, "Gate must not flag evt topics; failures={report.failures}" + + +# --------------------------------------------------------------------------- +# CHANGED-CONTRACT MODE (OMN-12879): only changed contracts are checked +# --------------------------------------------------------------------------- +@pytest.mark.unit +def test_changed_contract_mode_only_checks_specified(tmp_path: Path) -> None: + """OMN-12879: in changed-contract mode, only specified paths are checked. + + Two unrouted contracts exist; only the one listed in changed_contract_paths + should appear as a failure. + """ + dir_a = _write_contract( + tmp_path, + "node_a", + """ + name: node_a + node_type: EFFECT_GENERIC + contract_version: {major: 1, minor: 0, patch: 0} + event_bus: + subscribe_topics: + - "onex.cmd.omnimarket.aislop-sweep-start.v1" + """, + ) + _write_contract( + tmp_path, + "node_b", + """ + name: node_b + node_type: EFFECT_GENERIC + contract_version: {major: 1, minor: 0, patch: 0} + event_bus: + subscribe_topics: + - "onex.cmd.omnimarket.bus-audit-start.v1" + """, + ) + + # Only node_a is "changed" in this PR + changed = frozenset([dir_a / "contract.yaml"]) + report: RouteCoverageReport = check_route_coverage( + [tmp_path], changed_contract_paths=changed + ) + + # Only node_a should be in failures; node_b is unchanged and skipped + assert not report.passed + failure_topics = {f.topic for f in report.failures} + assert "onex.cmd.omnimarket.aislop-sweep-start.v1" in failure_topics + assert "onex.cmd.omnimarket.bus-audit-start.v1" not in failure_topics, ( + "Unchanged contract node_b must not appear in failures" + ) + + +@pytest.mark.unit +def test_changed_contract_mode_passes_when_routed(tmp_path: Path) -> None: + """OMN-12879: gate PASSES in changed-contract mode when changed contract is routed.""" + node_dir = _write_contract( + tmp_path, + "node_routed_changed", + """ + name: node_routed_changed + node_type: EFFECT_GENERIC + contract_version: {major: 1, minor: 0, patch: 0} + event_bus: + subscribe_topics: + - "onex.cmd.omnimarket.compliance-sweep-start.v1" + handler_routing: + routing_strategy: operation_match + handlers: + - event_type: omnimarket.compliance-sweep-start + handler: + name: HandlerComplianceSweep + module: omnimarket.nodes.node_compliance_sweep.handlers.handler_compliance_sweep + """, + ) + + changed = frozenset([node_dir / "contract.yaml"]) + report: RouteCoverageReport = check_route_coverage( + [tmp_path], changed_contract_paths=changed + ) + + assert report.passed, f"Gate should PASS; failures={report.failures}" + + +# --------------------------------------------------------------------------- +# ALLOWLIST: pre-existing violations are not re-flagged +# --------------------------------------------------------------------------- +@pytest.mark.unit +def test_allowlisted_topic_not_in_failures(tmp_path: Path) -> None: + """Known pre-existing violations in the allowlist must not appear in failures.""" + _write_contract( + tmp_path, + "node_pattern_b_broker", + """ + name: node_pattern_b_broker + node_type: EFFECT_GENERIC + contract_version: {major: 1, minor: 0, patch: 0} + event_bus: + subscribe_topics: + - "onex.cmd.omnibase-infra.pattern-b-dispatch.v1" + """, + ) + + report: RouteCoverageReport = check_route_coverage([tmp_path]) + + assert report.passed, ( + "Allowlisted topic onex.cmd.omnibase-infra.pattern-b-dispatch.v1 " + f"must not cause gate failure; failures={report.failures}" + ) + assert len(report.allowlisted) == 1, ( + f"Expected 1 allowlisted entry; got {report.allowlisted}" + ) + + +# --------------------------------------------------------------------------- +# MULTI-DIR: both infra and omnimarket contracts scanned together +# --------------------------------------------------------------------------- +@pytest.mark.unit +def test_multiple_dirs_all_routed_passes(tmp_path: Path) -> None: + """Gate PASSES when all contracts across multiple directories are routed.""" + infra_dir = tmp_path / "infra_nodes" + market_dir = tmp_path / "market_nodes" + + _write_contract( + infra_dir, + "node_infra_effect", + """ + name: node_infra_effect + node_type: EFFECT_GENERIC + contract_version: {major: 1, minor: 0, patch: 0} + event_bus: + subscribe_topics: + - "onex.cmd.omnibase-infra.delegation-request.v1" + handler_routing: + routing_strategy: operation_match + handlers: + - event_type: omnibase-infra.delegation-request + handler: + name: HandlerDelegationRequestConsumer + module: omnibase_infra.nodes.node_delegation_effect.handlers.handler_delegation_request_consumer + """, + ) + _write_contract( + market_dir, + "node_market_effect", + """ + name: node_market_effect + node_type: EFFECT_GENERIC + contract_version: {major: 1, minor: 0, patch: 0} + event_bus: + subscribe_topics: + - "onex.cmd.omnimarket.node-generation-requested.v1" + handler_routing: + routing_strategy: operation_match + handlers: + - event_type: omnimarket.node-generation-requested + handler: + name: HandlerNodeGenerationConsumer + module: omnimarket.nodes.node_generation_consumer.handlers.handler_node_generation_consumer + """, + ) + + report: RouteCoverageReport = check_route_coverage([infra_dir, market_dir]) + + assert report.passed, f"Gate should PASS; failures={report.failures}" + assert len(report.scanned) == 2 + + +@pytest.mark.unit +def test_multiple_dirs_one_unrouted_fails(tmp_path: Path) -> None: + """Gate FAILS when one contract across multiple directories is unrouted.""" + infra_dir = tmp_path / "infra_nodes" + market_dir = tmp_path / "market_nodes" + + _write_contract( + infra_dir, + "node_infra_routed", + """ + name: node_infra_routed + node_type: EFFECT_GENERIC + contract_version: {major: 1, minor: 0, patch: 0} + event_bus: + subscribe_topics: + - "onex.cmd.omnibase-infra.delegation-request.v1" + handler_routing: + routing_strategy: operation_match + handlers: + - event_type: omnibase-infra.delegation-request + handler: + name: HandlerDelegationRequestConsumer + module: omnibase_infra.nodes.node_delegation_effect.handlers.handler_delegation_request_consumer + """, + ) + _write_contract( + market_dir, + "node_market_unrouted", + """ + name: node_market_unrouted + node_type: EFFECT_GENERIC + contract_version: {major: 1, minor: 0, patch: 0} + event_bus: + subscribe_topics: + - "onex.cmd.omnimarket.delegate-skill.v1" + """, + ) + + report: RouteCoverageReport = check_route_coverage([infra_dir, market_dir]) + + assert not report.passed + failure_topics = {f.topic for f in report.failures} + assert "onex.cmd.omnimarket.delegate-skill.v1" in failure_topics + assert "onex.cmd.omnibase-infra.delegation-request.v1" not in failure_topics + + +# --------------------------------------------------------------------------- +# LIVE REGRESSION PROOF: run gate against real contracts in the repo tree +# Passes only when the current checked-out contracts are fully wired. +# --------------------------------------------------------------------------- +@pytest.mark.unit +def test_live_infra_contracts_pass(tmp_path: Path) -> None: + """Gate PASSES against the actual omnibase_infra contract tree. + + This test is the proof that would have blocked the June 9 DLQ regression + and the June 12 DEL-01 finding had it been wired as a pre-merge check. + """ + infra_nodes_dir = ( + Path(__file__).resolve().parent.parent.parent + / "src" + / "omnibase_infra" + / "nodes" + ) + + if not infra_nodes_dir.is_dir(): + pytest.skip(f"omnibase_infra nodes dir not found: {infra_nodes_dir}") + + report: RouteCoverageReport = check_route_coverage([infra_nodes_dir]) + + assert report.passed, ( + "Live omnibase_infra contracts have unrouted command topics:\n" + + "\n".join(f" {f.topic} ({f.contract_name})" for f in report.failures) + ) diff --git a/tests/ci/test_receipt_gate_install_guard.py b/tests/ci/test_receipt_gate_install_guard.py index 5d0935bd03..ac7c8f8743 100644 --- a/tests/ci/test_receipt_gate_install_guard.py +++ b/tests/ci/test_receipt_gate_install_guard.py @@ -145,8 +145,15 @@ def test_install_uses_no_index_no_deps_wheel(self) -> None: "candidate-selection and force the local wheel (OMN-9283)" ) - def test_install_uninstalls_stale_pypi_package_first(self) -> None: - """The install step must uninstall omnibase-core/compat before reinstalling from source.""" + def test_install_uses_clean_workspace_venv(self) -> None: + """The install step must start from a freshly cleared workspace venv. + + OMN-12565 (core PR #1189) replaced the 'uninstall stale PyPI package + first' step (OMN-9198) with `uv venv --clear` into a workspace-rooted + venv: a cleared venv cannot carry a stale PyPI omnibase-core, which is + the same contamination guarantee without writing the system + interpreter. + """ workflow = self._load_receipt_gate() steps = workflow["jobs"]["verify"]["steps"] install_step = next( @@ -155,9 +162,10 @@ def test_install_uninstalls_stale_pypi_package_first(self) -> None: ) assert install_step is not None run_script: str = install_step.get("run", "") - assert "uv pip uninstall" in run_script and "omnibase-core" in run_script, ( - "Install omnibase_core step must uninstall the stale PyPI omnibase-core " - "package before installing from source (OMN-9198)" + assert "uv venv" in run_script and "--clear" in run_script, ( + "Install omnibase_core step must create a cleared workspace venv " + "(`uv venv --clear`) so no stale PyPI omnibase-core package can " + "leak into the gate environment (OMN-12565, supersedes OMN-9198)" ) def test_install_builds_wheel_from_source(self) -> None: diff --git a/tests/ci/test_runtime_policy_contract.py b/tests/ci/test_runtime_policy_contract.py index 763f007a4e..46eac882f3 100644 --- a/tests/ci/test_runtime_policy_contract.py +++ b/tests/ci/test_runtime_policy_contract.py @@ -117,8 +117,14 @@ def test_runtime_policy_env_has_expected_lane_values() -> None: assert env["ONEX_ACTIVE_RUNTIME_PACKAGES"] == "omnibase_infra,omnimarket" assert ( env["LLM_CLOUD_ENDPOINT_HOST_ALLOWLIST"] - == "generativelanguage.googleapis.com,api.z.ai" + == "generativelanguage.googleapis.com,api.z.ai,aiplatform.googleapis.com" ) + assert ( + env["BIFROST_VERTEX_GEMINI_ENDPOINT_URL"] + == "https://us-central1-aiplatform.googleapis.com/v1beta1/projects/gen-lang-client-0084338881/locations/us-central1/endpoints/openapi/chat/completions" + ) + assert env["GOOGLE_CLOUD_PROJECT"] == "gen-lang-client-0084338881" + assert env["GOOGLE_CLOUD_LOCATION"] == "us-central1" assert env["DEV_RUNTIME_MAIN_PORT"] == "8085" assert env["STABILITY_TEST_RUNTIME_MAIN_PORT"] == "18085" assert env["STABILITY_TEST_TOPIC_PROVISIONER_MAX_PARTITIONS"] == "1" @@ -172,3 +178,53 @@ def test_compose_consumes_policy_env_instead_of_hardcoded_policy_values() -> Non assert "ONEX_RUNTIME_CAPABILITIES: market.skill-proof" not in text assert "ONEX_RUNTIME_CAPABILITIES: effects.consumer" not in text assert "ONEX_RUNTIME_CAPABILITIES: workflow.dispatch" not in text + + +def test_worker_replicas_pinned_in_contract_for_every_lane() -> None: + """OMN-12990: each lane's worker process declares an explicit replica pin. + + The base compose default is ``${WORKER_REPLICAS:-0}`` (silent drop). The + contract worker process must carry a contract-declared replica count >= 1 so + the rendered policy env preserves the worker on a lane recreate. + """ + contract = _load_contract() + + for profile_name, profile in contract.profiles.items(): + worker = profile.processes["worker"] + assert worker.replicas >= 1, profile_name + + +def test_worker_replicas_rendered_into_policy_env_for_every_lane() -> None: + """OMN-12990: the renderer emits ``{PROFILE}_WORKER_REPLICAS`` per lane. + + These are the ledgered config values the lane overrides reference fail-fast. + """ + env = _load_dotenv(POLICY_ENV_PATH) + + assert env["DEV_WORKER_REPLICAS"] == "1" + assert env["STABILITY_TEST_WORKER_REPLICAS"] == "1" + assert env["JUDGE_WORKER_REPLICAS"] == "1" + assert env["PROD_WORKER_REPLICAS"] == "1" + + +def test_runtime_worker_replicas_are_fail_fast_not_silent_default() -> None: + """OMN-12990: lane overrides reference the policy value with fail-fast ``:?``. + + A soft ``:-1`` / ``:-0`` default would silently re-introduce the silent-drop + hole on any recreate that omitted the policy env. The base compose keeps its + ``${WORKER_REPLICAS:-0}`` default (that is the gap the overrides close), so a + plain recreate against the base alone still scales to zero — by design the + lane overrides MUST be applied, and they now abort loudly when the policy env + is missing rather than dropping the worker with no signal. + """ + stability_text = STABILITY_COMPOSE_PATH.read_text(encoding="utf-8") + prod_text = PROD_COMPOSE_PATH.read_text(encoding="utf-8") + + assert "${STABILITY_TEST_WORKER_REPLICAS:?" in stability_text, ( + "stability worker replicas must be fail-fast on the ledgered policy value" + ) + assert "${STABILITY_TEST_WORKER_REPLICAS:-" not in stability_text + assert "${PROD_WORKER_REPLICAS:?" in prod_text, ( + "prod worker replicas must be fail-fast on the ledgered policy value" + ) + assert "${PROD_WORKER_REPLICAS:-" not in prod_text diff --git a/tests/integration/docker/test_docker_integration.py b/tests/integration/docker/test_docker_integration.py index 7a6f0707d1..689dddf7e3 100644 --- a/tests/integration/docker/test_docker_integration.py +++ b/tests/integration/docker/test_docker_integration.py @@ -1130,6 +1130,12 @@ def test_compose_config_valid( "LLM_GLM_API_KEY": "render-only-glm-api-key", "GEMINI_API_KEY": "render-only-gemini-api-key", "GOOGLE_API_KEY": "render-only-google-api-key", + "BIFROST_VERTEX_GEMINI_ENDPOINT_URL": ( + "https://us-central1-aiplatform.googleapis.com/v1beta1/projects/" + "gen-lang-client-0084338881/locations/us-central1/endpoints/openapi/chat/completions" + ), + "GOOGLE_CLOUD_PROJECT": "gen-lang-client-0084338881", + "GOOGLE_CLOUD_LOCATION": "us-central1", # OMN-10943: HTTP request signing and CIDR allowlist for the # local LLM HTTP transport added with :? fail-fast. "LOCAL_LLM_SHARED_SECRET": "render-only-local-llm-secret", diff --git a/tests/integration/docker/test_runtime_image_identity_build.py b/tests/integration/docker/test_runtime_image_identity_build.py new file mode 100644 index 0000000000..064cc53206 --- /dev/null +++ b/tests/integration/docker/test_runtime_image_identity_build.py @@ -0,0 +1,167 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Integration coverage for runtime image identity stamping (OMN-12965). + +These tests build the runtime-stage identity block (ARG + LABEL + workspace +guard) extracted verbatim from ``docker/Dockerfile.runtime`` against a tiny base +image and assert, via ``docker inspect``, that: + +1. A workspace build WITH the identity quad populates + ``org.opencontainers.image.version`` and ``.revision`` (the fixed behavior). +2. A workspace build WITHOUT the identity args FAILS the guard (the broken + blank-identity image is refused, exit 64). +3. A release build without args is allowed to carry the placeholder default + (no regression for release tooling). + +This makes the local throwaway-build proof a permanent gate. The block under +test is the real runtime-stage logic, so a regression in the Dockerfile guard or +label wiring is caught here. +""" + +from __future__ import annotations + +import re +import subprocess +import uuid +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.integration, pytest.mark.slow] + +_REPO_ROOT = Path(__file__).resolve().parents[3] +_DOCKERFILE = _REPO_ROOT / "docker" / "Dockerfile.runtime" + + +def _runtime_identity_block() -> str: + """Extract the runtime-stage ARG/guard/LABEL identity block from the real Dockerfile. + + Pulls the exact lines under test so the integration build exercises the + shipped logic, not a hand-copied paraphrase. Spans from the runtime-stage + ``ARG RUNTIME_VERSION`` line through the end of the OCI ``LABEL`` block. + """ + text = _DOCKERFILE.read_text(encoding="utf-8") + # Runtime stage begins at the second `FROM python:...-slim`. Slice from the + # runtime-stage RUNTIME_VERSION arg to the end of the first LABEL block after + # it (the OCI image labels including version/revision). + runtime_start = text.index("FROM python:${PYTHON_VERSION}-slim AS runtime") + runtime_text = text[runtime_start:] + arg_idx = runtime_text.index("ARG RUNTIME_VERSION=0.1.0") + label_idx = runtime_text.index( + 'com.omninode.workspace_provenance_manifest="/app/build-provenance.json"' + ) + # Extend to the end of that physical line. + label_end = runtime_text.index("\n", label_idx) + return runtime_text[arg_idx : label_end + 1] + + +def _write_proof_dockerfile(tmp_path: Path) -> Path: + """Write a throwaway Dockerfile wrapping the real identity block on busybox.""" + block = _runtime_identity_block() + # The block references BUILD_SOURCE/EXPECTED_BUILD_SOURCE/OMNI_HOME args that + # the runtime stage declares earlier; declare them here so the slice builds. + proof = "FROM busybox:latest AS runtime\n" + block + dockerfile = tmp_path / "Dockerfile.identity-proof" + dockerfile.write_text(proof, encoding="utf-8") + return dockerfile + + +def _build( + dockerfile: Path, context: Path, tag: str, build_args: dict[str, str] +) -> subprocess.CompletedProcess[str]: + cmd = ["docker", "build", "-f", str(dockerfile), "-t", tag] + for key, value in build_args.items(): + cmd += ["--build-arg", f"{key}={value}"] + cmd.append(str(context)) + return subprocess.run(cmd, capture_output=True, text=True, check=False) + + +def _inspect_label(tag: str, label: str) -> str: + result = subprocess.run( + [ + "docker", + "inspect", + tag, + "--format", + f'{{{{index .Config.Labels "{label}"}}}}', + ], + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + + +@pytest.mark.integration +def test_workspace_build_with_identity_args_populates_labels( + skip_if_no_docker: None, tmp_path: Path +) -> None: + """Workspace build + identity quad → populated version + revision labels.""" + dockerfile = _write_proof_dockerfile(tmp_path) + tag = f"omn12965-good-{uuid.uuid4().hex[:8]}" + try: + result = _build( + dockerfile, + tmp_path, + tag, + { + "BUILD_SOURCE": "workspace", + "EXPECTED_BUILD_SOURCE": "workspace", + "OMNI_HOME": "/x", + "RUNTIME_VERSION": "0.38.3", + "VCS_REF": "abc123def456", + "BUILD_DATE": "2026-06-11T00:00:00Z", + }, + ) + assert result.returncode == 0, f"build failed: {result.stderr}" + assert _inspect_label(tag, "org.opencontainers.image.version") == "0.38.3" + assert ( + _inspect_label(tag, "org.opencontainers.image.revision") == "abc123def456" + ) + assert re.fullmatch( + r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", + _inspect_label(tag, "org.opencontainers.image.created"), + ) + finally: + subprocess.run(["docker", "rmi", "-f", tag], capture_output=True, check=False) + + +@pytest.mark.integration +def test_workspace_build_without_identity_args_fails_guard( + skip_if_no_docker: None, tmp_path: Path +) -> None: + """Workspace build without identity args → guard fails (blank identity refused).""" + dockerfile = _write_proof_dockerfile(tmp_path) + tag = f"omn12965-bad-{uuid.uuid4().hex[:8]}" + try: + result = _build( + dockerfile, + tmp_path, + tag, + { + "BUILD_SOURCE": "workspace", + "EXPECTED_BUILD_SOURCE": "workspace", + "OMNI_HOME": "/x", + }, + ) + assert result.returncode != 0, ( + "workspace build with blank identity must fail the guard (OMN-12965)" + ) + assert "OMN-12965" in (result.stderr + result.stdout) + finally: + subprocess.run(["docker", "rmi", "-f", tag], capture_output=True, check=False) + + +@pytest.mark.integration +def test_release_build_allows_placeholder_default( + skip_if_no_docker: None, tmp_path: Path +) -> None: + """Release build without args keeps the placeholder default (no regression).""" + dockerfile = _write_proof_dockerfile(tmp_path) + tag = f"omn12965-rel-{uuid.uuid4().hex[:8]}" + try: + result = _build(dockerfile, tmp_path, tag, {"BUILD_SOURCE": "release"}) + assert result.returncode == 0, f"release build failed: {result.stderr}" + assert _inspect_label(tag, "org.opencontainers.image.version") == "0.1.0" + finally: + subprocess.run(["docker", "rmi", "-f", tag], capture_output=True, check=False) diff --git a/tests/integration/gateway/test_bus_forwarder_contract_integration.py b/tests/integration/gateway/test_bus_forwarder_contract_integration.py new file mode 100644 index 0000000000..22a4944b51 --- /dev/null +++ b/tests/integration/gateway/test_bus_forwarder_contract_integration.py @@ -0,0 +1,177 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Any +from uuid import UUID, uuid4 + +import pytest + +from omnibase_infra.nodes.node_bus_forwarder_effect.models import ( + ModelGatewayCloudBusConfig, + ModelGatewayEnvelope, + ModelGatewayForwarderConfig, + ModelGatewayMirrorTopics, + ModelGatewayTenantIdentity, +) +from omnibase_infra.nodes.node_bus_forwarder_effect.services.service_gateway_forwarder import ( + ServiceGatewayForwarder, +) + +pytestmark = pytest.mark.integration + +TENANT_ID = UUID("11111111-1111-1111-1111-111111111111") +BROKER_PROVIDER_ID = UUID("22222222-2222-2222-2222-222222222222") +PRINCIPAL_ID = UUID("33333333-3333-3333-3333-333333333333") +CORRELATION_ID = UUID("44444444-4444-4444-4444-444444444444") +INBOUND_TOPIC = "onex.cmd.omnibase-infra.delegation-inference-request.v1" +OUTBOUND_TOPIC = "onex.evt.omnibase-infra.inference-response.v1" +WIRE_INBOUND_TOPIC = f"tenant-acme.{INBOUND_TOPIC}" +WIRE_OUTBOUND_TOPIC = f"tenant-acme.{OUTBOUND_TOPIC}" + + +@dataclass(frozen=True) +class _Message: + topic: str + key: bytes | None + value: bytes + headers: object | None = None + + +class _RecordingBus: + def __init__(self) -> None: + self.subscriptions: dict[str, Callable[[Any], Awaitable[None]]] = {} + self.subscription_groups: dict[str, str] = {} + self.published: list[_Message] = [] + + async def subscribe( + self, + topic: str, + node_identity: object | None = None, + on_message: Callable[[Any], Awaitable[None]] | None = None, + *, + group_id: str | None = None, + **_kwargs: object, + ) -> Callable[[], Awaitable[None]]: + assert on_message is not None + assert group_id is not None + self.subscriptions[topic] = on_message + self.subscription_groups[topic] = group_id + + async def _unsubscribe() -> None: + self.subscriptions.pop(topic, None) + self.subscription_groups.pop(topic, None) + + return _unsubscribe + + async def publish( + self, + topic: str, + key: bytes | None, + value: bytes, + headers: object | None = None, + ) -> None: + self.published.append(_Message(topic, key, value, headers)) + + async def emit(self, topic: str, envelope: ModelGatewayEnvelope) -> None: + await self.subscriptions[topic]( + _Message( + topic=topic, + key=b"tenant-key", + value=envelope.model_dump_json().encode("utf-8"), + headers={"traceparent": "00-test"}, + ) + ) + + +def _config() -> ModelGatewayForwarderConfig: + return ModelGatewayForwarderConfig( + tenant_identity=ModelGatewayTenantIdentity( + tenant_id=TENANT_ID, + tenant_slug="acme", + principal_id=PRINCIPAL_ID, + ), + cloud_bus=ModelGatewayCloudBusConfig( + broker_provider_id=BROKER_PROVIDER_ID, + cloud_broker_ref="gateway.cloud.kafka.broker", + cloud_auth_ref="gateway.cloud.kafka.oauth", + acl_provisioner_ref="gateway.cloud.kafka.authorization", + client_id_ref="gateway.cloud.kafka.oauth.client_id", + client_secret_api_key_ref="infisical://gateway/redpanda-events", + ), + local_transport_flavor="containerized", + mirror_topics=ModelGatewayMirrorTopics( + inbound=(INBOUND_TOPIC,), + outbound=(OUTBOUND_TOPIC,), + ), + ) + + +def _envelope(**overrides: object) -> ModelGatewayEnvelope: + values = { + "tenant_id": TENANT_ID, + "tenant_slug": "acme", + "envelope_id": uuid4(), + "correlation_id": CORRELATION_ID, + "causation_id": None, + "event_type": "LlmInferenceResponse", + "source_topic": OUTBOUND_TOPIC, + "wire_topic": "", + "canonical_topic": OUTBOUND_TOPIC, + "payload": {"ok": True}, + } + values.update(overrides) + return ModelGatewayEnvelope(**values) + + +@pytest.mark.asyncio +async def test_gateway_forwarder_preserves_envelope_across_both_bus_legs() -> None: + local_bus = _RecordingBus() + cloud_bus = _RecordingBus() + service = ServiceGatewayForwarder( + config=_config(), + local_bus=local_bus, + cloud_bus=cloud_bus, + ) + + await service.start() + await local_bus.emit(OUTBOUND_TOPIC, _envelope()) + await cloud_bus.emit( + WIRE_INBOUND_TOPIC, + _envelope( + event_type="DelegationInferenceRequest", + source_topic=WIRE_INBOUND_TOPIC, + wire_topic=WIRE_INBOUND_TOPIC, + canonical_topic=INBOUND_TOPIC, + ), + ) + + assert set(local_bus.subscriptions) == {OUTBOUND_TOPIC} + assert set(cloud_bus.subscriptions) == {WIRE_INBOUND_TOPIC} + assert local_bus.subscription_groups[OUTBOUND_TOPIC] == ( + "tenant-acme-gateway-forwarder-outbound" + ) + assert cloud_bus.subscription_groups[WIRE_INBOUND_TOPIC] == ( + "tenant-acme-gateway-forwarder-inbound" + ) + + outbound = cloud_bus.published[0] + assert outbound.topic == WIRE_OUTBOUND_TOPIC + assert outbound.key == b"tenant-key" + assert outbound.headers == {"traceparent": "00-test"} + outbound_envelope = ModelGatewayEnvelope.model_validate_json(outbound.value) + assert outbound_envelope.wire_topic == WIRE_OUTBOUND_TOPIC + assert outbound_envelope.canonical_topic == OUTBOUND_TOPIC + assert outbound_envelope.tenant_id == TENANT_ID + + inbound = local_bus.published[0] + assert inbound.topic == INBOUND_TOPIC + assert inbound.key == b"tenant-key" + assert inbound.headers == {"traceparent": "00-test"} + inbound_envelope = ModelGatewayEnvelope.model_validate_json(inbound.value) + assert inbound_envelope.source_topic == WIRE_INBOUND_TOPIC + assert inbound_envelope.canonical_topic == INBOUND_TOPIC + assert inbound_envelope.tenant_slug == "acme" diff --git a/tests/integration/infra/test_omn_12765_release_backmerge_identity.py b/tests/integration/infra/test_omn_12765_release_backmerge_identity.py index f47ec3af97..e17e3254b2 100644 --- a/tests/integration/infra/test_omn_12765_release_backmerge_identity.py +++ b/tests/integration/infra/test_omn_12765_release_backmerge_identity.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. # SPDX-License-Identifier: MIT """Release backmerge identity checks for OMN-12765.""" @@ -15,7 +15,9 @@ def test_release_backmerge_preserves_proven_runtime_core_pin() -> None: pyproject = (ROOT / "pyproject.toml").read_text(encoding="utf-8") uv_lock = (ROOT / "uv.lock").read_text(encoding="utf-8") - expected_core = "c7d6b5ec010692b6724de1d49a0ee5a599d7ee06" + # OMN-13094: pin advanced to the merged ArtifactStore slice (OMN-13093) — + # receipt mode consumes omnibase_core.artifacts + ModelSkillResult. + expected_core = "ae8793bdad96c12a0b47e2f4d1d0f179618553b8" assert expected_core in pyproject assert expected_core in uv_lock @@ -28,5 +30,9 @@ def test_release_backmerge_preserves_runner_identity_lock() -> None: (ROOT / "docker/runners/runner-image.lock.json").read_text(encoding="utf-8") ) - assert lock["identity_digest"] == "7f4a28050457a278a6712b05de154cb8" - assert lock["shared_env_digest"] == "83c23e55321a8c8cb95d48c1" + # OMN-13096: identity regenerated after adding the 'delegate' onex.cli + # entry point to pyproject.toml, merged with the 'skill' entry point from + # OMN-13097 (both digests bind pyproject.toml + uv.lock; same mechanical + # regeneration OMN-13094 did for the advanced core pin). + assert lock["identity_digest"] == "efc067283a47eb66bfae03322e966b0c" + assert lock["shared_env_digest"] == "215aa5ffa92e228bf703615d" diff --git a/tests/integration/infra/test_runtime_worker_replicas_policy_integration.py b/tests/integration/infra/test_runtime_worker_replicas_policy_integration.py new file mode 100644 index 0000000000..b841c5f38d --- /dev/null +++ b/tests/integration/infra/test_runtime_worker_replicas_policy_integration.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Integration coverage for runtime worker replica policy rendering.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from omnibase_infra.runtime.models.model_runtime_policy_contract import ( + ModelRuntimePolicyContract, +) +from scripts.render_runtime_policy_env import format_dotenv, render_env + +REPO_ROOT = Path(__file__).resolve().parents[3] +CONTRACT_PATH = REPO_ROOT / "contracts" / "services" / "runtime_policy.contract.yaml" +POLICY_ENV_PATH = REPO_ROOT / "docker" / "runtime-policy.env" +PROD_COMPOSE_PATH = REPO_ROOT / "docker" / "docker-compose.prod.yml" +STABILITY_COMPOSE_PATH = REPO_ROOT / "docker" / "docker-compose.stability-test.yml" + +PROFILE_ENV_PREFIXES = { + "dev": "DEV", + "stability-test": "STABILITY_TEST", + "judge": "JUDGE", + "prod": "PROD", +} + +pytestmark = pytest.mark.integration + + +def _construct_compose_value( + loader: yaml.SafeLoader, + node: yaml.Node, +) -> object: + if isinstance(node, yaml.MappingNode): + return loader.construct_mapping(node) + if isinstance(node, yaml.SequenceNode): + return loader.construct_sequence(node) + assert isinstance(node, yaml.ScalarNode) + return loader.construct_scalar(node) + + +class _ComposeLoader(yaml.SafeLoader): + """Test-local YAML loader with Docker Compose tag support.""" + + +_ComposeLoader.add_constructor("!override", _construct_compose_value) + + +def _load_contract() -> ModelRuntimePolicyContract: + raw = yaml.safe_load(CONTRACT_PATH.read_text(encoding="utf-8")) + return ModelRuntimePolicyContract.model_validate(raw) + + +def _load_dotenv(path: Path) -> dict[str, str]: + env: dict[str, str] = {} + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + key, value = line.split("=", maxsplit=1) + env[key] = value + return env + + +def _load_compose(path: Path) -> dict[str, Any]: + raw = yaml.load(path.read_text(encoding="utf-8"), Loader=_ComposeLoader) # noqa: S506 + assert isinstance(raw, dict) + return raw + + +def test_worker_replica_policy_integrates_contract_env_and_lane_overrides() -> None: + """OMN-12990: lane recreates must fail-fast instead of dropping workers.""" + contract = _load_contract() + + for profile_name, profile in contract.profiles.items(): + worker = profile.processes["worker"] + assert worker.replicas >= 1, profile_name + + expected_env = format_dotenv(render_env(contract)) + observed_env = POLICY_ENV_PATH.read_text(encoding="utf-8") + assert observed_env == expected_env + + policy_env = _load_dotenv(POLICY_ENV_PATH) + for profile_name, env_prefix in PROFILE_ENV_PREFIXES.items(): + expected_replicas = str( + contract.profiles[profile_name].processes["worker"].replicas + ) + assert policy_env[f"{env_prefix}_WORKER_REPLICAS"] == expected_replicas + assert int(expected_replicas) >= 1 + + stability_worker = _load_compose(STABILITY_COMPOSE_PATH)["services"][ + "runtime-worker" + ] + prod_worker = _load_compose(PROD_COMPOSE_PATH)["services"]["runtime-worker"] + + assert stability_worker["deploy"]["replicas"].startswith( + "${STABILITY_TEST_WORKER_REPLICAS:?" + ) + assert ":-" not in stability_worker["deploy"]["replicas"] + assert prod_worker["deploy"]["replicas"].startswith("${PROD_WORKER_REPLICAS:?") + assert ":-" not in prod_worker["deploy"]["replicas"] diff --git a/tests/integration/infra/test_stability_test_runtime_compose_render.py b/tests/integration/infra/test_stability_test_runtime_compose_render.py index c5fc492407..445805d0a8 100644 --- a/tests/integration/infra/test_stability_test_runtime_compose_render.py +++ b/tests/integration/infra/test_stability_test_runtime_compose_render.py @@ -84,7 +84,7 @@ "omnibase-infra-postgres", "omnibase-infra-redpanda", "omnibase-infra-valkey", - "omnibase-forward-migration", + "omnibase-infra-forward-migration", "omnibase-infra-infisical", "omninode-agent-actions-consumer", "omninode-skill-lifecycle-consumer", @@ -420,6 +420,27 @@ def test_stability_lane_render_contains_isolated_runtime_identity() -> None: ] == ("service_healthy") +@pytest.mark.integration +def test_stability_lane_render_pins_worker_replicas_to_one() -> None: + """The stability worker must render with deploy.replicas == 1 (OMN-12988). + + The base infra compose defaults runtime-worker to replicas 0 + (``${WORKER_REPLICAS:-0}``); without a hard pin in the stability override a + plain compose recreate silently drops the worker (4-container census). This + ratchet fails if the override regresses to 0 or to an env-interpolation + default that resolves to anything other than 1. + """ + rendered_config = _compose_config_json() + worker = rendered_config["services"]["runtime-worker"] + + deploy = worker.get("deploy") + assert deploy is not None, "runtime-worker must declare a deploy block" + assert deploy.get("replicas") == 1, ( + "stability-test runtime-worker must render deploy.replicas == 1; got " + f"{deploy.get('replicas')!r}" + ) + + @pytest.mark.integration def test_stability_lane_render_inherits_release_build_source() -> None: rendered_config = _compose_config_json() diff --git a/tests/integration/runtime/test_runtime_policy_contract.py b/tests/integration/runtime/test_runtime_policy_contract.py index 2e3b6ad265..da20693f56 100644 --- a/tests/integration/runtime/test_runtime_policy_contract.py +++ b/tests/integration/runtime/test_runtime_policy_contract.py @@ -39,11 +39,24 @@ def test_runtime_policy_contract_controls_all_runtime_lane_ports() -> None: assert contract.llm_cloud_endpoint_host_allowlist == ( "generativelanguage.googleapis.com", "api.z.ai", + "aiplatform.googleapis.com", ) assert ( rendered_env["LLM_CLOUD_ENDPOINT_HOST_ALLOWLIST"] - == "generativelanguage.googleapis.com,api.z.ai" + == "generativelanguage.googleapis.com,api.z.ai,aiplatform.googleapis.com" ) + assert ( + contract.bifrost_vertex_gemini_endpoint_url + == "https://us-central1-aiplatform.googleapis.com/v1beta1/projects/gen-lang-client-0084338881/locations/us-central1/endpoints/openapi/chat/completions" + ) + assert ( + rendered_env["BIFROST_VERTEX_GEMINI_ENDPOINT_URL"] + == contract.bifrost_vertex_gemini_endpoint_url + ) + assert contract.google_cloud_project == "gen-lang-client-0084338881" + assert rendered_env["GOOGLE_CLOUD_PROJECT"] == contract.google_cloud_project + assert contract.google_cloud_location == "us-central1" + assert rendered_env["GOOGLE_CLOUD_LOCATION"] == contract.google_cloud_location expected_ports = { "DEV_RUNTIME_MAIN_PORT": contract.profiles["dev"].main_port, @@ -81,6 +94,7 @@ def test_runtime_policy_contract_controls_logical_secret_resolver_refs() -> None "llm.openrouter.api_key", "llm.glm.api_key", "llm.gemini.api_key", + "llm.vertex.access_token", } assert judge_logical_names == logical_names assert ( diff --git a/tests/integration/runtime/test_runtime_profile_registry_integration.py b/tests/integration/runtime/test_runtime_profile_registry_integration.py new file mode 100644 index 0000000000..9d884108ce --- /dev/null +++ b/tests/integration/runtime/test_runtime_profile_registry_integration.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Integration coverage for core-owned runtime profile registry parity.""" + +from __future__ import annotations + +from omnibase_core.constants.constants_runtime_profiles import ( + CONSUMER_ATTACHED_RUNTIME_PROFILES, + REGISTERED_RUNTIME_PROFILES, +) +from omnibase_infra.runtime.runtime_profile import _PROFILES, load_runtime_profile + + +def test_infra_runtime_profiles_match_core_registry() -> None: + """Infra must be able to boot every profile accepted by core validators.""" + assert frozenset(_PROFILES) == REGISTERED_RUNTIME_PROFILES + + +def test_consumer_attached_runtime_profiles_preserve_identity() -> None: + """Consumer lanes declared by core must not silently fall back to default.""" + for profile_name in sorted(CONSUMER_ATTACHED_RUNTIME_PROFILES): + assert load_runtime_profile(profile_name).name == profile_name diff --git a/tests/integration/test_pattern_b_broker_terminal_waiter.py b/tests/integration/test_pattern_b_broker_terminal_waiter.py index cdb5bac7f5..d8a50310e4 100644 --- a/tests/integration/test_pattern_b_broker_terminal_waiter.py +++ b/tests/integration/test_pattern_b_broker_terminal_waiter.py @@ -46,6 +46,9 @@ def __init__(self, *topics: object, **kwargs: object) -> None: self.topics_calls = 0 self.set_topics_calls: list[tuple[str, ...]] = [] self.seeked_to_end = False + # True once the read position is pinned to the current end (synchronous + # end_offsets + seek, OMN-13118). + self.positioned = False self.started = False self.stopped = False type(self).created.append(self) @@ -72,6 +75,16 @@ def assignment(self) -> set[object]: async def seek_to_end(self, *_assignment: object) -> None: self.seeked_to_end = True + self.positioned = True + + async def end_offsets(self, partitions: object) -> dict[object, int]: + # Synchronous HWM round-trip (OMN-13118). Queue-backed fake delivers + # regardless of offset, so 0 is faithful for the next-message offset. + return dict.fromkeys(partitions, 0) + + def seek(self, _partition: object, _offset: int) -> None: + # Synchronous position pin (OMN-13118). + self.positioned = True async def getone(self) -> SimpleNamespace: return await self.messages.get() @@ -117,7 +130,7 @@ async def publish( assert consumer.kwargs["group_id"] is None assert consumer.topics == () assert consumer.assigned_partitions - assert consumer.seeked_to_end is True + assert consumer.positioned is True command_envelope = ModelEventEnvelope[object].model_validate_json(value) terminal_envelope = ModelEventEnvelope[object]( diff --git a/tests/integration/test_terminal_consumer_longlived_correlator.py b/tests/integration/test_terminal_consumer_longlived_correlator.py new file mode 100644 index 0000000000..cb09840509 --- /dev/null +++ b/tests/integration/test_terminal_consumer_longlived_correlator.py @@ -0,0 +1,437 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Long-lived correlator full-battery regression for the consume-leg wedge. + +Ticket: OMN-13118 (Tier B canonical redesign; epic OMN-12525). + +Why this REPLACES the per-trial-substrate repros +------------------------------------------------- +``test_terminal_consumer_battery_load_wedge.py`` (#1970), +``test_terminal_consumer_seek_reset_race.py`` (#1971), and +``test_terminal_consumer_subscription_flip_wedge.py`` (#1972) each modeled a +facet of the PER-TRIAL ephemeral consumer (a brand-new thread+loop+consumer+ +client per (trial x terminal-topic)) and each went RED->GREEN offline while the +LIVE K>=10 battery wedged every time (HALT_K10_WEDGE_PERSISTS, strikes 3-5). The +five offset/subscription patches all tuned that ephemeral substrate; the design +re-examination (CONSUME_LEG_REDESIGN_DESIGN.md, B1_M1_M2_PROBE_RESULT.md) +concluded the EPHEMERALITY itself is the defect class and replaced it with ONE +long-lived correlator (one consumer subscribed once to both terminal topics, a +single poll loop, demux by correlation_id). With that substrate there is no +per-trial open/assign/pin/teardown to race, so those repros model machinery that +no longer exists. + +What this test models HONESTLY (the live fault the prior fakes masked) +---------------------------------------------------------------------- +The decisive live fact (HALT doc): the COMPLETED terminal IS published every +trial (its HWM climbs) but the per-trial COMPLETED consumer never surfaced it — +the COMPLETED leg logged NOTHING while only the empty-set FAILED leg timed out at +the 30s assign cap. The prior fakes could not reproduce that because each built a +fresh in-memory consumer per trial whose ``getone`` reliably returned the record. + +This test drives the REAL long-lived correlator +(``make_terminal_event_consumer`` -> ``TerminalEventConsumer.open()/wait()`` -> +``LongLivedTerminalCorrelator``) the way ``HandlerContextRoiRunner._run_trial`` +does — open BOTH terminal topics, register the cid before publish, race the wait +— across K=10 x 2 cells x 2 arms. The fake ``AIOKafkaConsumer`` is built ONCE +(the correlator never rebuilds it); it models a shared cluster where the +COMPLETED topic has a partition and the FAILED topic (never produced to) does +not. The terminal lands in the open->wait gap, and the single always-running poll +loop demuxes it to the waiting future. + +The structural assertion that distinguishes the redesign from the old substrate: +exactly ONE consumer is constructed for the WHOLE battery (not 2 per trial). +""" + +from __future__ import annotations + +import json +import threading +import time +from types import SimpleNamespace +from typing import Any, ClassVar + +import pytest + +from omnibase_infra.runtime import service_pattern_b_broker +from omnibase_infra.runtime.service_terminal_event_consumer import ( + make_terminal_event_consumer, +) + +pytestmark = [pytest.mark.integration] + +_COMPLETED_TOPIC = "onex.evt.omnimarket.node-generation-completed.v1" +_FAILED_TOPIC = "onex.evt.omnimarket.node-generation-failed.v1" +_HANDLER = "HandlerContextRoiRunner" + +_TRIALS_PER_CELL = 10 +_CELLS = ("bool_flag", "address_norm") +_ARMS = ("off", "golden_exemplar") + + +class _PartitionLog: + """Append-only partition log for one terminal topic with a high-water-mark.""" + + def __init__(self) -> None: + self._records: list[tuple[str, bytes]] = [] + self._lock = threading.Lock() + + @property + def hwm(self) -> int: + with self._lock: + return len(self._records) + + def append(self, correlation_id: str, value: bytes) -> None: + with self._lock: + self._records.append((correlation_id, value)) + + def read_at(self, offset: int) -> tuple[str, bytes] | None: + with self._lock: + if offset < len(self._records): + return self._records[offset] + return None + + +class _Cluster: + """Shared broker state for the single long-lived consumer. + + The COMPLETED topic has a partition (produced to every trial). The FAILED + topic does NOT (never written) — Redpanda auto-create-on-produce, so the + correlator assigns it the empty set (a valid steady state). + """ + + def __init__(self) -> None: + self.logs: dict[str, _PartitionLog] = { + _COMPLETED_TOPIC: _PartitionLog(), + _FAILED_TOPIC: _PartitionLog(), + } + self.topics_with_partitions: set[str] = {_COMPLETED_TOPIC} + self.consumers_built = 0 + + +class _FakeClient: + def __init__(self, consumer: _FakeConsumer) -> None: + self._consumer = consumer + self.tracked: set[str] = set() + + async def _refresh(self) -> bool: + return True + + def set_topics(self, topics: list[str]) -> Any: + self.tracked = set(topics) + return self._refresh() + + def force_metadata_update(self) -> Any: + return self._refresh() + + +class _FakeConsumer: + """Single long-lived ``AIOKafkaConsumer`` stand-in for the correlator. + + The correlator builds exactly ONE of these for the whole batch and runs a + single poll loop over it. ``assign`` accumulates partitions across both + terminal topics; ``getone`` reads the next record across all assigned + partitions in append order, advancing a per-partition cursor. The FAILED + topic contributes no partition (never produced to). + """ + + cluster: ClassVar[_Cluster | None] = None + + def __init__(self, *args: object, **kwargs: object) -> None: + self._client = _FakeClient(self) + self._assigned: set[Any] = set() + self._cursors: dict[Any, int] = {} + assert type(self).cluster is not None + type(self).cluster.consumers_built += 1 + + @property + def _c(self) -> _Cluster: + assert type(self).cluster is not None + return type(self).cluster + + async def start(self) -> None: + return None + + def partitions_for_topic(self, topic: str) -> set[int]: + self._client.tracked.add(topic) + if topic not in self._c.topics_with_partitions: + return set() + return {0} + + def assign(self, partitions: Any) -> None: + self._assigned = set(partitions) + + def assignment(self) -> set[Any]: + return set(self._assigned) + + async def seek_to_end(self, *assignment: object) -> None: + return None + + async def end_offsets(self, partitions: Any) -> dict[Any, int]: + out: dict[Any, int] = {} + for tp in partitions: + out[tp] = self._c.logs[tp.topic].hwm + return out + + def seek(self, partition: Any, offset: int) -> None: + self._cursors[partition] = offset + + async def getone(self) -> SimpleNamespace: + import asyncio + + while True: + for tp in sorted(self._assigned, key=lambda p: p.topic): + cursor = self._cursors.get(tp, 0) + record = self._c.logs[tp.topic].read_at(cursor) + if record is not None: + self._cursors[tp] = cursor + 1 + _cid, value = record + return SimpleNamespace(topic=tp.topic, value=value) + await asyncio.sleep(0.002) + + async def stop(self) -> None: + return None + + +class _KafkaLikeBus: + config = SimpleNamespace( + session_timeout_ms=45000, + heartbeat_interval_ms=15000, + max_poll_interval_ms=1800000, + reconnect_backoff_ms=2000, + ) + _bootstrap_servers = "omn-13118-tierb-test-broker" + + def _build_auth_kwargs(self) -> dict[str, object]: + return {} + + +def _terminal_envelope_json(correlation_id: str) -> bytes: + return json.dumps( + { + "correlation_id": correlation_id, + "payload": { + "attempt_count": 1, + "contract_passed": True, + "model_id": "Qwen3.6-35B-A3B", + "provider": "local", + "prompt_tokens": 128, + "completion_tokens": 64, + }, + } + ).encode("utf-8") + + +def _run_one_trial( + *, + consumer_factory: Any, + cluster: _Cluster, + correlation_id: str, + timeout_seconds: float, +) -> dict[str, object] | None: + """Drive ONE trial as ``HandlerContextRoiRunner._run_trial`` does. + + open both terminals -> register cid -> publish (terminal lands in the gap) + -> wait on COMPLETED. The single long-lived poll loop demuxes the gap-landed + terminal to the registered future. + """ + completed = consumer_factory.open(_COMPLETED_TOPIC) + failed = consumer_factory.open(_FAILED_TOPIC) + completed.register(correlation_id) + failed.register(correlation_id) + + # FAST generation: correlated COMPLETED terminal lands in the open->wait gap. + cluster.logs[_COMPLETED_TOPIC].append( + correlation_id, _terminal_envelope_json(correlation_id) + ) + + try: + return completed.wait(correlation_id, timeout_seconds) + finally: + completed.close() + failed.close() + + +@pytest.mark.timeout(180) +def test_longlived_correlator_does_not_wedge_under_full_battery( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """K>=10 x 2 cells x 2 arms must each correlate via ONE long-lived consumer. + + RED on the per-trial substrate (the old ``TerminalConsumerSession`` rebuilt a + consumer per topic per trial whose COMPLETED delivery raced its own + bootstrap/teardown). GREEN on the long-lived correlator: one consumer, + subscribed once, single poll loop demuxes every gap-landed terminal. + """ + cluster = _Cluster() + _FakeConsumer.cluster = cluster + monkeypatch.setattr( + "omnibase_infra.runtime.service_pattern_b_broker.AIOKafkaConsumer", + _FakeConsumer, + ) + monkeypatch.setattr( + service_pattern_b_broker, + "_DIRECT_TERMINAL_PARTITIONLESS_GRACE_SECONDS", + 0.05, + raising=False, + ) + + bus = _KafkaLikeBus() + consumer = make_terminal_event_consumer(event_bus=bus, handler_name=_HANDLER) + timeout_seconds = 5.0 + + correlated: list[tuple[str, str, int]] = [] + degenerate: list[tuple[str, str, int]] = [] + arms_with_real_rows: set[str] = set() + cells_with_real_rows: set[str] = set() + + try: + run_order = 0 + for cell in _CELLS: + for trial in range(1, _TRIALS_PER_CELL + 1): + for arm in _ARMS: + run_order += 1 + cid = f"omn-13118-tierb-{cell}-{arm}-t{trial}" + result = _run_one_trial( + consumer_factory=consumer, + cluster=cluster, + correlation_id=cid, + timeout_seconds=timeout_seconds, + ) + if result is not None and str(result.get("correlation_id")) == cid: + correlated.append((cell, arm, trial)) + arms_with_real_rows.add(arm) + cells_with_real_rows.add(cell) + else: + degenerate.append((cell, arm, trial)) + + total = len(_CELLS) * _TRIALS_PER_CELL * len(_ARMS) + + # Structural proof of the redesign: exactly ONE consumer for the WHOLE + # battery, not 2 per trial. This is the property the five prior fixes + # could not establish (they multiplied #1-7 per trial-topic). + assert cluster.consumers_built == 1, ( + f"the long-lived correlator must build exactly ONE consumer for the " + f"whole battery; built {cluster.consumers_built} " + f"(per-trial ephemeral churn is the defect class being removed)" + ) + + # The COMPLETED-topic HWM CLIMBED (every trial published its terminal) — + # the wedge is a consume-leg correlation failure, not a missing publish. + assert cluster.logs[_COMPLETED_TOPIC].hwm == total, ( + f"every trial's COMPLETED terminal must be published (HWM should be " + f"{total}, got {cluster.logs[_COMPLETED_TOPIC].hwm})" + ) + assert cluster.logs[_FAILED_TOPIC].hwm == 0, ( + "the FAILED topic must stay at HWM 0 (nothing lands there)" + ) + + assert not degenerate, ( + f"{len(degenerate)}/{total} trials produced a degenerate row — the " + "correlator failed to demux the gap-landed COMPLETED terminal to the " + f"registered future. First few: {degenerate[:5]}" + ) + assert len(correlated) == total, ( + f"expected all {total} trials correlated, got {len(correlated)}" + ) + assert cells_with_real_rows == set(_CELLS), ( + f"matrix did not advance through both cells: {cells_with_real_rows}" + ) + assert arms_with_real_rows == set(_ARMS), ( + f"matrix did not advance through both arms: {arms_with_real_rows}" + ) + finally: + consumer.close() + + +@pytest.mark.timeout(60) +def test_correlator_reads_terminal_published_in_open_wait_gap( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Focused repro: a terminal published in the register->wait gap is demuxed. + + The minimal shape of the wedge: open + register the cid BEFORE publish, the + correlated terminal lands in the gap, then wait reads it off the single poll + loop. The session.close() is a no-op and must NOT tear the shared correlator + down (the teardown-races-fetch class the redesign eliminates). + """ + cluster = _Cluster() + _FakeConsumer.cluster = cluster + monkeypatch.setattr( + "omnibase_infra.runtime.service_pattern_b_broker.AIOKafkaConsumer", + _FakeConsumer, + ) + + bus = _KafkaLikeBus() + consumer = make_terminal_event_consumer(event_bus=bus, handler_name=_HANDLER) + cid = "omn-13118-tierb-single-gap-0001" + + try: + session = consumer.open(_COMPLETED_TOPIC) + session.register(cid) + cluster.logs[_COMPLETED_TOPIC].append(cid, _terminal_envelope_json(cid)) + + result = session.wait(cid, timeout_seconds=5.0) + + assert result is not None, ( + "the COMPLETED terminal published in the register->wait gap was not " + "demuxed to the registered future by the long-lived poll loop" + ) + assert str(result.get("correlation_id")) == cid + # The shared consumer survives the per-session close (no per-trial churn). + assert cluster.consumers_built == 1 + session.close() + + # A second trial reuses the SAME consumer (no rebuild). + cid2 = "omn-13118-tierb-single-gap-0002" + session2 = consumer.open(_COMPLETED_TOPIC) + session2.register(cid2) + cluster.logs[_COMPLETED_TOPIC].append(cid2, _terminal_envelope_json(cid2)) + result2 = session2.wait(cid2, timeout_seconds=5.0) + assert result2 is not None + assert cluster.consumers_built == 1, ( + "a second trial must reuse the long-lived consumer, not rebuild it" + ) + finally: + consumer.close() + + +@pytest.mark.timeout(60) +def test_correlator_single_call_form_correlates( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The legacy ``(topic, cid, timeout)`` callable form still correlates. + + Open + register + wait in one step against the shared correlator. The + terminal arrives DURING the wait (a correlated terminal published at-or-after + the pinned end offset), and the always-running poll loop demuxes it to the + registered future. (The single-call form pins the read position at call time, + so a record published in the wait window is delivered; a record produced + strictly BEFORE the call is past the pinned end — the documented race-prone + legacy behavior the two-phase open/register path avoids.) + """ + import threading as _threading + + cluster = _Cluster() + _FakeConsumer.cluster = cluster + monkeypatch.setattr( + "omnibase_infra.runtime.service_pattern_b_broker.AIOKafkaConsumer", + _FakeConsumer, + ) + + bus = _KafkaLikeBus() + consumer = make_terminal_event_consumer(event_bus=bus, handler_name=_HANDLER) + cid = "omn-13118-tierb-single-call-0001" + try: + # Publish the correlated terminal shortly AFTER the single-call wait + # begins, so it lands at-or-after the pinned end offset (the in-window + # delivery the poll loop demuxes). + def _publish_during_wait() -> None: + time.sleep(0.2) + cluster.logs[_COMPLETED_TOPIC].append(cid, _terminal_envelope_json(cid)) + + _threading.Thread(target=_publish_during_wait, daemon=True).start() + result = consumer(_COMPLETED_TOPIC, cid, 5.0) + assert result is not None + assert str(result.get("correlation_id")) == cid + finally: + consumer.close() diff --git a/tests/integration/test_volume_config_drift_provenance_e2e.py b/tests/integration/test_volume_config_drift_provenance_e2e.py new file mode 100644 index 0000000000..78fca03099 --- /dev/null +++ b/tests/integration/test_volume_config_drift_provenance_e2e.py @@ -0,0 +1,176 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""End-to-end coverage for the volume-config drift + re-seed cycle (OMN-12958). + +Reproduces the OMN-12945 defect mechanically: the deployed Bifrost delegation +contract lives on a Docker named volume that survives image rebuilds, so the +volume copy can silently diverge from the packaged source. This test wires the +real runtime modules together against a temp dir that stands in for the volume: + +1. render the contract from packaged source into the "volume" target, +2. mutate the volume copy so it drifts from source, +3. compute provenance and assert drift is detected + logged + sidecared, +4. assert the health check degrades on drift (operator-visible signal), +5. re-render (the deploy re-seed) and assert the volume copy is back in-sync. + +Unlike the per-module unit tests, this exercises render -> provenance -> sidecar +-> health as one flow against on-disk artifacts, which is the surface the deploy +procedure and drift sweep actually depend on. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from omnibase_infra.runtime.config_provenance import ( + PROVENANCE_SIDECAR_NAME, + build_config_provenance, + write_provenance_sidecar, +) +from omnibase_infra.runtime.health.health_config_provenance import ( + check_config_provenance_health, +) +from omnibase_infra.runtime.render_bifrost_delegation_contract import ( + render_bifrost_delegation_contract, +) + +_SOURCE_CONTRACT: dict[str, object] = { + "backends": [ + { + "backend_id": "local-primary", + "model_name": "qwen3-coder", + "endpoint_url": "", + "base_url_env": "OMN_INT_DRIFT_PRIMARY_URL", + "required": True, + } + ], + "routing_rules": [ + {"rule_id": "default", "backend_ids": ["local-primary"]}, + ], + "default_backends": ["local-primary"], +} + +_PRIMARY_URL = "http://192.0.2.10:8000/v1/chat/completions" + + +def _write_source(tmp_path: Path) -> Path: + source = tmp_path / "packaged" / "bifrost_delegation.yaml" + source.parent.mkdir(parents=True, exist_ok=True) + source.write_text(yaml.safe_dump(_SOURCE_CONTRACT, sort_keys=False), "utf-8") + return source + + +async def test_volume_config_drift_detected_and_reseeded(tmp_path: Path) -> None: + source = _write_source(tmp_path) + volume_target = tmp_path / "volume" / "delegation" / "bifrost_delegation.yaml" + env = {"OMN_INT_DRIFT_PRIMARY_URL": _PRIMARY_URL} + + # 1. Initial render seeds the volume copy from packaged source. + rendered = render_bifrost_delegation_contract( + source_path=source, + target_path=volume_target, + environ=env, + verify_endpoints=False, + ) + assert rendered == volume_target + assert volume_target.exists() + + provenance = build_config_provenance( + config_name="bifrost_delegation", + deployed_path=volume_target, + source_path=source, + ) + # Rendered output is derived from (not byte-identical to) source — both must + # be present and the health check must not be unhealthy on a fresh seed. + assert provenance.deployed_present is True + assert provenance.source_present is True + seed_health = check_config_provenance_health(provenance) + assert seed_health.status in {"healthy", "degraded"} + assert seed_health.status != "unhealthy" + + # 2. Drift the volume copy (simulate a hand-mutated stale binding that + # survives a rebuild — the exact OMN-12945 mechanism). + drifted = yaml.safe_load(volume_target.read_text("utf-8")) + drifted["backends"][0]["model_name"] = "stale-model-from-jun-9" + volume_target.write_text(yaml.safe_dump(drifted, sort_keys=False), "utf-8") + + drift_prov = build_config_provenance( + config_name="bifrost_delegation", + deployed_path=volume_target, + source_path=source, + ) + assert drift_prov.has_drifted is True + assert drift_prov.deployed_sha256 != drift_prov.source_sha256 + assert "DRIFT" in drift_prov.provenance_line() + + # 3. The sidecar records drift so the sweep + proof packets read it directly. + sidecar = write_provenance_sidecar(drift_prov, deployed_path=volume_target) + assert sidecar.name == PROVENANCE_SIDECAR_NAME + sidecar_payload = json.loads(sidecar.read_text("utf-8")) + assert sidecar_payload["deployed_sha256"] == drift_prov.deployed_sha256 + assert sidecar_payload["source_sha256"] == drift_prov.source_sha256 + + # 4. Health degrades on drift — operator-visible, not a silent failure. + drift_health = check_config_provenance_health(drift_prov) + assert drift_health.status == "degraded" + assert drift_health.details["has_drifted"] is True + assert "re-seed required" in (drift_health.error or "") + + # 5. Re-render is the deploy re-seed: the volume copy is rebuilt from source + # and drift is cleared with zero manual volume mutation. + reseeded = render_bifrost_delegation_contract( + source_path=source, + target_path=volume_target, + environ=env, + verify_endpoints=False, + ) + assert reseeded == volume_target + post_reseed = build_config_provenance( + config_name="bifrost_delegation", + deployed_path=volume_target, + source_path=source, + ) + post_data = yaml.safe_load(volume_target.read_text("utf-8")) + assert post_data["backends"][0]["model_name"] == "qwen3-coder" + assert check_config_provenance_health(post_reseed).status != "unhealthy" + + +async def test_absent_volume_config_is_unhealthy(tmp_path: Path) -> None: + """A missing deployed copy is unhealthy: the runtime has no config to load.""" + source = _write_source(tmp_path) + missing_volume = tmp_path / "volume" / "delegation" / "bifrost_delegation.yaml" + + provenance = build_config_provenance( + config_name="bifrost_delegation", + deployed_path=missing_volume, + source_path=source, + ) + assert provenance.deployed_present is False + assert provenance.has_drifted is False # absence is not drift + + health = check_config_provenance_health(provenance) + assert health.status == "unhealthy" + + +def test_empty_bifrost_contract_path_disables_render_no_provenance( + tmp_path: Path, +) -> None: + """Empty BIFROST_CONTRACT_PATH disables rendering — the documented kill switch. + + Guards the second competing authority OMN-12958 calls out: an empty + BIFROST_CONTRACT_PATH silently disables the packaged-source render entirely. + The renderer must return None (no volume copy materialized) rather than + writing to the default volume path. + """ + source = _write_source(tmp_path) + rendered = render_bifrost_delegation_contract( + source_path=source, + target_path=None, + environ={"BIFROST_CONTRACT_PATH": " "}, + verify_endpoints=False, + ) + assert rendered is None diff --git a/tests/scripts/test_check_sibling_lock_pins.py b/tests/scripts/test_check_sibling_lock_pins.py new file mode 100644 index 0000000000..2da77bb77e --- /dev/null +++ b/tests/scripts/test_check_sibling_lock_pins.py @@ -0,0 +1,257 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Compatibility tests for sibling lock-pin ratchet scripts (OMN-12977).""" + +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path +from typing import Any + +import pytest + +pytestmark = pytest.mark.unit + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_SCRIPT = _REPO_ROOT / "scripts" / "runtime_build" / "check_sibling_lock_pins.py" +_PROVENANCE_SCRIPT = ( + _REPO_ROOT / "scripts" / "runtime_build" / "compute_workspace_provenance.py" +) +_PIN_SCRIPT = _REPO_ROOT / "scripts" / "runtime_build" / "resolve_workspace_pins.py" + +_LOCK_INFRA_REV = "e2dbdc950540df8bc59ca4370b2d4a0f5b8d6c59" +_LOCK_CORE_REV = "c97c2c9a45c5fb0def5fb7dacfd5f01278bb9f55" + + +def _load_module() -> Any: + mod_name = "check_sibling_lock_pins_script_tests" + spec = importlib.util.spec_from_file_location(mod_name, _SCRIPT) + assert spec is not None + mod = importlib.util.module_from_spec(spec) + sys.modules[mod_name] = mod + assert spec.loader is not None + spec.loader.exec_module(mod) # type: ignore[attr-defined] + return mod + + +_mod = _load_module() + + +LOCK_FIXTURE = f"""\ +[[package]] +name = "omnibase-core" +version = "0.44.0" +source = {{ git = "https://github.com/OmniNode-ai/omnibase_core.git?rev={_LOCK_CORE_REV}#{_LOCK_CORE_REV}" }} + +[[package]] +name = "omnibase-infra" +version = "0.38.1" +source = {{ git = "https://github.com/OmniNode-ai/omnibase_infra.git?rev={_LOCK_INFRA_REV}#{_LOCK_INFRA_REV}" }} + +[[package]] +name = "omnibase-spi" +version = "0.20.6" +source = {{ registry = "https://pypi.org/simple" }} + +[[package]] +name = "omnimarket" +version = "0.4.3" +source = {{ editable = "." }} +dependencies = [ + {{ name = "omnibase-infra", git = "https://github.com/OmniNode-ai/omnibase_infra.git?rev={_LOCK_INFRA_REV}" }}, +] +""" + + +def _write_lock(path: Path) -> Path: + path.write_text(LOCK_FIXTURE, encoding="utf-8") + return path + + +def _make_clone(tmp_path: Path, name: str, version: str) -> Path: + root = tmp_path / name + root.mkdir() + (root / "pyproject.toml").write_text( + f'[project]\nname = "{name}"\nversion = "{version}"\n', + encoding="utf-8", + ) + subprocess.run(["git", "init", "-q", str(root)], check=True) + subprocess.run(["git", "-C", str(root), "add", "-A"], check=True) + subprocess.run( + [ + "git", + "-C", + str(root), + "-c", + "user.email=t@example.test", + "-c", + "user.name=t", + "commit", + "-qm", + "init", + ], + check=True, + ) + return root + + +def test_parse_lock_pins_extracts_requested_git_and_registry_pins() -> None: + pins = _mod.parse_lock_pins( + LOCK_FIXTURE, + packages=["omnibase-infra", "omnibase-core", "omnibase-spi"], + ) + + assert pins["omnibase-infra"].version == "0.38.1" + assert pins["omnibase-infra"].git_rev == _LOCK_INFRA_REV + assert pins["omnibase-core"].git_rev == _LOCK_CORE_REV + assert pins["omnibase-spi"].git_rev is None + + +def test_parse_lock_pins_ignores_dependency_revs_for_editable_package() -> None: + pins = _mod.parse_lock_pins(LOCK_FIXTURE, packages=["omnimarket"]) + + assert pins["omnimarket"].version == "0.4.3" + assert pins["omnimarket"].git_rev is None + + +def test_parse_lock_pins_missing_requested_package_raises() -> None: + with pytest.raises(KeyError): + _mod.parse_lock_pins(LOCK_FIXTURE, packages=["omnibase-infra", "missing"]) + + +def test_check_pins_passes_registry_version_match(tmp_path: Path) -> None: + lock = _write_lock(tmp_path / "uv.lock") + clone = _make_clone(tmp_path, "omnibase-spi", "0.20.6") + out = tmp_path / "pins.json" + + rc = _mod.check_pins(lock, {"omnibase-spi": clone}, output_path=out) + + assert rc == 0 + payload = json.loads(out.read_text(encoding="utf-8")) + assert payload["drift_count"] == 0 + assert payload["comparisons"][0]["package"] == "omnibase-spi" + + +def test_check_pins_fails_registry_version_drift(tmp_path: Path) -> None: + lock = _write_lock(tmp_path / "uv.lock") + clone = _make_clone(tmp_path, "omnibase-spi", "0.19.0") + out = tmp_path / "pins.json" + + rc = _mod.check_pins(lock, {"omnibase-spi": clone}, output_path=out) + + assert rc == 1 + payload = json.loads(out.read_text(encoding="utf-8")) + assert payload["drift_count"] == 1 + assert payload["comparisons"][0]["drift_direction"] == "backward" + + +def test_cli_writes_comparison_output(tmp_path: Path) -> None: + lock = _write_lock(tmp_path / "uv.lock") + clone = _make_clone(tmp_path, "omnibase-spi", "0.20.6") + out = tmp_path / "pins.json" + + result = subprocess.run( + [ + sys.executable, + str(_SCRIPT), + "--lock", + str(lock), + "--repo", + f"omnibase-spi={clone}", + "--output", + str(out), + ], + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + payload = json.loads(out.read_text(encoding="utf-8")) + assert payload["drift_count"] == 0 + + +def _write_patched_provenance_script( + tmp_path: Path, + sibling_dir: Path, + manifest_path: Path, + comparison_path: Path, +) -> Path: + script_src = _PROVENANCE_SCRIPT.read_text(encoding="utf-8") + script_src = ( + script_src.replace( + 'SIBLING_REPOS_DIR = Path("/workspace/sibling-repos")', + f'SIBLING_REPOS_DIR = Path("{sibling_dir}")', + ) + .replace( + 'OUTPUT_MANIFEST = Path("/app/build-provenance.json")', + f'OUTPUT_MANIFEST = Path("{manifest_path}")', + ) + .replace( + 'PIN_COMPARISON_PATH = Path("/workspace/sibling-pin-comparison.json")', + f'PIN_COMPARISON_PATH = Path("{comparison_path}")', + ) + ) + patched = tmp_path / "compute_workspace_provenance_patched.py" + patched.write_text(script_src, encoding="utf-8") + (tmp_path / "resolve_workspace_pins.py").write_text( + _PIN_SCRIPT.read_text(encoding="utf-8"), + encoding="utf-8", + ) + return patched + + +def _make_sibling_dirs(root: Path) -> None: + for repo in ("omnibase_compat", "onex_change_control", "omnimarket"): + repo_root = root / repo + repo_root.mkdir(parents=True) + (repo_root / "pyproject.toml").write_text( + f'[project]\nname = "{repo}"\nversion = "0.1.0"\n', + encoding="utf-8", + ) + + +def test_provenance_script_folds_sibling_pin_comparison(tmp_path: Path) -> None: + sibling_dir = tmp_path / "workspace" / "sibling-repos" + _make_sibling_dirs(sibling_dir) + manifest_path = tmp_path / "build-provenance.json" + comparison_path = tmp_path / "sibling-pin-comparison.json" + pin_payload = { + "lock_source": "uv.lock", + "allow_drift": False, + "drift_count": 0, + "comparisons": [{"package": "omnibase-infra", "matches": True}], + } + comparison_path.write_text(json.dumps(pin_payload), encoding="utf-8") + patched = _write_patched_provenance_script( + tmp_path, sibling_dir, manifest_path, comparison_path + ) + + result = subprocess.run( + [sys.executable, str(patched)], capture_output=True, text=True, check=False + ) + + assert manifest_path.exists(), result.stderr + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + assert manifest["sibling_pin_comparison"] == pin_payload + + +def test_provenance_script_uses_none_when_pin_comparison_absent(tmp_path: Path) -> None: + sibling_dir = tmp_path / "workspace" / "sibling-repos" + _make_sibling_dirs(sibling_dir) + manifest_path = tmp_path / "build-provenance.json" + comparison_path = tmp_path / "missing-sibling-pin-comparison.json" + patched = _write_patched_provenance_script( + tmp_path, sibling_dir, manifest_path, comparison_path + ) + + result = subprocess.run( + [sys.executable, str(patched)], capture_output=True, text=True, check=False + ) + + assert manifest_path.exists(), result.stderr + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + assert manifest["sibling_pin_comparison"] is None diff --git a/tests/scripts/test_deploy_runtime_build_context.py b/tests/scripts/test_deploy_runtime_build_context.py index 37103ed04a..611f8e54e0 100644 --- a/tests/scripts/test_deploy_runtime_build_context.py +++ b/tests/scripts/test_deploy_runtime_build_context.py @@ -6,6 +6,7 @@ from __future__ import annotations import os +import re import subprocess from pathlib import Path @@ -15,6 +16,35 @@ DEPLOY_SCRIPT = REPO_ROOT / "scripts" / "deploy-runtime.sh" DOCKERFILE = REPO_ROOT / "docker" / "Dockerfile.runtime" +# Matches a COPY directive's argument list. We discard `--from=` lines +# (those copy from a prior build stage, not the host build context) and any +# remaining `--flag` tokens (e.g. `--chown=...`), then keep the source operands +# that pull from the workspace/ tree. +_COPY_LINE_RE = re.compile(r"^COPY\s+(?P.+)$", re.MULTILINE) + + +def _dockerfile_workspace_copy_sources() -> list[str]: + """Every Dockerfile.runtime COPY source that pulls from the workspace/ tree. + + The deployed build context is assembled by deploy-runtime.sh's sync_files; + each of these paths must be rsynced (or generated) into that context or the + workspace-mode `docker build` fails with "failed to calculate checksum ...: + not found" (the OMN-12987 regression). This list is derived from the live + Dockerfile so a future COPY workspace/ without a matching rsync is caught. + """ + dockerfile = DOCKERFILE.read_text(encoding="utf-8") + sources: list[str] = [] + for match in _COPY_LINE_RE.finditer(dockerfile): + tokens = match.group("args").split() + # `--from=` copies from a build stage, never the host context. + if any(tok.startswith("--from=") for tok in tokens): + continue + # Drop flag tokens (--chown=, --link, ...); the last operand is the + # destination, everything before it is a build-context source. + operands = [tok for tok in tokens if not tok.startswith("--")] + sources.extend(src for src in operands[:-1] if src.startswith("workspace/")) + return sources + @pytest.mark.unit def test_deploy_runtime_syncs_runtime_dockerfile_copy_sources() -> None: @@ -33,6 +63,47 @@ def test_deploy_runtime_syncs_runtime_dockerfile_copy_sources() -> None: assert '"${repo_root}/scripts/runtime_build/"' in deploy_script +@pytest.mark.unit +def test_deploy_runtime_stages_every_workspace_copy_source() -> None: + """Every `COPY workspace/` in Dockerfile.runtime must be staged. + + Regression guard for OMN-12987: Dockerfile.runtime COPYs + workspace/sibling-pin-comparison.json (and workspace/sibling-repos/), but an + earlier deploy-runtime.sh only rsynced workspace/sibling-repos/ into the + deployed build context. The root-level comparison file was never carried + over, so every workspace-mode `docker build` failed with "failed to + calculate checksum of ref ...:/workspace/sibling-pin-comparison.json: not + found". The dev compose build masked this because it runs from the repo root + where the committed placeholder exists. + + This test derives the workspace/ COPY sources from the live Dockerfile and + asserts deploy-runtime.sh references each as an rsync source for the deployed + context, so a future Dockerfile COPY without a matching rsync fails CI. + """ + deploy_script = DEPLOY_SCRIPT.read_text(encoding="utf-8") + + workspace_sources = _dockerfile_workspace_copy_sources() + # The Dockerfile must at minimum COPY the two known workspace/ paths; a regex + # that silently matched nothing would make this guard vacuously pass. + assert "workspace/sibling-repos/" in workspace_sources + assert "workspace/sibling-pin-comparison.json" in workspace_sources + + missing: list[str] = [] + for source in workspace_sources: + # A directory source (trailing slash) and a file source both appear in + # deploy-runtime.sh as an rsync argument quoted under ${repo_root}. + staged = f'"${{repo_root}}/{source}"' in deploy_script + if not staged: + missing.append(source) + + assert not missing, ( + "Dockerfile.runtime COPYs these workspace/ paths but deploy-runtime.sh " + f"does not stage them into the deployed build context: {missing}. Add an " + "rsync of each into sync_files() or workspace-mode `docker build` will " + "fail with 'failed to calculate checksum ...: not found' (OMN-12987)." + ) + + def _init_git_repo(path: Path, marker: str) -> str: path.mkdir(parents=True) (path / "marker.txt").write_text(marker, encoding="utf-8") @@ -142,3 +213,75 @@ def test_deploy_runtime_stages_workspace_and_passes_omni_home_arg() -> None: '--build-arg "EXPECTED_BUILD_SOURCE=${expected_build_source}"' in deploy_script ) assert '--build-arg "OMNI_HOME=${omni_home}"' in deploy_script + + +@pytest.mark.unit +def test_deploy_runtime_runs_sibling_lock_pin_preflight() -> None: + """Workspace staging must run the OMN-12987 lock-pin preflight before build. + + Recurrence guard: the 2026-06-11 stability crash shipped a 13-day-stale + infra 0.37.0 because the build ignored the omnimarket dev lock. The deploy + script must invoke check_sibling_lock_pins after staging and abort on + failure. + """ + deploy_script = DEPLOY_SCRIPT.read_text(encoding="utf-8") + + # The preflight is called from stage_workspace_if_needed after staging. + assert 'check_sibling_lock_pins "${repo_root}" "${omni_home}"' in deploy_script + # The preflight function references the guard script and aborts on failure. + assert "scripts/runtime_build/check_sibling_lock_pins.py" in deploy_script + assert "Refusing to build a stale image." in deploy_script + + +@pytest.mark.unit +def test_deploy_runtime_uses_current_lock_pin_preflight_interface() -> None: + """The preflight caller must match check_sibling_lock_pins.py's current CLI. + + Regression guard: OMN-12977/12987 replaced the original ``--provenance-out`` + flag with ``--lock`` (required pin authority), repeatable ``--repo + PACKAGE=PATH`` (the canonical clones the build vendors), and ``--output`` + (where to write the comparison JSON). The deploy-runtime.sh caller was left + pinned to the removed ``--provenance-out`` flag, so EVERY workspace + ``--execute`` deploy failed at argparse (``the following arguments are + required: --lock``) before any build started. This test pins the corrected + interface so the stale-flag invocation can never come back. + """ + deploy_script = DEPLOY_SCRIPT.read_text(encoding="utf-8") + + # Removed flag must be gone — its presence is the exact regression we hit. + assert "--provenance-out" not in deploy_script + + # Current required + supported flags must be wired into the guard_args. + assert '--lock "${lock_path}"' in deploy_script + assert "--repo " in deploy_script + assert '--output "${provenance_out}"' in deploy_script + + # The consuming repo's omnimarket uv.lock is the pin authority, and every + # vendored sibling must be passed as a --repo PACKAGE=PATH entry. + assert 'lock_path="${omni_home}/omnimarket/uv.lock"' in deploy_script + for package in ( + "omnibase-infra", + "omnibase-core", + "omnibase-spi", + "omnibase-compat", + "onex-change-control", + ): + assert f'--repo "{package}=' in deploy_script + + # The OMN-12977 operator override must be honored via --allow-drift. + assert "ALLOW_SIBLING_PIN_DRIFT" in deploy_script + assert "--allow-drift" in deploy_script + + +@pytest.mark.unit +def test_stage_workspace_emits_build_sha_marker() -> None: + """stage_workspace.sh must record each sibling's HEAD SHA (OMN-12987). + + rsync drops .git, so without a .build-sha marker the staged tree has no + recoverable SHA and the lock-pin preflight cannot verify the vendored commit. + """ + stage_script = ( + DEPLOY_SCRIPT.parent / "runtime_build" / "stage_workspace.sh" + ).read_text(encoding="utf-8") + + assert 'git -C "${src}" rev-parse HEAD > "${dst}/.build-sha"' in stage_script diff --git a/tests/unit/cli/test_cli_delegate.py b/tests/unit/cli/test_cli_delegate.py new file mode 100644 index 0000000000..1de2db7bf8 --- /dev/null +++ b/tests/unit/cli/test_cli_delegate.py @@ -0,0 +1,253 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Tests for ``onex delegate`` single-command delegation (OMN-13096). + +The acceptance probe is STRUCTURAL (no size assertions, plan Phase 2 item 1): + +- ``classify_task_type`` maps prompt keywords to the delegate task taxonomy + (first match wins, research fallback); +- ``run_delegate`` writes its scratch payload under ``/tmp/`` with + a run_id-suffixed name — never ``/tmp`` (``feedback_no_tmp_use_workspace``); +- the payload validates against the delegate node's input model + (``ModelDelegateSkillRequest``) — prompt, task_type, source, max_tokens; +- the command dispatches through receipt mode so stdout is exactly ONE + ``ModelSkillResult`` JSON with zero RuntimeLocal log leakage. + +The end-to-end probe against the live delegate node (which requires a vLLM +endpoint) lives in the OCC evidence run, not the unit suite. These unit tests +exercise the REAL CLI wiring against a committed proof contract by pointing +``_resolve_packaged_contract`` at it — the dispatch path, payload write, and +receipt envelope are all real. +""" + +from __future__ import annotations + +import json +import tempfile +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from omnibase_core.models.dispatch.model_skill_result import ModelSkillResult +from omnibase_infra.cli import cli_delegate +from omnibase_infra.cli.cli_delegate import ( + DEFAULT_MAX_TOKENS, + DEFAULT_TASK_TYPE, + DELEGATE_SOURCE, + classify_task_type, + delegate_command, + run_delegate, +) + +pytestmark = pytest.mark.unit + +# A proof contract that runs a deterministic in-process handler — no vLLM, no +# network. It stands in for the delegate node so the CLI wiring (payload write, +# receipt-mode dispatch, single typed result) is exercised end-to-end. +_PROOF_NOOP_CONTRACT = ( + "---\n" + "name: proof_noop\n" + "node_type: compute\n" + "terminal_event: onex.evt.proof.noop-completed.v1\n" + "handler:\n" + " module: tests.fixtures.handler_proof_noop\n" + " class: HandlerProofNoop\n" + " input_model: tests.fixtures.handler_proof_noop.ModelProofNoopRequest\n" + "handler_routing:\n" + " default_handler: tests.fixtures.handler_proof_noop:HandlerProofNoop\n" +) + + +class TestClassifyTaskType: + @pytest.mark.parametrize( + ("prompt", "expected"), + [ + ("write unit tests for verify.py", "test"), + ("add a pytest for the parser", "test"), + ("document the routing module", "document"), + ("write a docstring for this fn", "document"), + ("refactor the dispatch loop", "refactor"), + ("simplify the config parsing", "refactor"), + ("review this PR for correctness", "review"), + ("audit the auth flow", "review"), + ("reason through the tradeoffs", "reasoning"), + ("compare two architectures", "reasoning"), + ("implement an HTTP server", "code_generation"), + ("build a CLI scaffold", "code_generation"), + ("what does a calendar app need", DEFAULT_TASK_TYPE), + ], + ) + def test_keyword_mapping(self, prompt: str, expected: str) -> None: + assert classify_task_type(prompt) == expected + + def test_first_match_wins_test_before_code_generation(self) -> None: + # "write" maps to code_generation, "test" maps to test; "test" rule is + # ordered first, so a prompt with both classifies as test. + assert classify_task_type("write a test for the handler") == "test" + + def test_case_insensitive(self) -> None: + assert classify_task_type("REFACTOR the LOOP") == "refactor" + + +class TestPayloadScratch: + def test_payload_written_under_state_root_tmp_not_slash_tmp( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + contract_path = tmp_path / "contract.yaml" + contract_path.write_text(_PROOF_NOOP_CONTRACT, encoding="utf-8") + monkeypatch.setattr( + cli_delegate, + "_resolve_packaged_contract", + lambda _name: contract_path, + ) + monkeypatch.setenv("ONEX_ARTIFACT_STORE_ROOT", str(tmp_path / "artifacts")) + state_root = tmp_path / "state" + + run_delegate( + prompt="implement an HTTP server", + task_type=None, + max_tokens=DEFAULT_MAX_TOKENS, + state_root=state_root, + timeout=60, + verbose=False, + emit_socket=tmp_path / "no-daemon.sock", + ) + + scratch_dir = state_root / "tmp" + assert scratch_dir.is_dir(), "scratch dir must be under /tmp/" + payloads = list(scratch_dir.glob("delegate-input-*.json")) + assert len(payloads) == 1, "exactly one run_id-suffixed scratch payload" + # No scratch leaked to the system temp dir. + assert not list(Path(tempfile.gettempdir()).glob("delegate-input-*.json")) + + def test_payload_validates_against_delegate_request_model( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + contract_path = tmp_path / "contract.yaml" + contract_path.write_text(_PROOF_NOOP_CONTRACT, encoding="utf-8") + monkeypatch.setattr( + cli_delegate, + "_resolve_packaged_contract", + lambda _name: contract_path, + ) + monkeypatch.setenv("ONEX_ARTIFACT_STORE_ROOT", str(tmp_path / "artifacts")) + state_root = tmp_path / "state" + + run_delegate( + prompt="refactor the loop", + task_type=None, + max_tokens=4096, + state_root=state_root, + timeout=60, + verbose=False, + emit_socket=tmp_path / "no-daemon.sock", + ) + + payload_path = next((state_root / "tmp").glob("delegate-input-*.json")) + payload = json.loads(payload_path.read_text(encoding="utf-8")) + # The payload carries exactly the fields the delegate node's input + # model (ModelDelegateSkillRequest) requires from a consumer: prompt, + # task_type, source, max_tokens. omnibase_infra does NOT depend on + # omnimarket (layering), so the node owns model validation at dispatch; + # the CLI's contract here is the payload shape. + assert payload == { + "prompt": "refactor the loop", + "task_type": "refactor", + "source": DELEGATE_SOURCE, + "max_tokens": 4096, + } + + def test_explicit_task_type_overrides_classification( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + contract_path = tmp_path / "contract.yaml" + contract_path.write_text(_PROOF_NOOP_CONTRACT, encoding="utf-8") + monkeypatch.setattr( + cli_delegate, + "_resolve_packaged_contract", + lambda _name: contract_path, + ) + monkeypatch.setenv("ONEX_ARTIFACT_STORE_ROOT", str(tmp_path / "artifacts")) + state_root = tmp_path / "state" + + # Prompt would classify as code_generation; explicit flag wins. + run_delegate( + prompt="write an HTTP server", + task_type="research", + max_tokens=DEFAULT_MAX_TOKENS, + state_root=state_root, + timeout=60, + verbose=False, + emit_socket=tmp_path / "no-daemon.sock", + ) + payload_path = next((state_root / "tmp").glob("delegate-input-*.json")) + payload = json.loads(payload_path.read_text(encoding="utf-8")) + assert payload["task_type"] == "research" + + +class TestSingleReceiptOnStdout: + def test_stdout_is_exactly_one_validated_skill_result( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + contract_path = tmp_path / "contract.yaml" + contract_path.write_text(_PROOF_NOOP_CONTRACT, encoding="utf-8") + monkeypatch.setattr( + cli_delegate, + "_resolve_packaged_contract", + lambda _name: contract_path, + ) + monkeypatch.setenv("ONEX_ARTIFACT_STORE_ROOT", str(tmp_path / "artifacts")) + state_root = tmp_path / "state" + + runner = CliRunner() + result = runner.invoke( + delegate_command, + [ + "implement an HTTP server", + "--state-root", + str(state_root), + "--emit-socket", + str(tmp_path / "no-daemon.sock"), + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0, result.output + stripped = result.stdout.strip() + # Exactly one JSON object — any RuntimeLocal log line would break this. + parsed = json.loads(stripped) + assert isinstance(parsed, dict) + assert "\n" not in stripped, "receipt must be a single JSON line" + ModelSkillResult.model_validate(parsed) + + def test_no_runtime_info_logs_on_stdout( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + contract_path = tmp_path / "contract.yaml" + contract_path.write_text(_PROOF_NOOP_CONTRACT, encoding="utf-8") + monkeypatch.setattr( + cli_delegate, + "_resolve_packaged_contract", + lambda _name: contract_path, + ) + monkeypatch.setenv("ONEX_ARTIFACT_STORE_ROOT", str(tmp_path / "artifacts")) + state_root = tmp_path / "state" + + runner = CliRunner() + result = runner.invoke( + delegate_command, + [ + "research the routing architecture", + "--state-root", + str(state_root), + "--emit-socket", + str(tmp_path / "no-daemon.sock"), + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0, result.output + assert "INFO" not in result.stdout + assert "RuntimeLocal" not in result.stdout diff --git a/tests/unit/cli/test_cli_node.py b/tests/unit/cli/test_cli_node.py index c4ce0fb935..4f046b2e08 100644 --- a/tests/unit/cli/test_cli_node.py +++ b/tests/unit/cli/test_cli_node.py @@ -14,6 +14,7 @@ from __future__ import annotations import importlib.util +import json from importlib.machinery import ModuleSpec from importlib.metadata import entry_points from pathlib import Path @@ -176,6 +177,55 @@ class _FakeEP: find_spec.assert_called_once_with("fake_pkg") +def test_default_mode_failure_emits_skill_routing_error_envelope( + tmp_path: Path, +) -> None: + """A non-zero default-mode run emits a ``SkillRoutingError`` JSON envelope. + + The ``golden_chain_sweep`` skill (and others) document that ``onex node`` + surfaces a ``SkillRoutingError`` JSON envelope on non-zero exit rather than a + raw traceback (OMN-8724). This drives a FAILED run via a bogus handler module + and asserts the documented envelope is on stdout with the expected keys. + """ + node_name = _first_resolvable_packaged_node_name() + if node_name is None: + pytest.skip("No resolvable onex.nodes packaged contract registered in this env") + + override = tmp_path / "custom_contract.yaml" + override.write_text( + "---\n" + "name: custom\n" + "node_type: compute\n" + "handler:\n" + " module: does.not.exist.anywhere_zzzqqq\n" + " class: Nope\n", + encoding="utf-8", + ) + runner = CliRunner() + result = runner.invoke( + run_node_by_name, + [ + node_name, + "--contract", + str(override), + "--state-root", + str(tmp_path / "state"), + "--timeout", + "5", + ], + ) + + assert result.exit_code == 1 + # The last JSON object in stdout is the SkillRoutingError envelope. + start = result.output.rfind("{") + assert start != -1, f"no JSON envelope in output: {result.output!r}" + envelope = json.loads(result.output[start:]) + assert envelope["error_type"] == "SkillRoutingError" + assert envelope["node_id"] == node_name + assert envelope["result"] == "failed" + assert envelope["message"] + + def test_input_file_not_found_surfaces_cli_error(tmp_path: Path) -> None: """``--input `` must fail at the CLI layer with a Click-level error.""" runner = CliRunner() diff --git a/tests/unit/cli/test_cli_node_receipt.py b/tests/unit/cli/test_cli_node_receipt.py new file mode 100644 index 0000000000..06adc2d4f6 --- /dev/null +++ b/tests/unit/cli/test_cli_node_receipt.py @@ -0,0 +1,341 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Tests for ``onex node --output receipt`` (OMN-13094). + +Acceptance is STRUCTURAL, not size-based (plan Open Question 3 resolution): + +- stdout parses as exactly ONE JSON object validating against + ``ModelSkillResult`` — zero intermediate context leaks; +- zero RuntimeLocal INFO lines on stdout/stderr; +- the full capture log and handler result are content-addressed in the + artifact store and hash-verified on retrieval; +- ``artifact.captured`` + ``tool.output.captured`` are emitted via the emit + daemon socket, or spooled locally when the daemon is unreachable; +- artifact-write failure prints the FULL output instead of a receipt + (no hidden loss); +- node failure produces ``status=failed`` with the full error output inline. + +These tests run the REAL dispatch path: the actual click command through the +actual RuntimeLocal with the in-memory bus and the committed proof-noop +fixture handler (no mocked runtime). +""" + +from __future__ import annotations + +import json +import shutil +import socket +import tempfile +import threading +from pathlib import Path + +import pytest +from click.testing import CliRunner, Result + +from omnibase_core.artifacts.artifact_store import ArtifactStore +from omnibase_core.models.artifacts.model_artifact_ref import ModelArtifactRef +from omnibase_core.models.dispatch.model_skill_result import ModelSkillResult +from omnibase_infra.cli.cli_node import run_node_by_name +from omnibase_infra.cli.receipt_mode import CAPTURE_DIR_NAME, SPOOL_DIR_NAME + +pytestmark = pytest.mark.unit + +_PROOF_NOOP_CONTRACT = ( + "---\n" + "name: proof_noop\n" + "node_type: compute\n" + "terminal_event: onex.evt.proof.noop-completed.v1\n" + "handler:\n" + " module: tests.fixtures.handler_proof_noop\n" + " class: HandlerProofNoop\n" + " input_model: tests.fixtures.handler_proof_noop.ModelProofNoopRequest\n" + "handler_routing:\n" + " default_handler: tests.fixtures.handler_proof_noop:HandlerProofNoop\n" +) + +_BROKEN_CONTRACT = ( + "---\n" + "name: broken_node\n" + "node_type: compute\n" + "terminal_event: onex.evt.proof.broken-completed.v1\n" + "handler:\n" + " module: tests.fixtures.does_not_exist_module\n" + " class: HandlerDoesNotExist\n" + "handler_routing:\n" + " default_handler: tests.fixtures.does_not_exist_module:HandlerDoesNotExist\n" +) + + +def _write_fixture_inputs(tmp_path: Path, contract_text: str) -> tuple[Path, Path]: + contract_path = tmp_path / "contract.yaml" + contract_path.write_text(contract_text, encoding="utf-8") + input_path = tmp_path / "input.json" + input_path.write_text( + json.dumps({"name": "receipt-proof", "count": 3}), encoding="utf-8" + ) + return contract_path, input_path + + +def _invoke_receipt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + *, + contract_text: str = _PROOF_NOOP_CONTRACT, + store_root: Path | None | str = "default", + socket_path: Path | None = None, +) -> tuple[Result, Path]: + """Run the real CLI in receipt mode against the proof-noop fixture.""" + contract_path, input_path = _write_fixture_inputs(tmp_path, contract_text) + state_root = tmp_path / "state" + + if store_root == "default": + store_root = tmp_path / "artifacts" + if store_root is None: + monkeypatch.delenv("ONEX_ARTIFACT_STORE_ROOT", raising=False) + else: + monkeypatch.setenv("ONEX_ARTIFACT_STORE_ROOT", str(store_root)) + + resolved_socket = socket_path or (tmp_path / "no-daemon.sock") + + runner = CliRunner() + result = runner.invoke( + run_node_by_name, + [ + "proof_noop", + "--contract", + str(contract_path), + "--input", + str(input_path), + "--state-root", + str(state_root), + "--output", + "receipt", + "--emit-socket", + str(resolved_socket), + ], + catch_exceptions=False, + ) + return result, state_root + + +def _parse_single_receipt(stdout: str) -> dict[str, object]: + """stdout must be exactly one JSON object — anything else is a leak.""" + stripped = stdout.strip() + parsed: object = json.loads(stripped) # raises if anything else leaked + assert isinstance(parsed, dict) + # Round-trip guard: the single line IS the whole stdout payload. + assert stripped.startswith("{") and stripped.endswith("}") + assert "\n" not in stripped, "receipt must be a single JSON line" + return parsed + + +class TestReceiptModeSuccess: + def test_stdout_is_exactly_one_validated_skill_result( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + result, _ = _invoke_receipt(tmp_path, monkeypatch) + assert result.exit_code == 0, result.output + payload = _parse_single_receipt(result.stdout) + receipt = ModelSkillResult.model_validate(payload) + assert receipt.node_name == "proof_noop" + assert receipt.status.is_success_like + assert receipt.exit_code == 0 + # The FULL handler result travels in the receipt, untruncated. + assert payload["result"] == { + "status": "success", + "echoed_name": "receipt-proof", + "echoed_count": 3, + } + assert str(payload["result_model"]).endswith("ModelProofNoopResult") + assert receipt.artifact_refs, "capture log must be artifact-backed" + + def test_zero_runtime_log_lines_on_stdout_or_stderr( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + result, state_root = _invoke_receipt(tmp_path, monkeypatch) + assert result.exit_code == 0 + combined = result.stdout + result.stderr + assert "RuntimeLocal" not in combined + assert "INFO" not in combined + # The runtime stream went to the run_id-suffixed capture file instead. + captures = list((state_root / CAPTURE_DIR_NAME).glob("proof_noop-*.log")) + assert len(captures) == 1 + capture_text = captures[0].read_text(encoding="utf-8") + assert "RuntimeLocal" in capture_text + + def test_artifacts_are_retrievable_and_hash_verified( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + result, state_root = _invoke_receipt(tmp_path, monkeypatch) + assert result.exit_code == 0 + payload = _parse_single_receipt(result.stdout) + refs = payload["artifact_refs"] + assert isinstance(refs, list) and len(refs) == 2 # capture log + result + + store = ArtifactStore() + blobs = [] + for raw_ref in refs: + assert isinstance(raw_ref, dict) + ref = ModelArtifactRef.model_validate(raw_ref) + # read_blob re-hashes and raises on mismatch — retrieval IS the + # hash verification. + blobs.append(store.read_blob(ref)) + meta = store.read_meta(ref) + assert meta["source_system"] == "onex_cli" + + capture_file = next((state_root / CAPTURE_DIR_NAME).glob("proof_noop-*.log")) + assert capture_file.read_bytes() in blobs + assert json.dumps(payload["result"]).encode("utf-8") in blobs + + def test_events_spooled_when_daemon_unreachable( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + result, state_root = _invoke_receipt(tmp_path, monkeypatch) + assert result.exit_code == 0 + # Receipt still printed — telemetry failure never re-floods the caller. + _parse_single_receipt(result.stdout) + + spool_dir = state_root / SPOOL_DIR_NAME + artifact_events = sorted(spool_dir.glob("artifact-captured-*.json")) + tool_events = sorted(spool_dir.glob("tool-output-captured-*.json")) + assert len(artifact_events) == 2 + assert len(tool_events) == 1 + + for spool_file in artifact_events: + record = json.loads(spool_file.read_text(encoding="utf-8")) + assert record["event_type"] == "artifact.captured" + for field in ( + "artifact_ref", + "artifact_hash", + "artifact_size_bytes", + "artifact_kind", + "source_system", + "correlation_id", + ): + assert field in record["payload"], field + + tool_record = json.loads(tool_events[0].read_text(encoding="utf-8")) + assert tool_record["event_type"] == "tool.output.captured" + for field in ("tool_name", "suppression_decision", "correlation_id"): + assert field in tool_record["payload"], field + assert tool_record["payload"]["suppression_decision"] == "receipt_mode" + + def test_events_emitted_when_daemon_available( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # UDS paths are capped (~104 chars on macOS); pytest tmp_path under + # xdist can exceed that, so the socket gets its own short tempdir. + socket_dir = Path(tempfile.mkdtemp(prefix="omn13094-")) + socket_path = socket_dir / "emit.sock" + if len(str(socket_path)) > 100: + pytest.skip("temp dir too long for a Unix domain socket") + + received: list[dict[str, object]] = [] + server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + server.bind(str(socket_path)) + server.listen(8) + server.settimeout(10.0) + stop = threading.Event() + + def serve() -> None: + while not stop.is_set(): + try: + conn, _ = server.accept() + except (TimeoutError, OSError): + return + with conn: + data = b"" + while not data.endswith(b"\n"): + chunk = conn.recv(4096) + if not chunk: + break + data += chunk + if data: + received.append(json.loads(data.decode("utf-8"))) + conn.sendall(b'{"status": "queued", "event_id": "test"}\n') + + thread = threading.Thread(target=serve, daemon=True) + thread.start() + try: + result, state_root = _invoke_receipt( + tmp_path, monkeypatch, socket_path=socket_path + ) + finally: + stop.set() + server.close() + thread.join(timeout=5.0) + shutil.rmtree(socket_dir, ignore_errors=True) + + assert result.exit_code == 0 + _parse_single_receipt(result.stdout) + event_types = sorted(str(r["event_type"]) for r in received) + assert event_types == [ + "artifact.captured", + "artifact.captured", + "tool.output.captured", + ] + # Daemon accepted everything — nothing may be spooled. + spool_dir = state_root / SPOOL_DIR_NAME + assert not spool_dir.exists() or not list(spool_dir.iterdir()) + + +class TestReceiptModeFailureAsymmetry: + def test_artifact_write_failure_prints_full_output_not_receipt( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Missing ONEX_ARTIFACT_STORE_ROOT ⇒ FULL output, no receipt.""" + result, _ = _invoke_receipt(tmp_path, monkeypatch, store_root=None) + # The run itself succeeded; capture failed; nothing may be hidden. + assert result.exit_code == 0 + assert "no hidden loss" in result.stderr + # Full capture stream passes through instead of a receipt. + assert "RuntimeLocal" in result.stdout + with pytest.raises(json.JSONDecodeError): + json.loads(result.stdout.strip()) + + def test_node_failure_reports_failed_with_error_inline( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + result, _ = _invoke_receipt( + tmp_path, monkeypatch, contract_text=_BROKEN_CONTRACT + ) + assert result.exit_code != 0 + payload = _parse_single_receipt(result.stdout) + receipt = ModelSkillResult.model_validate(payload) + assert not receipt.status.is_success_like + result_body = payload["result"] + assert isinstance(result_body, dict) + # Errors are never hidden: the full capture log travels inline. + assert result_body["capture_log"], "failure must inline the capture log" + assert str(payload["result_model"]).endswith("ModelReceiptRuntimeSummary") + # The capture log is ADDITIONALLY artifact-backed. + assert payload["artifact_refs"] + + +class TestDefaultModeUnchanged: + def test_default_mode_does_not_print_receipt_json( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + contract_path, input_path = _write_fixture_inputs( + tmp_path, _PROOF_NOOP_CONTRACT + ) + monkeypatch.delenv("ONEX_ARTIFACT_STORE_ROOT", raising=False) + runner = CliRunner() + result = runner.invoke( + run_node_by_name, + [ + "proof_noop", + "--contract", + str(contract_path), + "--input", + str(input_path), + "--state-root", + str(tmp_path / "state"), + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + # No single-JSON receipt contract in default mode (behavior preserved). + with pytest.raises(json.JSONDecodeError): + json.loads(result.stdout.strip() or "not-json") diff --git a/tests/unit/cli/test_cli_skill.py b/tests/unit/cli/test_cli_skill.py new file mode 100644 index 0000000000..c806b9285d --- /dev/null +++ b/tests/unit/cli/test_cli_skill.py @@ -0,0 +1,391 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Tests for ``onex skill `` (OMN-13097). + +The skill subcommand resolves the declarative skill→node mapping +(``skill_mapping.yaml``), builds the backing node's input payload from the +skill's CLI args, and dispatches through the proven receipt-mode path. These +tests assert the DECLARATIVE LAYER (registry validity, arg parsing, payload +construction, classifiers) directly, plus the command wiring through a stubbed +``run_receipt_mode`` so we verify the constructed payload without standing up +a live runtime. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from omnibase_infra.cli import cli_skill +from omnibase_infra.cli.cli_skill import ( + _apply_classifiers, + _parse_skill_args, + load_skill_registry, + run_skill_by_name, +) +from omnibase_infra.cli.enum_skill_arg_type import EnumSkillArgType +from omnibase_infra.cli.model_skill_arg_spec import ModelSkillArgSpec +from omnibase_infra.cli.model_skill_classifier import ModelSkillClassifier +from omnibase_infra.cli.model_skill_mapping import ModelSkillMapping +from omnibase_infra.cli.model_skill_mapping_registry import ModelSkillMappingRegistry + +pytestmark = pytest.mark.unit + +# The 24 dispatch shims this ticket migrates (ticket deliverable 1). +_EXPECTED_SKILLS = frozenset( + { + "aislop_sweep", + "data_flow_sweep", + "auto_merge", + "build_loop", + "coderabbit_triage", + "compliance_sweep", + "coverage_sweep", + "create_ticket", + "database_sweep", + "design_to_plan", + "doc_freshness_sweep", + "dod_verify", + "duplication_sweep", + "hostile_reviewer", + "linear_housekeeping", + "merge_sweep", + "plan_to_tickets", + "platform_readiness", + "pr_polish", + "pr_review", + "pr_review_bot", + "session", + "shim_audit", + "delegate", + } +) + + +@pytest.fixture(autouse=True) +def _clear_registry_cache() -> None: + """The registry loader is lru_cached; clear it per test for isolation.""" + load_skill_registry.cache_clear() + + +def test_registry_loads_and_validates() -> None: + registry = load_skill_registry() + assert isinstance(registry, ModelSkillMappingRegistry) + assert len(registry.skills) >= len(_EXPECTED_SKILLS) + + +def test_registry_covers_all_24_dispatch_shims() -> None: + registry = load_skill_registry() + declared = {s.skill_name for s in registry.skills} + missing = _EXPECTED_SKILLS - declared + assert not missing, f"mapping is missing skills: {sorted(missing)}" + + +def test_registry_result_models_are_fully_qualified() -> None: + registry = load_skill_registry() + for skill in registry.skills: + assert "." in skill.result_model, ( + f"{skill.skill_name}: result_model must be a fully qualified name, " + f"got {skill.result_model!r}" + ) + assert skill.node_name.startswith("node_"), skill.node_name + + +def test_registry_rejects_duplicate_skill_names() -> None: + with pytest.raises(ValueError, match="duplicate skill_name"): + ModelSkillMappingRegistry( + skills=( + ModelSkillMapping( + skill_name="dup", + node_name="node_x", + result_model="a.B", + ), + ModelSkillMapping( + skill_name="dup", + node_name="node_y", + result_model="a.C", + ), + ) + ) + + +def test_mapping_rejects_multiple_positionals() -> None: + with pytest.raises(ValueError, match="at most one positional"): + ModelSkillMapping( + skill_name="s", + node_name="node_x", + result_model="a.B", + args=( + ModelSkillArgSpec( + name="a", + payload_field="a", + arg_type=EnumSkillArgType.STRING, + positional=True, + ), + ModelSkillArgSpec( + name="b", + payload_field="b", + arg_type=EnumSkillArgType.STRING, + positional=True, + ), + ), + ) + + +def test_arg_spec_required_forbids_default() -> None: + with pytest.raises(ValueError, match="required args must not declare a default"): + ModelSkillArgSpec( + name="x", + payload_field="x", + arg_type=EnumSkillArgType.STRING, + required=True, + default="oops", + ) + + +def _mapping_with(*args: ModelSkillArgSpec, **kw: object) -> ModelSkillMapping: + return ModelSkillMapping( + skill_name="t", + node_name="node_t", + result_model="a.B", + args=tuple(args), + **kw, # type: ignore[arg-type] + ) + + +def test_parse_string_list_and_boolean() -> None: + mapping = _mapping_with( + ModelSkillArgSpec( + name="repos", + payload_field="repos", + arg_type=EnumSkillArgType.STRING_LIST, + ), + ModelSkillArgSpec( + name="dry-run", + payload_field="dry_run", + arg_type=EnumSkillArgType.BOOLEAN, + default=False, + ), + ) + payload = _parse_skill_args(mapping, ("--repos", "a, b ,c", "--dry-run")) + assert payload == {"repos": ["a", "b", "c"], "dry_run": True} + + +def test_parse_applies_boolean_default_when_omitted() -> None: + mapping = _mapping_with( + ModelSkillArgSpec( + name="dry-run", + payload_field="dry_run", + arg_type=EnumSkillArgType.BOOLEAN, + default=False, + ), + ) + assert _parse_skill_args(mapping, ()) == {"dry_run": False} + + +def test_parse_integer_coercion_and_failure() -> None: + mapping = _mapping_with( + ModelSkillArgSpec( + name="pr-number", + payload_field="pr_number", + arg_type=EnumSkillArgType.INTEGER, + ), + ) + assert _parse_skill_args(mapping, ("--pr-number", "42")) == {"pr_number": 42} + from click import ClickException + + with pytest.raises(ClickException, match="expects an integer"): + _parse_skill_args(mapping, ("--pr-number", "notanint")) + + +def test_parse_positional_joins_tokens() -> None: + mapping = _mapping_with( + ModelSkillArgSpec( + name="prompt", + payload_field="prompt", + arg_type=EnumSkillArgType.STRING, + positional=True, + required=True, + ), + ) + assert _parse_skill_args(mapping, ("summarize", "this", "text")) == { + "prompt": "summarize this text" + } + + +def test_parse_missing_required_fails() -> None: + from click import ClickException + + mapping = _mapping_with( + ModelSkillArgSpec( + name="repo", + payload_field="repo", + arg_type=EnumSkillArgType.STRING, + required=True, + ), + ) + with pytest.raises(ClickException, match="requires --repo"): + _parse_skill_args(mapping, ()) + + +def test_parse_unknown_flag_fails() -> None: + from click import ClickException + + mapping = _mapping_with() + with pytest.raises(ClickException, match="Unknown argument '--nope'"): + _parse_skill_args(mapping, ("--nope", "x")) + + +def test_static_payload_merged() -> None: + mapping = _mapping_with( + ModelSkillArgSpec( + name="prompt", + payload_field="prompt", + arg_type=EnumSkillArgType.STRING, + positional=True, + required=True, + ), + static_payload={"source": "claude-code"}, + ) + payload = _parse_skill_args(mapping, ("hello",)) + assert payload == {"source": "claude-code", "prompt": "hello"} + + +def test_classifier_assigns_first_match() -> None: + classifier = ModelSkillClassifier( + target_field="task_type", + source_field="prompt", + rules=( + (("test", "pytest"), "test"), + (("write", "implement"), "code_generation"), + ), + fallback="research", + ) + mapping = _mapping_with(classifiers=(classifier,)) + payload: dict[str, object] = {"prompt": "please write a pytest"} + _apply_classifiers(mapping, payload) # type: ignore[arg-type] + # "test"/"pytest" group comes first → wins over "write". + assert payload["task_type"] == "test" + + +def test_classifier_fallback_when_no_match() -> None: + classifier = ModelSkillClassifier( + target_field="task_type", + source_field="prompt", + rules=((("write",), "code_generation"),), + fallback="research", + ) + mapping = _mapping_with(classifiers=(classifier,)) + payload: dict[str, object] = {"prompt": "tell me about the weather"} + _apply_classifiers(mapping, payload) # type: ignore[arg-type] + assert payload["task_type"] == "research" + + +def test_classifier_does_not_override_explicit_value() -> None: + classifier = ModelSkillClassifier( + target_field="task_type", + source_field="prompt", + rules=((("write",), "code_generation"),), + fallback="research", + ) + mapping = _mapping_with(classifiers=(classifier,)) + payload: dict[str, object] = {"prompt": "write code", "task_type": "document"} + _apply_classifiers(mapping, payload) # type: ignore[arg-type] + assert payload["task_type"] == "document" + + +def test_delegate_mapping_classifies_and_builds_payload() -> None: + """End-to-end declarative-layer check for the worst-case skill.""" + registry = load_skill_registry() + delegate = registry.get("delegate") + assert delegate is not None + payload = _parse_skill_args( + delegate, ("write", "a", "unit", "test", "for", "the", "parser") + ) + _apply_classifiers(delegate, payload) + assert payload["prompt"] == "write a unit test for the parser" + assert payload["source"] == "claude-code" + assert payload["max_tokens"] == 2048 + # "unit test" / "test" keyword group wins. + assert payload["task_type"] == "test" + + +def test_command_unknown_skill_fails() -> None: + runner = CliRunner() + result = runner.invoke(run_skill_by_name, ["does_not_exist"]) + assert result.exit_code != 0 + assert "Unknown skill" in result.output + + +def test_command_dispatches_via_receipt_mode( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The command builds the payload and hands it to run_receipt_mode.""" + captured: dict[str, object] = {} + + def _fake_receipt_mode(**kwargs: object) -> int: + captured.update(kwargs) + # Read back the payload the command wrote. + input_path = kwargs["input_path"] + assert isinstance(input_path, Path) + captured["payload"] = json.loads(input_path.read_text(encoding="utf-8")) + return 0 + + def _fake_resolve(node_name: str) -> Path: + return tmp_path / f"{node_name}-contract.yaml" + + monkeypatch.setattr(cli_skill, "run_receipt_mode", _fake_receipt_mode) + monkeypatch.setattr(cli_skill, "_resolve_packaged_contract", _fake_resolve) + + runner = CliRunner() + result = runner.invoke( + run_skill_by_name, + [ + "compliance_sweep", + "--state-root", + str(tmp_path / "state"), + "--repos", + "omnibase_core,omnibase_infra", + "--dry-run", + ], + ) + assert result.exit_code == 0, result.output + assert captured["node_name"] == "node_compliance_sweep" + payload = captured["payload"] + assert isinstance(payload, dict) + assert payload["repos"] == ["omnibase_core", "omnibase_infra"] + assert payload["dry_run"] is True + assert captured["backend_overrides"] == {"event_bus": "inmemory"} + + +def test_command_writes_payload_under_state_root_not_tmp( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + seen_paths: list[Path] = [] + + def _fake_receipt_mode(**kwargs: object) -> int: + input_path = kwargs["input_path"] + assert isinstance(input_path, Path) + seen_paths.append(input_path) + return 0 + + monkeypatch.setattr(cli_skill, "run_receipt_mode", _fake_receipt_mode) + monkeypatch.setattr( + cli_skill, "_resolve_packaged_contract", lambda n: tmp_path / "c.yaml" + ) + + state_root = tmp_path / "state" + runner = CliRunner() + result = runner.invoke( + run_skill_by_name, + ["platform_readiness", "--state-root", str(state_root)], + ) + assert result.exit_code == 0, result.output + assert len(seen_paths) == 1 + # Payload scratch lives under the state root, never /tmp. + assert state_root in seen_paths[0].parents + assert "tmp" in seen_paths[0].parts diff --git a/tests/unit/contracts/test_protocol_ownership.py b/tests/unit/contracts/test_protocol_ownership.py index dba9c33ec9..b4147d8ec4 100644 --- a/tests/unit/contracts/test_protocol_ownership.py +++ b/tests/unit/contracts/test_protocol_ownership.py @@ -155,6 +155,8 @@ # [NODE] DI boundary for lifecycle event emission — lets the remote-agent effect # test lifecycle publication without coupling the handler to EventBusKafka (OMN-9637). "ProtocolLifecycleEventSink": "nodes/node_remote_agent_invoke_effect/services/lifecycle_event_sink.py", + # [NODE] OMN-12909 gateway forwarder bus adapter surface for local/cloud bus DI. + "ProtocolGatewayBus": "nodes/node_bus_forwarder_effect/services/service_gateway_forwarder.py", # [NODE] OMN-10392 replay compute narrows the AIOKafkaConsumer surface and # Kafka record/key shapes for isolated replay handler tests. "ProtocolKafkaMessage": "nodes/node_kafka_replay_compute/protocols/protocol_kafka_message.py", diff --git a/tests/unit/infra/test_catalog_healthcheck_start_period.py b/tests/unit/infra/test_catalog_healthcheck_start_period.py new file mode 100644 index 0000000000..f90e499f42 --- /dev/null +++ b/tests/unit/infra/test_catalog_healthcheck_start_period.py @@ -0,0 +1,237 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Ratchet tests for migration-gate healthcheck start_period (OMN-12973). + +These tests are the enforcement half of the P2.8 fix. They run in the standard +``uv run pytest tests/`` suite, so they gate every PR (CI) and pre-commit. + +The classification (P2.8): the prod migration-gate was reported UNHEALTHY then +self-resolved to HEALTHY because its healthcheck ``start_period`` (10s) was far +shorter than the real migration window (~2 min on a cold prod volume). The gate +is a correctly-modeled long-running sentinel, NOT a mis-modeled one-shot — the +fix is to widen ``start_period`` so a still-applying gate stays in +``health: starting`` instead of flipping to UNHEALTHY. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from omnibase_infra.docker.catalog.enum_depends_on_condition import ( + EnumDependsOnCondition, +) +from omnibase_infra.docker.catalog.manifest_schema import ( + CatalogManifest, + DependsOnEntry, + HealthCheck, +) +from omnibase_infra.docker.catalog.resolver import CatalogResolver +from omnibase_infra.docker.catalog.validator_healthcheck_start_period import ( + MIGRATION_GATE_START_PERIOD_FLOOR_S, + is_migration_completion_gate, + validate_migration_gate_start_period, +) + +CATALOG_DIR = str(Path(__file__).resolve().parents[3] / "docker" / "catalog") + + +def _gate(start_period_s: int) -> CatalogManifest: + """A minimal migration-completion-gate-shaped manifest for unit testing.""" + return CatalogManifest( + name="migration-gate", + description="test gate", + image="postgres:16-alpine", + layer="infrastructure", + required_env=[], + hardcoded_env={}, + operational_defaults={}, + ports=None, + healthcheck=HealthCheck( + test="sh /check_migrations_complete.sh", + interval_s=10, + timeout_s=5, + retries=30, + start_period_s=start_period_s, + ), + volumes=[], + depends_on=[ + DependsOnEntry( + service="forward-migration", + condition=EnumDependsOnCondition.SERVICE_COMPLETED_SUCCESSFULLY, + ), + ], + ) + + +@pytest.mark.unit +def test_real_catalog_migration_gate_meets_start_period_floor() -> None: + """The shipped migration-gate manifest must satisfy the start_period floor. + + This is the regression lock for the prod fix: if anyone lowers + migration-gate.start_period_s below the floor, this fails. + """ + resolver = CatalogResolver(catalog_dir=CATALOG_DIR) + resolved = resolver.resolve(bundles=["runtime"]) + result = validate_migration_gate_start_period(resolved.manifests) + assert result.ok, result.report() + + +@pytest.mark.unit +def test_migration_gate_is_classified_as_completion_gate() -> None: + """migration-gate must be recognized as a migration-completion gate. + + Guards against the classifier silently no-op'ing (which would let a + too-short start_period slip through unvalidated). + """ + resolver = CatalogResolver(catalog_dir=CATALOG_DIR) + resolved = resolver.resolve(bundles=["runtime"]) + assert "migration-gate" in resolved.manifests + assert is_migration_completion_gate(resolved.manifests["migration-gate"]) + + +@pytest.mark.unit +def test_validator_flags_too_short_start_period() -> None: + """A gate below the floor (the pre-fix 10s value) must be flagged.""" + result = validate_migration_gate_start_period({"migration-gate": _gate(10)}) + assert not result.ok + assert len(result.violations) == 1 + violation = result.violations[0] + assert violation.service == "migration-gate" + assert violation.start_period_s == 10 + assert violation.floor_s == MIGRATION_GATE_START_PERIOD_FLOOR_S + assert "UNHEALTHY" in violation.message() + + +@pytest.mark.unit +def test_validator_passes_at_floor_boundary() -> None: + """A gate exactly at the floor passes (>= semantics).""" + result = validate_migration_gate_start_period( + {"migration-gate": _gate(MIGRATION_GATE_START_PERIOD_FLOOR_S)} + ) + assert result.ok + + +@pytest.mark.unit +def test_non_migration_gate_is_not_flagged() -> None: + """Services without a service_completed_successfully dep are ignored. + + A plain healthcheck service (e.g. postgres) with a short start_period must + not be swept into the migration-gate floor. + """ + postgres = CatalogManifest( + name="postgres", + description="db", + image="postgres:16-alpine", + layer="infrastructure", + required_env=[], + hardcoded_env={}, + operational_defaults={}, + ports=None, + healthcheck=HealthCheck( + test="pg_isready", + interval_s=30, + timeout_s=10, + retries=3, + start_period_s=10, + ), + volumes=[], + depends_on=[], + ) + assert not is_migration_completion_gate(postgres) + result = validate_migration_gate_start_period({"postgres": postgres}) + assert result.ok + + +@pytest.mark.unit +def test_app_service_depending_on_migration_runner_is_not_flagged() -> None: + """An app service whose healthcheck is a liveness probe is NOT a gate. + + Regression for the over-broad classifier (OMN-12973): ``intelligence-api`` + depends on ``intelligence-migration`` via ``service_completed_successfully`` + but its healthcheck is an HTTP liveness probe, not a migration-completion + poll. Its start_period (40s) is legitimately short and must not be swept + into the migration-gate floor. Only the sentinel whose healthcheck literally + polls migration completion is subject to the floor. + """ + resolver = CatalogResolver(catalog_dir=CATALOG_DIR) + resolved = resolver.resolve(bundles=["runtime"]) + assert "intelligence-api" in resolved.manifests + intelligence_api = resolved.manifests["intelligence-api"] + # It does depend on a migration runner completing... + assert any( + dep.condition is EnumDependsOnCondition.SERVICE_COMPLETED_SUCCESSFULLY + for dep in intelligence_api.depends_on + ) + # ...but its healthcheck is a liveness probe, so it is NOT a completion gate. + assert not is_migration_completion_gate(intelligence_api) + result = validate_migration_gate_start_period(resolved.manifests) + assert result.ok, result.report() + + +@pytest.mark.unit +def test_completion_gate_requires_migration_probe_healthcheck() -> None: + """A migration-runner dependant with a non-migration healthcheck is ignored. + + Unit-level guard for the discriminating signal: depending on a + ``service_completed_successfully`` sibling is necessary but not sufficient — + the healthcheck command must also poll migration completion. + """ + app_service = CatalogManifest( + name="some-api", + description="app", + image="runtime:latest", + layer="runtime", + required_env=[], + hardcoded_env={}, + operational_defaults={}, + ports=None, + healthcheck=HealthCheck( + test="curl -sf http://localhost:8053/health", + interval_s=30, + timeout_s=10, + retries=3, + start_period_s=40, + ), + volumes=[], + depends_on=[ + DependsOnEntry( + service="intelligence-migration", + condition=EnumDependsOnCondition.SERVICE_COMPLETED_SUCCESSFULLY, + ), + ], + ) + assert not is_migration_completion_gate(app_service) + result = validate_migration_gate_start_period({"some-api": app_service}) + assert result.ok + + +@pytest.mark.unit +def test_completion_gate_with_list_form_healthcheck_is_classified() -> None: + """The migration probe is detected in CMD (argv-list) healthcheck form too.""" + gate = CatalogManifest( + name="migration-gate", + description="gate", + image="postgres:16-alpine", + layer="infrastructure", + required_env=[], + hardcoded_env={}, + operational_defaults={}, + ports=None, + healthcheck=HealthCheck( + test=["CMD-SHELL", "sh /check_migrations_complete.sh"], + interval_s=10, + timeout_s=5, + retries=30, + start_period_s=180, + ), + volumes=[], + depends_on=[ + DependsOnEntry( + service="forward-migration", + condition=EnumDependsOnCondition.SERVICE_COMPLETED_SUCCESSFULLY, + ), + ], + ) + assert is_migration_completion_gate(gate) diff --git a/tests/unit/infra/test_compose_no_silent_fallbacks.py b/tests/unit/infra/test_compose_no_silent_fallbacks.py index 288437fed2..bb58790b39 100644 --- a/tests/unit/infra/test_compose_no_silent_fallbacks.py +++ b/tests/unit/infra/test_compose_no_silent_fallbacks.py @@ -47,6 +47,15 @@ # OMN-12196: bifrost source contract path; empty = resolve from omnimarket # package via importlib.resources (opt-in override for custom source path) "BIFROST_SOURCE_CONTRACT_PATH", + # OMN-12864: Bifrost local inference endpoints — provided by the committed + # lane overlay (docker/lane-overlays/dev.bifrost.env) at runtime. Compose-level + # :? is intentionally omitted so `docker compose config` renders cleanly in + # CI without the lane overlay pre-loaded. Validation at the Python layer + # (ModelBifrostLaneOverlay + render_bifrost_delegation_contract) is the + # enforcement point for dev-lane deployments. + "BIFROST_LOCAL_CODER_ENDPOINT_URL", + "BIFROST_LOCAL_REASONER_ENDPOINT_URL", + "BIFROST_LOCAL_EMBEDDING_ENDPOINT_URL", } diff --git a/tests/unit/infra/test_runtime_image_identity_labels.py b/tests/unit/infra/test_runtime_image_identity_labels.py new file mode 100644 index 0000000000..289cb27777 --- /dev/null +++ b/tests/unit/infra/test_runtime_image_identity_labels.py @@ -0,0 +1,157 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Enforcement ratchet for OMN-12965 — runtime image identity labels. + +The deep dive found the main runtime image stamped +``org.opencontainers.image.version=0.1.0`` with a blank +``org.opencontainers.image.revision`` after workspace rebuilds. A blank identity +degrades EVERY proof packet, because the runtime SHA + image digest are required +citations in accepted evidence. + +Root cause: the operator rebuild path ``onex up --build`` (``cmd_up`` in +``omnibase_infra.docker.catalog.cli``) passed only ``GIT_SHA`` as a build arg. +The runtime-stage OCI labels read ``VCS_REF`` (-> blank revision) and +``RUNTIME_VERSION`` (-> placeholder ``0.1.0``), neither of which were passed. + +These tests pin the fix so it cannot regress: + +1. ``_image_identity_build_args`` stamps the full identity quad + (GIT_SHA / VCS_REF / RUNTIME_VERSION / BUILD_DATE). +2. ``_image_identity_build_args`` fails fast when the git revision cannot be + resolved, rather than stamping a blank/``unknown`` revision. +3. ``_runtime_version`` resolves the real package version (never the Dockerfile + placeholder). +4. ``Dockerfile.runtime`` carries a workspace-mode guard that fails the build + when VCS_REF or RUNTIME_VERSION would be blank/placeholder. +""" + +from __future__ import annotations + +import re +import tomllib +from pathlib import Path + +import pytest + +from omnibase_infra.docker.catalog import cli + +pytestmark = pytest.mark.unit + +_REPO_ROOT = Path(__file__).resolve().parents[3] +_DOCKERFILE = _REPO_ROOT / "docker" / "Dockerfile.runtime" + +_IDENTITY_ARGS = ("GIT_SHA", "VCS_REF", "RUNTIME_VERSION", "BUILD_DATE") + + +def _parse_build_args(args: list[str]) -> dict[str, str]: + """Collapse a ``--build-arg KEY=VALUE`` list into a dict.""" + parsed: dict[str, str] = {} + it = iter(args) + for token in it: + if token == "--build-arg": + key, _, value = next(it).partition("=") + parsed[key] = value + return parsed + + +def test_runtime_version_matches_pyproject() -> None: + """_runtime_version must return the real package version, not the placeholder.""" + data = tomllib.loads((_REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8")) + expected = str(data["project"]["version"]).strip() + assert cli._runtime_version() == expected + assert cli._runtime_version() != "0.1.0", ( + "runtime version must not be the Dockerfile placeholder 0.1.0 (OMN-12965)" + ) + + +def test_identity_build_args_stamp_full_quad( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The identity quad must be present so the runtime-stage labels populate.""" + monkeypatch.setattr(cli, "_current_git_sha", lambda: "deadbeefcafef00d") + parsed = _parse_build_args(cli._image_identity_build_args()) + + for key in _IDENTITY_ARGS: + assert key in parsed, f"missing --build-arg {key} (OMN-12965)" + + # VCS_REF (-> org.opencontainers.image.revision) must carry the real SHA. + assert parsed["VCS_REF"] == "deadbeefcafef00d" + assert parsed["GIT_SHA"] == "deadbeefcafef00d" + # RUNTIME_VERSION (-> org.opencontainers.image.version) must be the real version. + assert parsed["RUNTIME_VERSION"] == cli._runtime_version() + assert parsed["RUNTIME_VERSION"] != "0.1.0" + # BUILD_DATE (-> org.opencontainers.image.created) must be an RFC3339 stamp. + assert re.fullmatch( + r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", parsed["BUILD_DATE"] + ), parsed["BUILD_DATE"] + + +@pytest.mark.parametrize("bad_sha", ["", "unknown"]) +def test_identity_build_args_fail_fast_on_unresolved_revision( + monkeypatch: pytest.MonkeyPatch, bad_sha: str +) -> None: + """A blank/unknown revision must abort the build, not ship blank identity.""" + monkeypatch.setattr(cli, "_current_git_sha", lambda: bad_sha) + with pytest.raises(RuntimeError, match="git revision"): + cli._image_identity_build_args() + + +def test_cmd_up_build_path_uses_identity_args() -> None: + """cmd_up's --build branch must call the identity-arg helper, not bare GIT_SHA. + + Static guard scoped to the ``cmd_up`` function body: the build branch must + spread ``_image_identity_build_args()`` and must NOT hand-roll a lone + ``GIT_SHA`` build-arg (the pre-fix regression that left VCS_REF + + RUNTIME_VERSION unstamped → blank-identity image, OMN-12965). + """ + source = Path(cli.__file__).read_text(encoding="utf-8") + body = _function_body(source, "cmd_up") + assert "_image_identity_build_args()" in body, ( + "cmd_up --build must stamp the identity quad via _image_identity_build_args()" + ) + assert "GIT_SHA=" not in body, ( + "cmd_up must not hand-roll a GIT_SHA build-arg; delegate to " + "_image_identity_build_args() so VCS_REF + RUNTIME_VERSION are stamped " + "(OMN-12965)." + ) + + +def _function_body(source: str, name: str) -> str: + """Return the source text of a top-level ``def name(...)`` block.""" + lines = source.splitlines() + start = next( + (i for i, line in enumerate(lines) if line.startswith(f"def {name}(")), + None, + ) + assert start is not None, f"function {name} not found in cli.py" + end = next( + ( + i + for i in range(start + 1, len(lines)) + if lines[i] and not lines[i][0].isspace() + ), + len(lines), + ) + return "\n".join(lines[start:end]) + + +def test_dockerfile_workspace_identity_guard_present() -> None: + """Dockerfile.runtime must fail workspace builds with blank/placeholder identity.""" + text = _DOCKERFILE.read_text(encoding="utf-8") + collapsed = re.sub(r"\\\n\s*", " ", text) + + # The runtime stage must label revision from VCS_REF and version from RUNTIME_VERSION. + assert 'org.opencontainers.image.revision="${VCS_REF}"' in collapsed + assert 'org.opencontainers.image.version="${RUNTIME_VERSION}"' in collapsed + + # Workspace-mode guard: reject blank/unknown VCS_REF. + guard = re.search( + r'if \[ "\$\{BUILD_SOURCE\}" = "workspace" \];.*?VCS_REF.*?' + r"exit 64.*?RUNTIME_VERSION.*?0\.1\.0.*?exit 64", + collapsed, + re.DOTALL, + ) + assert guard, ( + "Dockerfile.runtime must guard workspace builds against blank VCS_REF and " + "placeholder RUNTIME_VERSION=0.1.0 (OMN-12965)." + ) diff --git a/tests/unit/infra/test_stability_test_runtime_lane.py b/tests/unit/infra/test_stability_test_runtime_lane.py index 76f8c92fe8..ccb581d00e 100644 --- a/tests/unit/infra/test_stability_test_runtime_lane.py +++ b/tests/unit/infra/test_stability_test_runtime_lane.py @@ -34,7 +34,7 @@ "omnibase-infra-postgres", "omnibase-infra-redpanda", "omnibase-infra-valkey", - "omnibase-forward-migration", + "omnibase-infra-forward-migration", "omnibase-infra-infisical", "omninode-agent-actions-consumer", "omninode-skill-lifecycle-consumer", @@ -142,10 +142,36 @@ def test_stability_lane_runtime_services_do_not_reuse_production_names() -> None @pytest.mark.unit def test_stability_lane_runtime_worker_is_not_scaled_to_zero() -> None: + """OMN-12990: the worker replica count must fail-fast on the ledgered value. + + The base compose default is ``${WORKER_REPLICAS:-0}``; a soft ``:-1`` default + here would silently re-introduce the silent-drop hole on any recreate that + omitted the policy env. The override must reference the contract-rendered + ``STABILITY_TEST_WORKER_REPLICAS`` with the ``:?`` fail-fast form so a missing + policy env aborts the recreate loudly instead of dropping the worker. + """ overlay = _load_overlay() worker = overlay["services"]["runtime-worker"] + replicas = worker["deploy"]["replicas"] - assert worker["deploy"]["replicas"] == "${STABILITY_TEST_WORKER_REPLICAS:-1}" + assert replicas.startswith("${STABILITY_TEST_WORKER_REPLICAS:?") + assert ":-" not in replicas + assert replicas != "${STABILITY_TEST_WORKER_REPLICAS:-0}" + assert replicas != "${STABILITY_TEST_WORKER_REPLICAS:-1}" + + +@pytest.mark.unit +def test_stability_lane_worker_replicas_pinned_in_ledgered_policy_env() -> None: + """OMN-12990: the worker replica count is declared in the ledgered env. + + The compose override is fail-fast on STABILITY_TEST_WORKER_REPLICAS; the value + itself must live in docker/runtime-policy.env (rendered from the runtime policy + contract), pinned to a running worker so a lane recreate preserves it. + """ + policy_env = _load_runtime_policy_env() + + assert policy_env["STABILITY_TEST_WORKER_REPLICAS"] == "1" + assert int(policy_env["STABILITY_TEST_WORKER_REPLICAS"]) >= 1 @pytest.mark.unit diff --git a/tests/unit/migrations/test_migration_gate_vacuity_fix.py b/tests/unit/migrations/test_migration_gate_vacuity_fix.py new file mode 100644 index 0000000000..d8a1f619d7 --- /dev/null +++ b/tests/unit/migrations/test_migration_gate_vacuity_fix.py @@ -0,0 +1,398 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Regression tests for OMN-13062 — migration-gate vacuity fix (retro A-10). + +Three bugs fixed: + +(1) RUNNER SENTINEL DISCIPLINE + run-forward-migrations.sh must clear migrations_complete to FALSE at the + start of every run, and set it TRUE only as the FINAL act after all + migrations succeed. Any mid-run failure must leave the gate UNHEALTHY. + +(2) SYNC-NODE-MIGRATIONS VACUOUS GATE + sync-node-migrations.sh --check must exit nonzero (2) when the + omnimarket source tree is unresolvable. Silently passing hides drift. + SYNC_NODE_MIGRATIONS_SKIP_UNRESOLVABLE=1 provides the single opt-out. + +(3) WAIT-FOR-POSTGRES GUARD + run-forward-migrations.sh must wait for Postgres to be ready before + proceeding, guarding against the first-boot initdb race. + +(4) SKIP-MANIFEST + A committed docker/migrations/skip-manifest.yaml is the sole escape for + intentionally-skipped migrations. The runner reads this at startup. + +Recurrences: OMN-12885, OMN-12934 +""" + +from __future__ import annotations + +import os +import subprocess +import tempfile +import textwrap +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[3] +RUNNER = REPO_ROOT / "scripts" / "run-forward-migrations.sh" +SYNC_SCRIPT = REPO_ROOT / "scripts" / "sync-node-migrations.sh" +SKIP_MANIFEST = REPO_ROOT / "docker" / "migrations" / "skip-manifest.yaml" +MIGRATION_085 = ( + REPO_ROOT + / "docker" + / "migrations" + / "forward" + / "085_add_runner_completed_at_to_db_metadata.sql" +) +ROLLBACK_085 = ( + REPO_ROOT + / "docker" + / "migrations" + / "rollback" + / "rollback_085_add_runner_completed_at_to_db_metadata.sql" +) + +pytestmark = pytest.mark.unit + + +# --------------------------------------------------------------------------- +# (1) Runner sentinel discipline +# --------------------------------------------------------------------------- +class TestRunnerSentinelDiscipline: + """run-forward-migrations.sh clears sentinel at start, sets at end only.""" + + def test_runner_clears_sentinel_at_start(self) -> None: + script = RUNNER.read_text(encoding="utf-8") + # The runner must set migrations_complete = FALSE before any migration. + assert "migrations_complete = FALSE" in script, ( + "run-forward-migrations.sh must clear migrations_complete=FALSE " + "at the start of every run (OMN-13062)" + ) + + def test_runner_sets_sentinel_only_at_end(self) -> None: + script = RUNNER.read_text(encoding="utf-8") + # The sentinel-set TRUE block must come AFTER the node-migration loop. + node_discovery_pos = script.index("Node-owned migration auto-discovery") + sentinel_set_pos = script.index("Setting sentinel TRUE") + assert sentinel_set_pos > node_discovery_pos, ( + "The sentinel TRUE set must be the runner's final act, " + "after all infra and node migrations complete (OMN-13062)" + ) + + def test_runner_sets_runner_completed_at(self) -> None: + script = RUNNER.read_text(encoding="utf-8") + assert "runner_completed_at = NOW()" in script, ( + "run-forward-migrations.sh must stamp runner_completed_at " + "as its final act (OMN-13062)" + ) + + def test_runner_uses_on_error_stop_for_sentinel_set(self) -> None: + script = RUNNER.read_text(encoding="utf-8") + # Find the sentinel-set block at the end. + sentinel_pos = script.rindex("Setting sentinel TRUE") + block_after = script[sentinel_pos:] + assert "ON_ERROR_STOP=1" in block_after, ( + "The final sentinel-set psql call must use -v ON_ERROR_STOP=1 " + "so a DB error at that point still fails the runner (OMN-13062)" + ) + + def test_runner_migration_failure_leaves_gate_unhealthy(self) -> None: + """A failing migration must exit nonzero (via set -e + ON_ERROR_STOP).""" + script = RUNNER.read_text(encoding="utf-8") + # The runner uses set -e; psql uses -v ON_ERROR_STOP=1 for each apply. + assert "set -e" in script, ( + "run-forward-migrations.sh must use 'set -e' so any psql failure " + "aborts the runner before the sentinel is set (OMN-13062)" + ) + assert "-v ON_ERROR_STOP=1 -f" in script, ( + "Each migration apply must use -v ON_ERROR_STOP=1 so a SQL error " + "causes psql to exit nonzero and the runner to abort (OMN-13062)" + ) + + +# --------------------------------------------------------------------------- +# (2) sync-node-migrations --check vacuous gate +# --------------------------------------------------------------------------- +class TestSyncNodeMigrationsCheckVacuity: + """sync-node-migrations.sh --check must exit nonzero when source is absent.""" + + def test_check_mode_exits_nonzero_when_source_unresolvable(self) -> None: + """--check with no source must exit 2, not 0.""" + result = subprocess.run( + ["bash", str(SYNC_SCRIPT), "--check"], + cwd=str(REPO_ROOT), + env={ + k: v + for k, v in os.environ.items() + # Unset OMNI_HOME and OMNIMARKET_SRC to make source unresolvable. + if k not in ("OMNI_HOME", "OMNIMARKET_SRC") + }, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert result.returncode == 2, ( + f"sync-node-migrations.sh --check must exit 2 when omnimarket " + f"source is unresolvable (OMN-13062 vacuous-gate sibling fix). " + f"Got returncode={result.returncode}.\n" + f"stderr: {result.stderr}\nstdout: {result.stdout}" + ) + + def test_check_mode_skip_env_allows_exit_zero(self) -> None: + """SYNC_NODE_MIGRATIONS_SKIP_UNRESOLVABLE=1 provides the single opt-out.""" + result = subprocess.run( + ["bash", str(SYNC_SCRIPT), "--check"], + cwd=str(REPO_ROOT), + env={ + k: v + for k, v in os.environ.items() + if k not in ("OMNI_HOME", "OMNIMARKET_SRC") + } + | {"SYNC_NODE_MIGRATIONS_SKIP_UNRESOLVABLE": "1"}, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert result.returncode == 0, ( + f"sync-node-migrations.sh --check with " + f"SYNC_NODE_MIGRATIONS_SKIP_UNRESOLVABLE=1 must exit 0. " + f"Got returncode={result.returncode}.\n" + f"stderr: {result.stderr}\nstdout: {result.stdout}" + ) + + def test_sync_script_documents_exit_code_2_for_unresolvable(self) -> None: + script = SYNC_SCRIPT.read_text(encoding="utf-8") + assert "exit 2" in script, ( + "sync-node-migrations.sh must document and implement exit code 2 " + "for unresolvable omnimarket source (OMN-13062)" + ) + + def test_sync_script_has_skip_unresolvable_env_escape(self) -> None: + script = SYNC_SCRIPT.read_text(encoding="utf-8") + assert "SYNC_NODE_MIGRATIONS_SKIP_UNRESOLVABLE" in script, ( + "sync-node-migrations.sh must honour " + "SYNC_NODE_MIGRATIONS_SKIP_UNRESOLVABLE=1 as the single escape " + "for --check when source is absent (OMN-13062)" + ) + + +# --------------------------------------------------------------------------- +# (3) wait-for-postgres guard +# --------------------------------------------------------------------------- +class TestWaitForPostgresGuard: + """run-forward-migrations.sh must wait for Postgres before proceeding.""" + + def test_runner_has_postgres_wait_loop(self) -> None: + script = RUNNER.read_text(encoding="utf-8") + assert "Waiting for Postgres" in script, ( + "run-forward-migrations.sh must wait for Postgres before " + "applying migrations (OMN-13062 first-boot initdb race guard)" + ) + + def test_runner_wait_loop_has_retry_limit(self) -> None: + script = RUNNER.read_text(encoding="utf-8") + assert "PG_WAIT_RETRIES" in script, ( + "The Postgres wait loop must honour PG_WAIT_RETRIES so the " + "runner eventually exits nonzero on a stuck Postgres (OMN-13062)" + ) + + def test_runner_postgres_wait_exits_nonzero_on_timeout( + self, tmp_path: Path + ) -> None: + """A fake psql that always fails triggers the retry limit and exits 1.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + fake_psql = bin_dir / "psql" + fake_psql.write_text("#!/bin/sh\nexit 1\n", encoding="utf-8") + fake_psql.chmod(0o755) + + # Use an empty migrations dir so the runner stops after the wait loop. + migrations_dir = tmp_path / "migrations" + migrations_dir.mkdir() + + result = subprocess.run( + ["sh", str(RUNNER)], + check=False, + env={ + # Minimal env — no real Postgres vars so fake psql handles all calls. + "POSTGRES_PASSWORD": "test", + "POSTGRES_HOST": "127.0.0.1", + "POSTGRES_PORT": "65534", # unused port + "MIGRATIONS_DIR": str(migrations_dir), + "NODE_MIGRATIONS_DIR": str(tmp_path / "no-nodes"), + "PG_WAIT_RETRIES": "2", # fast timeout for tests + "PATH": f"{bin_dir}{os.pathsep}{os.environ.get('PATH', '/usr/bin:/bin')}", + "HOME": os.environ.get("HOME", str(tmp_path)), + }, + text=True, + capture_output=True, + timeout=30, + ) + + assert result.returncode != 0, ( + "run-forward-migrations.sh must exit nonzero when Postgres is not " + "reachable within PG_WAIT_RETRIES retries (OMN-13062)" + ) + assert "not ready" in result.stdout or "Aborting" in result.stdout, ( + f"Expected 'not ready' or 'Aborting' in runner output.\n" + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + + +# --------------------------------------------------------------------------- +# (4) Skip-manifest +# --------------------------------------------------------------------------- +class TestSkipManifest: + """The committed skip-manifest.yaml is the sole escape for skipped migrations.""" + + def test_skip_manifest_exists(self) -> None: + assert SKIP_MANIFEST.exists(), ( + f"docker/migrations/skip-manifest.yaml must exist (OMN-13062): {SKIP_MANIFEST}" + ) + + def test_skip_manifest_has_skipped_migrations_key(self) -> None: + content = SKIP_MANIFEST.read_text(encoding="utf-8") + assert "skipped_migrations:" in content, ( + "skip-manifest.yaml must contain a 'skipped_migrations:' key (OMN-13062)" + ) + + def test_runner_reads_skip_manifest(self) -> None: + script = RUNNER.read_text(encoding="utf-8") + assert "skip-manifest.yaml" in script, ( + "run-forward-migrations.sh must read skip-manifest.yaml (OMN-13062)" + ) + + def test_runner_has_is_skipped_by_manifest_function(self) -> None: + script = RUNNER.read_text(encoding="utf-8") + assert "is_skipped_by_manifest" in script, ( + "run-forward-migrations.sh must implement is_skipped_by_manifest " + "to check each migration against the skip-manifest (OMN-13062)" + ) + + def test_runner_honours_skip_manifest_in_migration_loop( + self, tmp_path: Path + ) -> None: + """A migration listed in skip-manifest.yaml must not be executed.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + + # psql that exits 0 on SELECT 1 (wait guard), and records calls. + call_log = tmp_path / "psql_calls.log" + fake_psql = bin_dir / "psql" + fake_psql.write_text( + textwrap.dedent(f"""\ + #!/bin/sh + printf '%s\\n' "$*" >> "{call_log}" + # If the -c arg contains SELECT 1, succeed (wait guard). + if echo "$*" | grep -q 'SELECT 1'; then + exit 0 + fi + # CREATE TABLE for schema_migrations — succeed. + if echo "$*" | grep -q 'CREATE TABLE'; then + exit 0 + fi + # INSERT INTO schema_migrations — succeed. + if echo "$*" | grep -q 'INSERT INTO'; then + exit 0 + fi + # DO block for sentinel clear — succeed. + if echo "$*" | grep -q 'BEGIN'; then + exit 0 + fi + # Actual -f migration file — succeed. + exit 0 + """), + encoding="utf-8", + ) + fake_psql.chmod(0o755) + + migrations_dir = tmp_path / "migrations" / "forward" + migrations_dir.mkdir(parents=True) + + # One migration that is listed in the skip-manifest. + skip_sql = migrations_dir / "001_should_be_skipped.sql" + skip_sql.write_text("SELECT 'THIS SHOULD NOT RUN';\n", encoding="utf-8") + + # skip-manifest one level above forward/. + skip_manifest_path = tmp_path / "migrations" / "skip-manifest.yaml" + skip_manifest_path.write_text( + textwrap.dedent("""\ + skipped_migrations: + - id: "docker/001_should_be_skipped.sql" + reason: "test skip" + ticket: "OMN-13062" + """), + encoding="utf-8", + ) + + result = subprocess.run( + ["sh", str(RUNNER)], + check=False, + env={ + # Minimal env — deliberately exclude real POSTGRES_HOST so + # the runner uses its default "postgres" (fake psql handles it). + "POSTGRES_PASSWORD": "test", + "POSTGRES_HOST": "localhost", + "POSTGRES_PORT": "5432", + "MIGRATIONS_DIR": str(migrations_dir), + "NODE_MIGRATIONS_DIR": str(tmp_path / "no-nodes"), + "PG_WAIT_RETRIES": "1", + # Put fake psql first so it shadows any real psql. + "PATH": f"{bin_dir}{os.pathsep}{os.environ.get('PATH', '/usr/bin:/bin')}", + "HOME": os.environ.get("HOME", str(tmp_path)), + }, + text=True, + capture_output=True, + timeout=30, + ) + + assert "skip-manifest" in result.stdout, ( + f"Runner must log 'skip-manifest' for the skipped migration.\n" + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + # The SQL file must not have been passed to psql with -f. + # The fake psql logs every call; check that the migration file never + # appears in a -f argument (it was skipped, not executed). + calls = call_log.read_text(encoding="utf-8") if call_log.exists() else "" + file_executed = f"-f {skip_sql}" in calls or f"-f {skip_sql.name}" in calls + assert not file_executed, ( + f"Skip-manifest migration must not be passed to psql with -f.\n" + f"psql calls logged:\n{calls}" + ) + + +# --------------------------------------------------------------------------- +# (5) Migration 085 — runner_completed_at schema +# --------------------------------------------------------------------------- +class TestMigration085RunnerCompletedAt: + """Migration 085 adds runner_completed_at column to db_metadata.""" + + def test_migration_085_exists(self) -> None: + assert MIGRATION_085.exists(), f"Migration 085 must exist: {MIGRATION_085}" + + def test_rollback_085_exists(self) -> None: + assert ROLLBACK_085.exists(), ( + f"Rollback for migration 085 must exist: {ROLLBACK_085}" + ) + + def test_migration_085_is_idempotent(self) -> None: + sql = MIGRATION_085.read_text(encoding="utf-8") + assert "IF NOT EXISTS" in sql, ( + "Migration 085 must use ADD COLUMN IF NOT EXISTS for idempotency" + ) + + def test_migration_085_adds_runner_completed_at(self) -> None: + sql = MIGRATION_085.read_text(encoding="utf-8") + assert "runner_completed_at" in sql + assert "TIMESTAMPTZ" in sql.upper() + + def test_migration_085_does_not_create_tables(self) -> None: + sql = MIGRATION_085.read_text(encoding="utf-8").upper() + assert "CREATE TABLE" not in sql, ( + "Migration 085 must only ALTER db_metadata, not create new tables" + ) diff --git a/tests/unit/models/test_model_serialization_roundtrip.py b/tests/unit/models/test_model_serialization_roundtrip.py index 6e5b8660d7..a18313155d 100644 --- a/tests/unit/models/test_model_serialization_roundtrip.py +++ b/tests/unit/models/test_model_serialization_roundtrip.py @@ -188,6 +188,9 @@ "ModelLocalStateStoreEntry": "Internal entry model for ModelLocalStateStore; covered by test_node_invocation_adapter.py", "ModelPluginDiscoveryEntry": "Converted from dataclass in OMN-12184; covered by test_model_plugin_discovery_report.py", "ModelPluginDiscoveryReport": "Converted from dataclass in OMN-12184; covered by test_model_plugin_discovery_report.py", + # OMN-12864: Bifrost lane overlay model — round-trip covered by + # tests/unit/runtime/models/test_model_bifrost_lane_overlay.py + "ModelBifrostLaneOverlay": "Typed deployment binding for Bifrost local endpoints; covered by test_model_bifrost_lane_overlay.py", } diff --git a/tests/unit/nodes/node_bus_forwarder_effect/__init__.py b/tests/unit/nodes/node_bus_forwarder_effect/__init__.py new file mode 100644 index 0000000000..f8bfe4ae6b --- /dev/null +++ b/tests/unit/nodes/node_bus_forwarder_effect/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Tests for node_bus_forwarder_effect.""" diff --git a/tests/unit/nodes/node_bus_forwarder_effect/test_gateway_forwarder_config.py b/tests/unit/nodes/node_bus_forwarder_effect/test_gateway_forwarder_config.py new file mode 100644 index 0000000000..a517d82e42 --- /dev/null +++ b/tests/unit/nodes/node_bus_forwarder_effect/test_gateway_forwarder_config.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +from uuid import UUID + +import pytest +from pydantic import ValidationError + +from omnibase_infra.nodes.node_bus_forwarder_effect.models import ( + ModelGatewayCloudBusConfig, + ModelGatewayForwarderConfig, + ModelGatewayMirrorTopics, + ModelGatewayTenantIdentity, +) + +BROKER_PROVIDER_ID = UUID("22222222-2222-2222-2222-222222222222") +PRINCIPAL_ID = UUID("33333333-3333-3333-3333-333333333333") + + +def _cloud_bus() -> ModelGatewayCloudBusConfig: + return ModelGatewayCloudBusConfig( + broker_provider_id=BROKER_PROVIDER_ID, + cloud_broker_ref="gateway.cloud.kafka.broker", + cloud_auth_ref="gateway.cloud.kafka.oauth", + acl_provisioner_ref="gateway.cloud.kafka.authorization", + client_id_ref="gateway.cloud.kafka.oauth.client_id", + client_secret_api_key_ref="infisical://gateway/redpanda-events", + ) + + +def test_config_rejects_tenant_prefixed_contract_topic() -> None: + with pytest.raises(ValidationError, match="tenant prefix"): + ModelGatewayMirrorTopics( + inbound=( + "tenant-acme.onex.cmd.omnibase-infra.delegation-inference-request.v1", + ), + outbound=("onex.evt.omnibase-infra.inference-response.v1",), + ) + + +def test_config_rejects_reserved_tenant_slug() -> None: + with pytest.raises(ValidationError, match="reserved"): + ModelGatewayTenantIdentity( + tenant_id=UUID("11111111-1111-1111-1111-111111111111"), + tenant_slug="system", + principal_id=PRINCIPAL_ID, + ) + + +def test_config_requires_silence_window_above_heartbeat() -> None: + with pytest.raises(ValidationError, match="max_silence_window_seconds"): + ModelGatewayForwarderConfig( + tenant_identity=ModelGatewayTenantIdentity( + tenant_id=UUID("11111111-1111-1111-1111-111111111111"), + tenant_slug="acme", + principal_id=PRINCIPAL_ID, + ), + cloud_bus=_cloud_bus(), + local_transport_flavor="containerized", + mirror_topics=ModelGatewayMirrorTopics( + inbound=("onex.cmd.omnibase-infra.delegation-inference-request.v1",), + outbound=("onex.evt.omnibase-infra.inference-response.v1",), + ), + heartbeat_interval_seconds=60, + max_silence_window_seconds=60, + ) + + +def test_cloud_bus_config_rejects_ambient_kafka_env_refs() -> None: + with pytest.raises(ValidationError, match=r"KAFKA_\* env"): + ModelGatewayCloudBusConfig( + broker_provider_id=BROKER_PROVIDER_ID, + cloud_broker_ref="KAFKA_BOOTSTRAP_SERVERS", + cloud_auth_ref="gateway.cloud.kafka.oauth", + acl_provisioner_ref="gateway.cloud.kafka.authorization", + client_id_ref="gateway.cloud.kafka.oauth.client_id", + client_secret_api_key_ref="infisical://gateway/redpanda-events", + ) + + +def test_cloud_bus_config_rejects_literal_bootstrap_server_field() -> None: + with pytest.raises(ValidationError, match="bootstrap_servers"): + ModelGatewayCloudBusConfig( + broker_provider_id=BROKER_PROVIDER_ID, + cloud_broker_ref="gateway.cloud.kafka.broker", + cloud_auth_ref="gateway.cloud.kafka.oauth", + acl_provisioner_ref="gateway.cloud.kafka.authorization", + client_id_ref="gateway.cloud.kafka.oauth.client_id", + client_secret_api_key_ref="infisical://gateway/redpanda-events", + bootstrap_servers=("kafka.omninode.ai:9093",), + ) diff --git a/tests/unit/nodes/node_bus_forwarder_effect/test_gateway_forwarder_service.py b/tests/unit/nodes/node_bus_forwarder_effect/test_gateway_forwarder_service.py new file mode 100644 index 0000000000..f000f0a3da --- /dev/null +++ b/tests/unit/nodes/node_bus_forwarder_effect/test_gateway_forwarder_service.py @@ -0,0 +1,237 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Any +from uuid import UUID, uuid4 + +import pytest + +from omnibase_infra.nodes.node_bus_forwarder_effect.models import ( + ModelGatewayCloudBusConfig, + ModelGatewayEnvelope, + ModelGatewayForwarderConfig, + ModelGatewayMirrorTopics, + ModelGatewayTenantIdentity, +) +from omnibase_infra.nodes.node_bus_forwarder_effect.services.service_gateway_forwarder import ( + ServiceGatewayForwarder, +) + +pytestmark = pytest.mark.asyncio + +TENANT_ID = UUID("11111111-1111-1111-1111-111111111111") +BROKER_PROVIDER_ID = UUID("22222222-2222-2222-2222-222222222222") +PRINCIPAL_ID = UUID("33333333-3333-3333-3333-333333333333") +CORRELATION_ID = UUID("44444444-4444-4444-4444-444444444444") +INBOUND_TOPIC = "onex.cmd.omnibase-infra.delegation-inference-request.v1" +OUTBOUND_TOPIC = "onex.evt.omnibase-infra.inference-response.v1" +WIRE_INBOUND_TOPIC = f"tenant-acme.{INBOUND_TOPIC}" +WIRE_OUTBOUND_TOPIC = f"tenant-acme.{OUTBOUND_TOPIC}" + + +@dataclass(frozen=True) +class _Message: + topic: str + key: bytes | None + value: bytes + headers: object | None = None + + +class _MockGatewayBus: + def __init__(self) -> None: + self.subscriptions: dict[str, Callable[[Any], Awaitable[None]]] = {} + self.subscription_groups: dict[str, str] = {} + self.published: list[_Message] = [] + self.unsubscribe_count = 0 + + async def subscribe( + self, + topic: str, + node_identity: object | None = None, + on_message: Callable[[Any], Awaitable[None]] | None = None, + *, + group_id: str | None = None, + **_kwargs: object, + ) -> Callable[[], Awaitable[None]]: + assert on_message is not None + assert group_id is not None + self.subscriptions[topic] = on_message + self.subscription_groups[topic] = group_id + + async def _unsubscribe() -> None: + self.unsubscribe_count += 1 + self.subscriptions.pop(topic, None) + self.subscription_groups.pop(topic, None) + + return _unsubscribe + + async def publish( + self, + topic: str, + key: bytes | None, + value: bytes, + headers: object | None = None, + ) -> None: + self.published.append(_Message(topic, key, value, headers)) + + async def emit(self, topic: str, envelope: ModelGatewayEnvelope) -> None: + await self.subscriptions[topic]( + _Message( + topic=topic, + key=b"key-1", + value=envelope.model_dump_json().encode("utf-8"), + headers={"trace": "preserved"}, + ) + ) + + +def _config() -> ModelGatewayForwarderConfig: + return ModelGatewayForwarderConfig( + tenant_identity=ModelGatewayTenantIdentity( + tenant_id=TENANT_ID, + tenant_slug="acme", + principal_id=PRINCIPAL_ID, + ), + cloud_bus=ModelGatewayCloudBusConfig( + broker_provider_id=BROKER_PROVIDER_ID, + cloud_broker_ref="gateway.cloud.kafka.broker", + cloud_auth_ref="gateway.cloud.kafka.oauth", + acl_provisioner_ref="gateway.cloud.kafka.authorization", + client_id_ref="gateway.cloud.kafka.oauth.client_id", + client_secret_api_key_ref="infisical://gateway/redpanda-events", + ), + local_transport_flavor="containerized", + mirror_topics=ModelGatewayMirrorTopics( + inbound=(INBOUND_TOPIC,), + outbound=(OUTBOUND_TOPIC,), + ), + ) + + +def _envelope(**overrides: object) -> ModelGatewayEnvelope: + values = { + "tenant_id": TENANT_ID, + "tenant_slug": "acme", + "envelope_id": uuid4(), + "correlation_id": CORRELATION_ID, + "causation_id": "cause-1", + "event_type": "LlmInferenceResponse", + "source_topic": OUTBOUND_TOPIC, + "wire_topic": "", + "canonical_topic": OUTBOUND_TOPIC, + "payload": {"ok": True}, + } + values.update(overrides) + return ModelGatewayEnvelope(**values) + + +async def test_start_subscribes_declared_topics_only() -> None: + local_bus = _MockGatewayBus() + cloud_bus = _MockGatewayBus() + service = ServiceGatewayForwarder( + config=_config(), + local_bus=local_bus, + cloud_bus=cloud_bus, + ) + + await service.start() + + assert set(local_bus.subscriptions) == {OUTBOUND_TOPIC} + assert set(cloud_bus.subscriptions) == {WIRE_INBOUND_TOPIC} + assert local_bus.subscription_groups[OUTBOUND_TOPIC].endswith("-outbound") + assert cloud_bus.subscription_groups[WIRE_INBOUND_TOPIC].endswith("-inbound") + + +async def test_outbound_local_message_is_published_to_cloud_wire_topic() -> None: + local_bus = _MockGatewayBus() + cloud_bus = _MockGatewayBus() + service = ServiceGatewayForwarder( + config=_config(), + local_bus=local_bus, + cloud_bus=cloud_bus, + ) + await service.start() + + await local_bus.emit(OUTBOUND_TOPIC, _envelope()) + + assert len(cloud_bus.published) == 1 + published = cloud_bus.published[0] + assert published.topic == WIRE_OUTBOUND_TOPIC + assert published.key == b"key-1" + assert published.headers == {"trace": "preserved"} + forwarded = ModelGatewayEnvelope.model_validate_json(published.value) + assert forwarded.wire_topic == WIRE_OUTBOUND_TOPIC + assert forwarded.canonical_topic == OUTBOUND_TOPIC + assert forwarded.correlation_id == CORRELATION_ID + assert forwarded.event_type == "LlmInferenceResponse" + + +async def test_inbound_cloud_message_is_published_to_local_canonical_topic() -> None: + local_bus = _MockGatewayBus() + cloud_bus = _MockGatewayBus() + service = ServiceGatewayForwarder( + config=_config(), + local_bus=local_bus, + cloud_bus=cloud_bus, + ) + await service.start() + + await cloud_bus.emit( + WIRE_INBOUND_TOPIC, + _envelope( + event_type="DelegationInferenceRequest", + source_topic=WIRE_INBOUND_TOPIC, + wire_topic=WIRE_INBOUND_TOPIC, + canonical_topic=INBOUND_TOPIC, + ), + ) + + assert len(local_bus.published) == 1 + published = local_bus.published[0] + assert published.topic == INBOUND_TOPIC + forwarded = ModelGatewayEnvelope.model_validate_json(published.value) + assert forwarded.source_topic == WIRE_INBOUND_TOPIC + assert forwarded.canonical_topic == INBOUND_TOPIC + assert forwarded.correlation_id == CORRELATION_ID + assert forwarded.event_type == "DelegationInferenceRequest" + + +async def test_outbound_rejects_undeclared_topic_before_cloud_publish() -> None: + local_bus = _MockGatewayBus() + cloud_bus = _MockGatewayBus() + service = ServiceGatewayForwarder( + config=_config(), + local_bus=local_bus, + cloud_bus=cloud_bus, + ) + await service.start() + + with pytest.raises(ValueError, match="not declared"): + await local_bus.emit( + OUTBOUND_TOPIC, + _envelope(canonical_topic="onex.evt.omnibase-infra.not-declared.v1"), + ) + + assert cloud_bus.published == [] + + +async def test_stop_unsubscribes_both_legs() -> None: + local_bus = _MockGatewayBus() + cloud_bus = _MockGatewayBus() + service = ServiceGatewayForwarder( + config=_config(), + local_bus=local_bus, + cloud_bus=cloud_bus, + ) + await service.start() + + await service.stop() + + assert local_bus.subscriptions == {} + assert cloud_bus.subscriptions == {} + assert local_bus.unsubscribe_count == 1 + assert cloud_bus.unsubscribe_count == 1 diff --git a/tests/unit/nodes/node_bus_forwarder_effect/test_gateway_topic_transform.py b/tests/unit/nodes/node_bus_forwarder_effect/test_gateway_topic_transform.py new file mode 100644 index 0000000000..1c9a3750c9 --- /dev/null +++ b/tests/unit/nodes/node_bus_forwarder_effect/test_gateway_topic_transform.py @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +from uuid import UUID, uuid4 + +import pytest + +from omnibase_infra.errors import ProtocolConfigurationError +from omnibase_infra.nodes.node_bus_forwarder_effect.handlers import ( + HandlerConsumeInbound, + HandlerForwardOutbound, +) +from omnibase_infra.nodes.node_bus_forwarder_effect.models import ( + ModelGatewayCloudBusConfig, + ModelGatewayEnvelope, + ModelGatewayForwarderConfig, + ModelGatewayMirrorTopics, + ModelGatewayTenantIdentity, +) + +TENANT_ID = UUID("11111111-1111-1111-1111-111111111111") +BROKER_PROVIDER_ID = UUID("22222222-2222-2222-2222-222222222222") +PRINCIPAL_ID = UUID("33333333-3333-3333-3333-333333333333") +CORRELATION_ID = UUID("44444444-4444-4444-4444-444444444444") +INBOUND_TOPIC = "onex.cmd.omnibase-infra.delegation-inference-request.v1" +OUTBOUND_TOPIC = "onex.evt.omnibase-infra.inference-response.v1" + + +def _config() -> ModelGatewayForwarderConfig: + return ModelGatewayForwarderConfig( + tenant_identity=ModelGatewayTenantIdentity( + tenant_id=TENANT_ID, + tenant_slug="acme", + principal_id=PRINCIPAL_ID, + ), + cloud_bus=ModelGatewayCloudBusConfig( + broker_provider_id=BROKER_PROVIDER_ID, + cloud_broker_ref="gateway.cloud.kafka.broker", + cloud_auth_ref="gateway.cloud.kafka.oauth", + acl_provisioner_ref="gateway.cloud.kafka.authorization", + client_id_ref="gateway.cloud.kafka.oauth.client_id", + client_secret_api_key_ref="infisical://gateway/redpanda-events", + ), + local_transport_flavor="containerized", + mirror_topics=ModelGatewayMirrorTopics( + inbound=(INBOUND_TOPIC,), + outbound=(OUTBOUND_TOPIC,), + ), + ) + + +def _envelope(**overrides: object) -> ModelGatewayEnvelope: + values = { + "tenant_id": TENANT_ID, + "tenant_slug": "acme", + "envelope_id": uuid4(), + "correlation_id": CORRELATION_ID, + "causation_id": None, + "event_type": "LlmInferenceResponse", + "source_topic": OUTBOUND_TOPIC, + "wire_topic": "", + "canonical_topic": OUTBOUND_TOPIC, + "payload": {"ok": True}, + } + values.update(overrides) + return ModelGatewayEnvelope(**values) + + +def test_outbound_adds_tenant_wire_prefix_and_preserves_event_fields() -> None: + envelope = _envelope() + + result = HandlerForwardOutbound(_config()).forward_outbound(envelope) + + assert result.wire_topic == f"tenant-acme.{OUTBOUND_TOPIC}" + assert result.canonical_topic == OUTBOUND_TOPIC + assert result.event_type == envelope.event_type + assert result.correlation_id == envelope.correlation_id + + +def test_outbound_rejects_undeclared_topic() -> None: + envelope = _envelope(canonical_topic="onex.evt.omnibase-infra.not-declared.v1") + + with pytest.raises(ValueError, match="not declared"): + HandlerForwardOutbound(_config()).forward_outbound(envelope) + + +def test_outbound_zero_arg_runtime_discovery_fails_fast_on_use() -> None: + with pytest.raises(ProtocolConfigurationError, match="ForwardOutbound"): + HandlerForwardOutbound().forward_outbound(_envelope()) + + +def test_inbound_strips_tenant_prefix_to_bare_contract_topic() -> None: + envelope = _envelope( + event_type="DelegationInferenceRequest", + source_topic=f"tenant-acme.{INBOUND_TOPIC}", + wire_topic=f"tenant-acme.{INBOUND_TOPIC}", + canonical_topic=INBOUND_TOPIC, + ) + + result = HandlerConsumeInbound(_config()).consume_inbound(envelope) + + assert result.source_topic == f"tenant-acme.{INBOUND_TOPIC}" + assert result.canonical_topic == INBOUND_TOPIC + assert result.event_type == envelope.event_type + assert result.correlation_id == envelope.correlation_id + + +def test_inbound_rejects_cross_tenant_wire_prefix() -> None: + envelope = _envelope( + source_topic=f"tenant-other.{INBOUND_TOPIC}", + wire_topic=f"tenant-other.{INBOUND_TOPIC}", + canonical_topic=INBOUND_TOPIC, + ) + + with pytest.raises(ValueError, match="tenant prefix"): + HandlerConsumeInbound(_config()).consume_inbound(envelope) + + +def test_inbound_rejects_envelope_tenant_mismatch() -> None: + envelope = _envelope( + tenant_id=UUID("22222222-2222-2222-2222-222222222222"), + source_topic=f"tenant-acme.{INBOUND_TOPIC}", + wire_topic=f"tenant-acme.{INBOUND_TOPIC}", + canonical_topic=INBOUND_TOPIC, + ) + + with pytest.raises(ValueError, match="tenant_id"): + HandlerConsumeInbound(_config()).consume_inbound(envelope) + + +def test_inbound_zero_arg_runtime_discovery_fails_fast_on_use() -> None: + envelope = _envelope( + source_topic=f"tenant-acme.{INBOUND_TOPIC}", + wire_topic=f"tenant-acme.{INBOUND_TOPIC}", + canonical_topic=INBOUND_TOPIC, + ) + + with pytest.raises(ProtocolConfigurationError, match="ConsumeInbound"): + HandlerConsumeInbound().consume_inbound(envelope) diff --git a/tests/unit/plugins/examples/test_plugin_json_normalizer.py b/tests/unit/plugins/examples/test_plugin_json_normalizer.py index eb69cd70ac..7ca765170c 100644 --- a/tests/unit/plugins/examples/test_plugin_json_normalizer.py +++ b/tests/unit/plugins/examples/test_plugin_json_normalizer.py @@ -250,8 +250,8 @@ def test_mixed_structure_performance(self, plugin: PluginJsonNormalizer) -> None assert list(first_category.keys()) == sorted(first_category.keys()) # Should complete efficiently - # Using CI-friendly threshold (0.75s) to account for environment variability - assert elapsed_time < 0.75, ( + # Using CI-friendly threshold (1.0s) to account for environment variability + assert elapsed_time < 1.0, ( f"Performance regression: took {elapsed_time:.3f}s for mixed structure" ) diff --git a/tests/unit/runtime/auto_wiring/test_event_consumer_injection.py b/tests/unit/runtime/auto_wiring/test_event_consumer_injection.py new file mode 100644 index 0000000000..1a1836c7e3 --- /dev/null +++ b/tests/unit/runtime/auto_wiring/test_event_consumer_injection.py @@ -0,0 +1,401 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Regression coverage for auto-wired handler event_consumer injection. + +The dispatch INPUT crash was OMN-13003. OMN-13005 was the SECOND defect: a +request/response EFFECT handler declares an injectable blocking ``event_consumer`` +(publish command -> block on correlated terminal event -> read result fields +back), but the runtime auto-wiring materialized no consumer, so the handler fell +back to a no-op default that returned ``None`` immediately and every row was a +degenerate generation-failure. + +OMN-13118 Tier B replaced the per-trial ephemeral consumer with ONE long-lived +correlator (one consumer subscribed once to the terminal topics, a single poll +loop, demux by ``correlation_id``). These tests drive the REAL wiring path +(``_prepare_handler_wiring`` -> ``_materialize_known_handler_dependencies`` -> +``make_terminal_event_consumer`` -> ``LongLivedTerminalCorrelator``) rather than +constructing the handler directly, so they exercise the dispatch surface that the +handler-isolation golden chain never touches (memory +``feedback_real_dispatch_path_tests``). The Kafka layer is faked at the single +seam the correlator builds its consumer through +(``service_pattern_b_broker.AIOKafkaConsumer``) via a shared in-memory partition +log; the real correlator subscribes, runs its poll loop, and demuxes terminals to +the registered futures. + +These assert the two surviving contracts: + - OMN-13005: the injected consumer blocks-and-correlates (a delayed terminal is + still returned, not a degenerate row); and + - OMN-13012/13118: a terminal that lands in the open->wait gap is delivered (the + correlator registers the cid before publish, so the always-running poll loop + demuxes the gap-landed record). +""" + +from __future__ import annotations + +import asyncio +import json +import threading +import time +from collections.abc import Callable +from pathlib import Path +from types import SimpleNamespace +from typing import Any, ClassVar +from unittest.mock import patch + +import pytest + +from omnibase_core.models.events.model_event_envelope import ModelEventEnvelope +from omnibase_core.services.service_handler_resolver import ServiceHandlerResolver +from omnibase_core.services.service_local_handler_ownership_query import ( + ServiceLocalHandlerOwnershipQuery, +) +from omnibase_infra.runtime.auto_wiring.handler_wiring import _prepare_handler_wiring +from omnibase_infra.runtime.auto_wiring.models import ( + ModelContractVersion, + ModelDiscoveredContract, + ModelEventBusWiring, + ModelHandlerRef, + ModelHandlerRouting, + ModelHandlerRoutingEntry, +) + +_TERMINAL_TOPIC = "onex.evt.omnimarket.node-generation-completed.v1" +_COMMAND_TOPIC = "onex.cmd.omnimarket.node-generation-requested.v1" + + +class RecordingEventBus: + """Event bus exposing the Kafka attributes the correlator's consumer needs.""" + + config = SimpleNamespace( + session_timeout_ms=45000, + heartbeat_interval_ms=15000, + max_poll_interval_ms=1800000, + reconnect_backoff_ms=2000, + ) + _bootstrap_servers = "omn-13118-injection-test-broker" + + def __init__(self) -> None: + self.published: list[tuple[str, bytes | None, bytes]] = [] + + async def publish(self, topic: str, key: bytes | None, value: bytes) -> None: + self.published.append((topic, key, value)) + + def _build_auth_kwargs(self) -> dict[str, object]: + return {} + + +# ---------------------------------------------------------------------------- +# Fake Kafka partition log + single long-lived consumer +# ---------------------------------------------------------------------------- + + +class _PartitionLog: + def __init__(self) -> None: + self._records: list[tuple[str, bytes]] = [] + self._lock = threading.Lock() + + @property + def hwm(self) -> int: + with self._lock: + return len(self._records) + + def append(self, correlation_id: str, value: bytes) -> None: + with self._lock: + self._records.append((correlation_id, value)) + + def read_at(self, offset: int) -> tuple[str, bytes] | None: + with self._lock: + if offset < len(self._records): + return self._records[offset] + return None + + +class FakeTerminalLog: + """In-memory terminal log the correlator's single consumer reads from.""" + + def __init__(self) -> None: + self.partition = _PartitionLog() + + def append(self, correlation_id: str, payload: dict[str, Any]) -> None: + self.partition.append(correlation_id, json.dumps(payload).encode("utf-8")) + + +class _FakeClient: + def __init__(self) -> None: + self.tracked: set[str] = set() + + async def _refresh(self) -> bool: + return True + + def set_topics(self, topics: list[str]) -> Any: + self.tracked = set(topics) + return self._refresh() + + def force_metadata_update(self) -> Any: + return self._refresh() + + +class _FakeConsumer: + """Single long-lived ``AIOKafkaConsumer`` stand-in over one terminal topic.""" + + log: ClassVar[FakeTerminalLog | None] = None + + def __init__(self, *args: object, **kwargs: object) -> None: + self._client = _FakeClient() + self._assigned: set[Any] = set() + self._cursor = 0 + + async def start(self) -> None: + return None + + def partitions_for_topic(self, topic: str) -> set[int]: + self._client.tracked.add(topic) + return {0} + + def assign(self, partitions: Any) -> None: + self._assigned = set(partitions) + + def assignment(self) -> set[Any]: + return set(self._assigned) + + async def seek_to_end(self, *assignment: object) -> None: + return None + + async def end_offsets(self, partitions: Any) -> dict[Any, int]: + assert type(self).log is not None + return dict.fromkeys(partitions, type(self).log.partition.hwm) + + def seek(self, partition: Any, offset: int) -> None: + self._cursor = offset + + async def getone(self) -> SimpleNamespace: + assert type(self).log is not None + while True: + record = type(self).log.partition.read_at(self._cursor) + if record is not None: + self._cursor += 1 + _cid, value = record + return SimpleNamespace(topic=_TERMINAL_TOPIC, value=value) + await asyncio.sleep(0.002) + + async def stop(self) -> None: + return None + + +# ---------------------------------------------------------------------------- +# Handler shape (two-phase open/register/wait — the only live shape) +# ---------------------------------------------------------------------------- + + +class HandlerRoiRunnerTwoPhase: + """Subscribe-before-publish shape: open + register, publish, then wait.""" + + ROW_TOPIC = "onex.evt.omnimarket.context-roi-run-completed.v1" + _TEST_LOG: FakeTerminalLog | None = None + + def __init__( + self, + event_publisher: Callable[[str, bytes], None] | None = None, + event_consumer: Any = None, + ) -> None: + self._event_publisher = event_publisher + self._event_consumer = event_consumer + + async def handle(self, envelope: object) -> None: + correlation_id = "cid-roi-runner-1" + session = None + opener = getattr(self._event_consumer, "open", None) + if callable(opener): + session = opener(_TERMINAL_TOPIC) + register = getattr(session, "register", None) + if callable(register): + register(correlation_id) + if self._event_publisher is not None: + self._event_publisher(_COMMAND_TOPIC, b'{"task":"invoice"}') + if self._TEST_LOG is not None: + self._TEST_LOG.append(correlation_id, _terminal_payload(correlation_id)) + if session is not None: + terminal = session.wait(correlation_id, 5.0) + elif self._event_consumer is not None: + terminal = self._event_consumer(_TERMINAL_TOPIC, correlation_id, 5.0) + else: + terminal = None + _emit_row(self._event_publisher, self.ROW_TOPIC, terminal) + + +def _terminal_payload(correlation_id: str) -> dict[str, Any]: + return { + "correlation_id": correlation_id, + "attempt_count": 1, + "model_id": "Qwen3.6-35B-A3B", + "provider": "local", + "contract_passed": True, + "prompt_tokens": 10, + "completion_tokens": 20, + } + + +def _emit_row( + publisher: Callable[[str, bytes], None] | None, + row_topic: str, + terminal: dict[str, Any] | None, +) -> None: + if terminal is None: + row = {"failure_stage": "generation", "attempt_count": 0, "model_id": ""} + else: + row = { + "failure_stage": "none", + "attempt_count": int(terminal["attempt_count"]), + "model_id": str(terminal["model_id"]), + } + if publisher is not None: + publisher(row_topic, json.dumps(row).encode("utf-8")) + + +def _make_roi_runner_contract() -> ModelDiscoveredContract: + return ModelDiscoveredContract( + name="node_context_roi_runner", + node_type="EFFECT_GENERIC", + contract_version=ModelContractVersion(major=1, minor=0, patch=0), + contract_path=Path("/fake/contract.yaml"), + entry_point_name="node_context_roi_runner", + package_name="omnimarket", + event_bus=ModelEventBusWiring( + subscribe_topics=("onex.cmd.omnimarket.context-roi-run-requested.v1",), + publish_topics=( + "onex.evt.omnimarket.context-roi-run-completed.v1", + _COMMAND_TOPIC, + ), + ), + handler_routing=ModelHandlerRouting( + routing_strategy="payload_type_match", + handlers=( + ModelHandlerRoutingEntry( + handler=ModelHandlerRef( + name="HandlerContextRoiRunner", + module="omnimarket.nodes.node_context_roi_runner.handlers.handler_context_roi_runner", + ), + ), + ), + ), + ) + + +async def _drive_handler( + handler_cls: type, log: FakeTerminalLog +) -> list[dict[str, Any]]: + """Wire ``handler_cls`` through the real injection path and dispatch once. + + Patches the single Kafka seam the correlator builds its consumer through + (``service_pattern_b_broker.AIOKafkaConsumer``) so the REAL + ``TerminalEventConsumer`` / ``LongLivedTerminalCorrelator`` logic runs against + the in-memory log. + """ + contract = _make_roi_runner_contract() + event_bus = RecordingEventBus() + resolver = ServiceHandlerResolver() + ownership_query = ServiceLocalHandlerOwnershipQuery( + local_node_names=frozenset({contract.name}) + ) + _FakeConsumer.log = log + handler_cls._TEST_LOG = log # type: ignore[attr-defined] + + with ( + patch( + "omnibase_infra.runtime.auto_wiring.handler_wiring._import_handler_class", + return_value=handler_cls, + ), + patch( + "omnibase_infra.runtime.service_pattern_b_broker.AIOKafkaConsumer", + _FakeConsumer, + ), + ): + prepared = _prepare_handler_wiring( + contract=contract, + entry=contract.handler_routing.handlers[0], # type: ignore[union-attr] + dispatch_engine=None, + resolver=resolver, + ownership_query=ownership_query, + event_bus=event_bus, + container=None, + ) + + await prepared.dispatcher( + ModelEventEnvelope[dict[str, str]]( + payload={"run_id": "t1"}, + event_type="omnimarket.context-roi-run-requested", + ) + ) + await asyncio.sleep(0) + + return [ + json.loads(value.decode("utf-8")) + for topic, _key, value in event_bus.published + if topic == handler_cls.ROW_TOPIC + ] + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_two_phase_consumer_catches_fast_terminal_in_gap() -> None: + """OMN-13118: a terminal landing in the open->wait gap is demuxed and returned. + + The correlator registers the cid before publish, so the always-running poll + loop delivers the gap-landed COMPLETED terminal to the registered future — a + NON-degenerate row through the real wiring path. + """ + log = FakeTerminalLog() + rows = await _drive_handler(HandlerRoiRunnerTwoPhase, log) + assert rows, "handler never emitted a result row" + row = rows[-1] + assert row["failure_stage"] != "generation", ( + "two-phase open/register/wait failed to catch the gap-landed terminal — " + "the long-lived correlator did not demux it to the registered future" + ) + assert row["attempt_count"] >= 1 + assert row["model_id"] == "Qwen3.6-35B-A3B" + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_auto_wired_event_consumer_blocks_and_returns_non_degenerate_row() -> ( + None +): + """OMN-13005 preserved: a delayed terminal is still correlated and returned.""" + log = FakeTerminalLog() + + class _DelayedTerminalTwoPhase(HandlerRoiRunnerTwoPhase): + async def handle(self, envelope: object) -> None: + correlation_id = "cid-roi-runner-1" + session = None + opener = getattr(self._event_consumer, "open", None) + if callable(opener): + session = opener(_TERMINAL_TOPIC) + register = getattr(session, "register", None) + if callable(register): + register(correlation_id) + if self._event_publisher is not None: + self._event_publisher(_COMMAND_TOPIC, b'{"task":"invoice"}') + + active_log = self._TEST_LOG + + def _delayed_append() -> None: + time.sleep(0.05) + if active_log is not None: + active_log.append(correlation_id, _terminal_payload(correlation_id)) + + threading.Thread(target=_delayed_append, daemon=True).start() + terminal = ( + session.wait(correlation_id, 5.0) if session is not None else None + ) + _emit_row(self._event_publisher, self.ROW_TOPIC, terminal) + + rows = await _drive_handler(_DelayedTerminalTwoPhase, log) + assert rows, "handler never emitted a result row" + row = rows[-1] + assert row["failure_stage"] != "generation", ( + "event_consumer was not injected or did not block-correlate — the " + "OMN-13005 degenerate-row defect" + ) + assert row["attempt_count"] >= 1 + assert row["model_id"] == "Qwen3.6-35B-A3B" diff --git a/tests/unit/runtime/config_discovery/test_config_prefetcher.py b/tests/unit/runtime/config_discovery/test_config_prefetcher.py index de486a51ea..8f633cd2f8 100644 --- a/tests/unit/runtime/config_discovery/test_config_prefetcher.py +++ b/tests/unit/runtime/config_discovery/test_config_prefetcher.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. # SPDX-License-Identifier: MIT -"""Unit tests for ConfigPrefetcher (OMN-2287).""" +"""Unit tests for ConfigPrefetcher (OMN-2287, OMN-13070).""" from __future__ import annotations @@ -140,11 +140,17 @@ def test_prefetch_missing_keys(self, monkeypatch: pytest.MonkeyPatch) -> None: assert len(result.missing) > 0 assert "POSTGRES_HOST" in result.missing - def test_prefetch_env_override(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Keys already in environment should skip Infisical fetch.""" + def test_prefetch_env_override_uncontrolled_lane( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """On uncontrolled lane, keys already in environment skip Infisical fetch. + + Legacy behaviour: local dev without Infisical continues to work. + infisical_required=False (default) means ambient env wins. + """ monkeypatch.setenv("POSTGRES_HOST", "from-env") handler = self._make_handler(secrets={}) - prefetcher = ConfigPrefetcher(handler=handler) + prefetcher = ConfigPrefetcher(handler=handler) # infisical_required=False reqs = self._make_requirements( transport_types=[EnumInfraTransportType.DATABASE] ) @@ -153,14 +159,15 @@ def test_prefetch_env_override(self, monkeypatch: pytest.MonkeyPatch) -> None: assert "POSTGRES_HOST" in result.resolved assert result.resolved["POSTGRES_HOST"].get_secret_value() == "from-env" - # Handler should NOT have been called for POSTGRES_HOST + # Handler should NOT have been called for POSTGRES_HOST on uncontrolled lane postgres_host_calls = [ call for call in handler.get_secret_sync.call_args_list if call.kwargs.get("secret_name") == "POSTGRES_HOST" ] assert len(postgres_host_calls) == 0, ( - "POSTGRES_HOST should not be fetched from Infisical when present in env" + "POSTGRES_HOST should not be fetched from Infisical on uncontrolled lane " + "when present in env" ) def test_prefetch_env_dependencies(self) -> None: @@ -206,8 +213,19 @@ def test_prefetch_handler_error(self) -> None: result = prefetcher.prefetch(reqs) assert result.failure_count > 0 - def test_prefetch_infisical_required(self) -> None: - """Should report errors for required keys when infisical_required=True.""" + def test_prefetch_infisical_required(self, monkeypatch: pytest.MonkeyPatch) -> None: + """On controlled lane, missing keys (not in Infisical or env) go to errors.""" + # Remove all DATABASE transport keys so neither Infisical nor env can + # supply them — they must land in result.errors on a controlled lane. + from omnibase_infra.runtime.config_discovery.transport_config_map import ( + TransportConfigMap, + ) + + for key in TransportConfigMap.keys_for_transport( + EnumInfraTransportType.DATABASE + ): + monkeypatch.delenv(key, raising=False) + handler = self._make_handler(secrets={}) prefetcher = ConfigPrefetcher(handler=handler, infisical_required=True) reqs = self._make_requirements( @@ -218,9 +236,8 @@ def test_prefetch_infisical_required(self) -> None: # When infisical_required=True, ConfigPrefetcher passes required=True # to specs_for_transports(), which sets spec.required=True on all - # returned specs. The condition ``self._infisical_required and - # spec.required`` therefore fires for every missing transport key, - # routing them to result.errors rather than result.missing. + # returned specs. Keys missing from both Infisical and ambient env go to + # result.errors rather than result.missing. assert result.failure_count > 0 assert len(result.errors) > 0 assert len(result.missing) == 0 @@ -255,14 +272,17 @@ def test_apply_to_environment(self, monkeypatch: pytest.MonkeyPatch) -> None: # Cleanup monkeypatch.delenv("TEST_PREFETCH_KEY", raising=False) - def test_apply_does_not_overwrite_existing( + def test_apply_does_not_overwrite_existing_uncontrolled_lane( self, monkeypatch: pytest.MonkeyPatch ) -> None: - """Should NOT overwrite keys already in environment.""" + """On uncontrolled lane, apply_to_environment must NOT overwrite existing env. + + Legacy behaviour: infisical_required=False means ambient env wins. + """ monkeypatch.setenv("EXISTING_KEY", "original") handler = self._make_handler(secrets={}) - prefetcher = ConfigPrefetcher(handler=handler) + prefetcher = ConfigPrefetcher(handler=handler) # infisical_required=False result = ModelPrefetchResult( resolved={"EXISTING_KEY": SecretStr("from-infisical")} @@ -305,3 +325,165 @@ def test_empty_requirements(self) -> None: result = prefetcher.prefetch(reqs) assert result.success_count == 0 assert result.failure_count == 0 + + # --- OMN-13070 controlled-lane precedence regression tests --- + + def test_controlled_lane_infisical_wins_over_stale_env( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """On controlled lane, Infisical value beats a stale ambient env value. + + Regression for OMN-13070: config_prefetcher previously skipped keys + already present in os.environ regardless of lane, allowing stale + shell/compose state to override fetched configuration. + """ + # Stale value already in env (e.g. from compose or developer's shell) + monkeypatch.setenv("POSTGRES_HOST", "stale-from-compose") + # Infisical has the authoritative value + handler = self._make_handler( + secrets={"POSTGRES_HOST": "authoritative-from-infisical"} + ) + prefetcher = ConfigPrefetcher(handler=handler, infisical_required=True) + reqs = self._make_requirements( + transport_types=[EnumInfraTransportType.DATABASE] + ) + + result = prefetcher.prefetch(reqs) + + assert "POSTGRES_HOST" in result.resolved + assert ( + result.resolved["POSTGRES_HOST"].get_secret_value() + == "authoritative-from-infisical" + ), "Controlled lane: Infisical value must win over stale ambient env" + # Handler MUST have been called — we must not short-circuit on env presence + postgres_host_calls = [ + call + for call in handler.get_secret_sync.call_args_list + if call.kwargs.get("secret_name") == "POSTGRES_HOST" + ] + assert len(postgres_host_calls) == 1, ( + "Controlled lane: handler must be called for POSTGRES_HOST even when " + "it is present in ambient env" + ) + + def test_controlled_lane_env_as_bootstrap_fallback( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """On controlled lane, ambient env is used as bootstrap fallback only when + Infisical returns None for a key, and the result is resolved (not missing). + + This covers the bootstrap case where a secret is seeded in env before + Infisical is available (e.g. POSTGRES_PASSWORD during first-time startup). + """ + # Infisical has no value for this key (e.g. not seeded yet) + handler = self._make_handler(secrets={}) + # But the key is present in the bootstrap env + monkeypatch.setenv("POSTGRES_HOST", "bootstrap-env-value") + prefetcher = ConfigPrefetcher(handler=handler, infisical_required=True) + reqs = self._make_requirements( + transport_types=[EnumInfraTransportType.DATABASE] + ) + + result = prefetcher.prefetch(reqs) + + # Key must be resolved from env fallback, not reported as error/missing + assert "POSTGRES_HOST" in result.resolved, ( + "Controlled lane: ambient env must be used as bootstrap fallback " + "when Infisical returns None" + ) + assert ( + result.resolved["POSTGRES_HOST"].get_secret_value() == "bootstrap-env-value" + ) + # Handler MUST have been called (we always try Infisical first) + postgres_host_calls = [ + call + for call in handler.get_secret_sync.call_args_list + if call.kwargs.get("secret_name") == "POSTGRES_HOST" + ] + assert len(postgres_host_calls) == 1, ( + "Controlled lane: handler must be called before falling back to env" + ) + + def test_controlled_lane_apply_overwrites_stale_env( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """On controlled lane, apply_to_environment must overwrite stale ambient env. + + Regression for OMN-13070: apply_to_environment previously skipped keys + already present in os.environ on all lanes, allowing stale shell/compose + state to survive the prefetch boundary. + """ + monkeypatch.setenv("DB_HOST", "stale-compose-value") + + handler = self._make_handler(secrets={}) + prefetcher = ConfigPrefetcher(handler=handler, infisical_required=True) + + result = ModelPrefetchResult( + resolved={"DB_HOST": SecretStr("authoritative-from-infisical")} + ) + + applied = prefetcher.apply_to_environment(result) + + assert applied == 1, "Controlled lane: stale env key must be overwritten" + assert os.environ.get("DB_HOST") == "authoritative-from-infisical", ( + "Controlled lane: apply_to_environment must replace stale ambient env " + "value with the fetched authoritative value" + ) + + def test_controlled_lane_missing_from_both_infisical_and_env( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """On controlled lane, a key absent from both Infisical and env is an error.""" + from omnibase_infra.runtime.config_discovery.transport_config_map import ( + TransportConfigMap, + ) + + for key in TransportConfigMap.keys_for_transport( + EnumInfraTransportType.DATABASE + ): + monkeypatch.delenv(key, raising=False) + + handler = self._make_handler(secrets={}) + prefetcher = ConfigPrefetcher(handler=handler, infisical_required=True) + reqs = self._make_requirements( + transport_types=[EnumInfraTransportType.DATABASE] + ) + + result = prefetcher.prefetch(reqs) + + # Must appear in errors, not missing — missing is for uncontrolled-lane + # soft failures; controlled-lane absence is a hard error. + assert len(result.errors) > 0, ( + "Controlled lane: keys absent from both Infisical and env must be errors" + ) + assert len(result.missing) == 0, ( + "Controlled lane: result.missing must be empty; use result.errors" + ) + + def test_precedence_order_uncontrolled_env_wins_infisical_skipped( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """On uncontrolled lane, Infisical is NOT called when the key is in env. + + Verifies the uncontrolled-lane legacy shortcut is still active and + does not regress to always-calling-Infisical after OMN-13070. + """ + monkeypatch.setenv("POSTGRES_HOST", "local-dev-value") + handler = self._make_handler(secrets={"POSTGRES_HOST": "infisical-value"}) + prefetcher = ConfigPrefetcher(handler=handler) # infisical_required=False + reqs = self._make_requirements( + transport_types=[EnumInfraTransportType.DATABASE] + ) + + result = prefetcher.prefetch(reqs) + + assert result.resolved["POSTGRES_HOST"].get_secret_value() == "local-dev-value" + # Handler must NOT have been called on uncontrolled lane when env is set + postgres_host_calls = [ + call + for call in handler.get_secret_sync.call_args_list + if call.kwargs.get("secret_name") == "POSTGRES_HOST" + ] + assert len(postgres_host_calls) == 0, ( + "Uncontrolled lane: Infisical must not be called when key is in env" + ) diff --git a/tests/unit/runtime/health/test_health_config_provenance.py b/tests/unit/runtime/health/test_health_config_provenance.py new file mode 100644 index 0000000000..3e76821b9e --- /dev/null +++ b/tests/unit/runtime/health/test_health_config_provenance.py @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Tests for the config-provenance health check (OMN-12958).""" + +from __future__ import annotations + +from omnibase_infra.runtime.config_provenance import ModelConfigProvenance +from omnibase_infra.runtime.health.health_config_provenance import ( + check_config_provenance_health, +) + + +def _provenance( + *, deployed_sha: str | None, source_sha: str | None +) -> ModelConfigProvenance: + return ModelConfigProvenance( + config_name="bifrost_delegation", + deployed_path="/app/data/delegation/bifrost_delegation.yaml", + deployed_sha256=deployed_sha, + source_path="/app/src/omnimarket/configs/bifrost_delegation.yaml", + source_sha256=source_sha, + ) + + +def test_in_sync_is_healthy() -> None: + health = check_config_provenance_health( + _provenance(deployed_sha="abc", source_sha="abc") + ) + assert health.status == "healthy" + assert health.details is not None + assert health.details["has_drifted"] is False + + +def test_drift_is_degraded() -> None: + health = check_config_provenance_health( + _provenance(deployed_sha="abc", source_sha="def") + ) + assert health.status == "degraded" + assert health.error is not None + assert "re-seed required" in health.error + assert health.details is not None + assert health.details["has_drifted"] is True + + +def test_absent_deployed_is_unhealthy() -> None: + health = check_config_provenance_health( + _provenance(deployed_sha=None, source_sha="abc") + ) + assert health.status == "unhealthy" + assert health.error is not None + assert "deployed config absent" in health.error + + +def test_absent_source_is_degraded() -> None: + health = check_config_provenance_health( + _provenance(deployed_sha="abc", source_sha=None) + ) + assert health.status == "degraded" + assert health.error is not None + assert "cannot prove config provenance" in health.error diff --git a/tests/unit/runtime/models/test_model_bifrost_lane_overlay.py b/tests/unit/runtime/models/test_model_bifrost_lane_overlay.py new file mode 100644 index 0000000000..65859a4c85 --- /dev/null +++ b/tests/unit/runtime/models/test_model_bifrost_lane_overlay.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Unit tests for ModelBifrostLaneOverlay (OMN-12864). + +Validates the typed deployment binding model: +- URL completeness enforcement (OMN-12815) +- env-var mapping +- reject bare-base URLs +""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from omnibase_infra.runtime.models.model_bifrost_lane_overlay import ( + ModelBifrostLaneOverlay, +) + + +def _complete_url(host: str, port: int) -> str: + return f"http://{host}:{port}/v1/chat/completions" + + +def _valid_overlay(**overrides: str) -> dict[str, str]: + base = { + "lane": "dev", + "coder_endpoint_url": _complete_url("192.168.86.201", 8000), + "reasoner_endpoint_url": _complete_url("192.168.86.201", 8001), + "embedding_endpoint_url": _complete_url("192.168.86.201", 8100), + "ds4_flash_endpoint_url": _complete_url("192.168.86.200", 8101), + } + base.update(overrides) + return base + + +@pytest.mark.unit +def test_valid_overlay_loads() -> None: + overlay = ModelBifrostLaneOverlay.model_validate(_valid_overlay()) + assert overlay.lane == "dev" + assert overlay.coder_endpoint_url == _complete_url("192.168.86.201", 8000) + assert overlay.reasoner_endpoint_url == _complete_url("192.168.86.201", 8001) + assert overlay.embedding_endpoint_url == _complete_url("192.168.86.201", 8100) + assert overlay.ds4_flash_endpoint_url == _complete_url("192.168.86.200", 8101) + + +@pytest.mark.unit +def test_as_env_dict_returns_correct_keys() -> None: + overlay = ModelBifrostLaneOverlay.model_validate(_valid_overlay()) + env = overlay.as_env_dict() + assert set(env.keys()) == { + "BIFROST_LOCAL_CODER_ENDPOINT_URL", + "BIFROST_LOCAL_REASONER_ENDPOINT_URL", + "BIFROST_LOCAL_EMBEDDING_ENDPOINT_URL", + "BIFROST_LOCAL_DS_V4_FLASH_ENDPOINT_URL", + } + assert env["BIFROST_LOCAL_CODER_ENDPOINT_URL"] == _complete_url( + "192.168.86.201", 8000 + ) + assert env["BIFROST_LOCAL_DS_V4_FLASH_ENDPOINT_URL"] == _complete_url( + "192.168.86.200", 8101 + ) + + +@pytest.mark.unit +@pytest.mark.parametrize( + ("field", "bare_url"), + [ + ("coder_endpoint_url", "http://192.168.86.201:8000"), + ("coder_endpoint_url", "http://192.168.86.201:8000/"), + ("coder_endpoint_url", "http://192.168.86.201:8000/v1"), + ("reasoner_endpoint_url", "http://192.168.86.201:8001/v1"), + ("embedding_endpoint_url", "http://192.168.86.201:8100/v1"), + ("ds4_flash_endpoint_url", "http://192.168.86.200:8101"), + ], +) +def test_bare_base_url_rejected(field: str, bare_url: str) -> None: + """Bare-base URLs without /chat/completions path must be rejected (OMN-12815).""" + with pytest.raises(ValidationError, match="endpoint_url must end in"): + ModelBifrostLaneOverlay.model_validate(_valid_overlay(**{field: bare_url})) + + +@pytest.mark.unit +def test_alternate_chat_completions_path_accepted() -> None: + """URLs ending in /chat/completions (without /v1 prefix) are also valid.""" + overlay = ModelBifrostLaneOverlay.model_validate( + _valid_overlay(coder_endpoint_url="http://192.168.86.201:8000/chat/completions") + ) + assert overlay.coder_endpoint_url == "http://192.168.86.201:8000/chat/completions" + + +@pytest.mark.unit +def test_empty_lane_rejected() -> None: + with pytest.raises(ValidationError): + ModelBifrostLaneOverlay.model_validate(_valid_overlay(lane="")) + + +@pytest.mark.unit +def test_empty_url_rejected() -> None: + with pytest.raises(ValidationError): + ModelBifrostLaneOverlay.model_validate(_valid_overlay(coder_endpoint_url="")) + + +@pytest.mark.unit +def test_extra_fields_rejected() -> None: + """Extra fields must be forbidden (strict schema).""" + with pytest.raises(ValidationError): + ModelBifrostLaneOverlay.model_validate( + {**_valid_overlay(), "unexpected_field": "value"} + ) diff --git a/tests/unit/runtime/test_config_provenance.py b/tests/unit/runtime/test_config_provenance.py new file mode 100644 index 0000000000..02be5e9a2e --- /dev/null +++ b/tests/unit/runtime/test_config_provenance.py @@ -0,0 +1,119 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Tests for runtime config provenance (OMN-12958).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from omnibase_infra.runtime.config_provenance import ( + PROVENANCE_SIDECAR_NAME, + ModelConfigProvenance, + build_config_provenance, + compute_sha256, + write_provenance_sidecar, +) + + +def _write(path: Path, content: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return path + + +def test_compute_sha256_absent_returns_none(tmp_path: Path) -> None: + assert compute_sha256(tmp_path / "missing.yaml") is None + + +def test_compute_sha256_is_byte_exact(tmp_path: Path) -> None: + a = _write(tmp_path / "a.yaml", "backends: []\n") + b = _write(tmp_path / "b.yaml", "backends: []\n") + c = _write(tmp_path / "c.yaml", "backends: [x]\n") + assert compute_sha256(a) == compute_sha256(b) + assert compute_sha256(a) != compute_sha256(c) + + +def test_in_sync_not_drifted(tmp_path: Path) -> None: + deployed = _write(tmp_path / "deployed.yaml", "backends: []\n") + source = _write(tmp_path / "source.yaml", "backends: []\n") + prov = build_config_provenance( + config_name="bifrost_delegation", + deployed_path=deployed, + source_path=source, + ) + assert prov.deployed_present + assert prov.source_present + assert not prov.has_drifted + assert "in-sync" in prov.provenance_line() + + +def test_drift_detected(tmp_path: Path) -> None: + deployed = _write(tmp_path / "deployed.yaml", "backends: [stale]\n") + source = _write(tmp_path / "source.yaml", "backends: []\n") + prov = build_config_provenance( + config_name="bifrost_delegation", + deployed_path=deployed, + source_path=source, + ) + assert prov.has_drifted + assert "DRIFT" in prov.provenance_line() + assert prov.deployed_sha256 != prov.source_sha256 + + +def test_absent_deployed_is_not_drift(tmp_path: Path) -> None: + source = _write(tmp_path / "source.yaml", "backends: []\n") + prov = build_config_provenance( + config_name="bifrost_delegation", + deployed_path=tmp_path / "missing.yaml", + source_path=source, + ) + assert not prov.deployed_present + assert prov.source_present + # Missing deployed copy is reported via *_present, not has_drifted. + assert not prov.has_drifted + + +def test_absent_source_is_not_drift(tmp_path: Path) -> None: + deployed = _write(tmp_path / "deployed.yaml", "backends: []\n") + prov = build_config_provenance( + config_name="bifrost_delegation", + deployed_path=deployed, + source_path=tmp_path / "missing.yaml", + ) + assert prov.deployed_present + assert not prov.source_present + assert not prov.has_drifted + + +def test_sidecar_roundtrips(tmp_path: Path) -> None: + deployed = _write( + tmp_path / "delegation" / "bifrost_delegation.yaml", "backends: []\n" + ) + source = _write(tmp_path / "source.yaml", "backends: [x]\n") + prov = build_config_provenance( + config_name="bifrost_delegation", + deployed_path=deployed, + source_path=source, + ) + sidecar = write_provenance_sidecar(prov, deployed_path=deployed) + assert sidecar.name == PROVENANCE_SIDECAR_NAME + assert sidecar.parent == deployed.parent + + loaded = ModelConfigProvenance.model_validate( + json.loads(sidecar.read_text(encoding="utf-8")) + ) + assert loaded == prov + assert loaded.has_drifted + + +def test_model_is_frozen(tmp_path: Path) -> None: + prov = build_config_provenance( + config_name="bifrost_delegation", + deployed_path=tmp_path / "d.yaml", + source_path=tmp_path / "s.yaml", + ) + with pytest.raises(Exception): + prov.config_name = "other" # type: ignore[misc] diff --git a/tests/unit/runtime/test_render_bifrost_delegation_contract.py b/tests/unit/runtime/test_render_bifrost_delegation_contract.py index fb1867c18e..31fa4650c9 100644 --- a/tests/unit/runtime/test_render_bifrost_delegation_contract.py +++ b/tests/unit/runtime/test_render_bifrost_delegation_contract.py @@ -489,3 +489,135 @@ def test_empty_bifrost_contract_path_disables_render(tmp_path: Path) -> None: ) assert rendered is None + + +# ========================================================================== +# OMN-12945 — force_reseed bypasses stale-volume cache +# ========================================================================== + + +@pytest.mark.unit +def test_force_reseed_overwrites_stale_volume_content(tmp_path: Path) -> None: + """force_reseed=True must re-render from source even when volume has populated endpoints. + + This is the OMN-12945 fix: stale volume content surviving image rebuilds no + longer pins wrong backends after a container restart. + """ + source = _source_contract(tmp_path / "source.yaml") + target = tmp_path / "rendered" / "bifrost_delegation.yaml" + + # 1. Initial render seeds the volume. + render_bifrost_delegation_contract( + source_path=source, + target_path=target, + environ={"LLM_CODER_URL": _http_url("coder.local:8000")}, + ) + assert target.exists() + + # 2. Mutate the volume copy to simulate drift (wrong endpoint). + loaded = yaml.safe_load(target.read_text(encoding="utf-8")) + loaded["backends"][0]["endpoint_url"] = _http_url("stale.local:9999") + target.write_text(yaml.safe_dump(loaded, sort_keys=False), encoding="utf-8") + + # 3. Without force_reseed, the volume copy is reused (env still matches stale). + # We verify the stale value is NOT corrected here to confirm the cache path is real. + loaded_stale = yaml.safe_load(target.read_text(encoding="utf-8")) + assert loaded_stale["backends"][0]["endpoint_url"] == _http_url("stale.local:9999") + + # 4. force_reseed=True re-renders from packaged source, replacing the stale copy. + rendered = render_bifrost_delegation_contract( + source_path=source, + target_path=target, + environ={"LLM_CODER_URL": _http_url("coder.local:8000")}, + force_reseed=True, + ) + + assert rendered == target + loaded_fresh = yaml.safe_load(target.read_text(encoding="utf-8")) + assert loaded_fresh["backends"][0]["endpoint_url"] == _http_url("coder.local:8000") + + +@pytest.mark.unit +def test_bifrost_force_reseed_env_flag_triggers_reseed(tmp_path: Path) -> None: + """BIFROST_FORCE_RESEED=1 env flag triggers re-seed (OMN-12945 entrypoint path).""" + source = _source_contract(tmp_path / "source.yaml") + target = tmp_path / "rendered" / "bifrost_delegation.yaml" + + # Seed initial content. + render_bifrost_delegation_contract( + source_path=source, + target_path=target, + environ={"LLM_CODER_URL": _http_url("coder.local:8000")}, + ) + + # Drift the volume copy. + loaded = yaml.safe_load(target.read_text(encoding="utf-8")) + loaded["backends"][0]["endpoint_url"] = _http_url("stale.local:9999") + target.write_text(yaml.safe_dump(loaded, sort_keys=False), encoding="utf-8") + + # BIFROST_FORCE_RESEED=1 in environ triggers re-seed. + rendered = render_bifrost_delegation_contract( + source_path=source, + target_path=target, + environ={ + "LLM_CODER_URL": _http_url("coder.local:8000"), + "BIFROST_FORCE_RESEED": "1", + }, + ) + + assert rendered == target + loaded_fresh = yaml.safe_load(target.read_text(encoding="utf-8")) + assert loaded_fresh["backends"][0]["endpoint_url"] == _http_url("coder.local:8000") + + +@pytest.mark.unit +def test_bifrost_force_reseed_env_false_does_not_override_cache(tmp_path: Path) -> None: + """BIFROST_FORCE_RESEED=0 must NOT bypass the cache — the flag only triggers on truthy values.""" + source = _source_contract(tmp_path / "source.yaml") + target = tmp_path / "rendered" / "bifrost_delegation.yaml" + + # Render with the real endpoint to seed the volume. + render_bifrost_delegation_contract( + source_path=source, + target_path=target, + environ={"LLM_CODER_URL": _http_url("coder.local:8000")}, + ) + + # Re-render with BIFROST_FORCE_RESEED=0 and a different env endpoint. + # The cache path should win — the existing volume content is reused + # because the env endpoint matches the existing volume endpoint. + render_bifrost_delegation_contract( + source_path=source, + target_path=target, + environ={ + "LLM_CODER_URL": _http_url("coder.local:8000"), + "BIFROST_FORCE_RESEED": "0", + }, + ) + + loaded = yaml.safe_load(target.read_text(encoding="utf-8")) + # The cache path was used (endpoint unchanged from initial render). + assert loaded["backends"][0]["endpoint_url"] == _http_url("coder.local:8000") + + +# ========================================================================== +# OMN-12814 — fail-loud on zero populated endpoints +# ========================================================================== + + +@pytest.mark.unit +def test_zero_populated_endpoints_raises_loud_error(tmp_path: Path) -> None: + """render_bifrost_delegation_contract must raise ProtocolConfigurationError + (not return empty) when no endpoint env vars are set (OMN-12814 fail-loud).""" + from omnibase_infra.errors import ProtocolConfigurationError + + source = _source_contract(tmp_path / "source.yaml") + target = tmp_path / "rendered.yaml" + + with pytest.raises(ProtocolConfigurationError, match="no populated endpoint_url"): + render_bifrost_delegation_contract( + source_path=source, + target_path=target, + environ={}, # No LLM_*_URL vars — all backends stay unpopulated. + force_reseed=True, # Bypass cache so we reach the rendering path. + ) diff --git a/tests/unit/runtime/test_run_event_driven_operation_match.py b/tests/unit/runtime/test_run_event_driven_operation_match.py new file mode 100644 index 0000000000..203f00f2da --- /dev/null +++ b/tests/unit/runtime/test_run_event_driven_operation_match.py @@ -0,0 +1,272 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""End-to-end boot tests for OMN-13141: _run_event_driven executor half. + +Disjoint companion to OMN-13137 (#1973). OMN-13137 fixed only the VALIDATOR +half (`_validate_routing` is strategy-aware). This file covers the EXECUTOR +half: `_run_event_driven` wiring loop. + +Root cause (L816): for `routing_strategy: operation_match`, handler entries carry +no `event_model` block, so `_resolve_routing_entries` yields +`event_model_module=""`. The wiring loop then calls +`importlib.import_module("")`, which raises `ValueError: Empty module name`. +That `ValueError` is NOT caught by the `(ImportError, AttributeError)` clause and +propagates to the outer `except Exception` in `run_async` -> result FAILED. The +node never boots end-to-end. + +Validator-half test 4 (`test_validate_routing_operation_match.py`) only calls +`_validate_routing` — it does NOT boot through `run_async()`, so it cannot catch +this path. These tests boot the real `run_async()` event-driven path. + +Fix: skip the event-model import when `event_model_module` is empty (operation +routes by `operation`, not a payload model); wire the adapter with no input +model. payload_type_match keeps importing its event model (regression guard). +""" + +from __future__ import annotations + +import sys +import types +from pathlib import Path +from typing import Any + +import pytest +from pydantic import BaseModel + +from omnibase_core.enums.enum_workflow_result import EnumWorkflowResult +from omnibase_infra.runtime.runtime_local import RuntimeLocal + +# --------------------------------------------------------------------------- +# Test 1 — operation_match contract boots via run_async without ValueError +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_operation_match_contract_boots_via_run_async(tmp_path: Path) -> None: + """An operation_match contract (no event_model) boots end-to-end. + + Before OMN-13141 the wiring loop calls importlib.import_module("") for the + empty event_model_module and raises ValueError: Empty module name, which the + outer except in run_async swallows -> result FAILED. After the fix the + handler is wired without importing a payload model and the workflow reaches + its terminal event -> COMPLETED. + """ + + class _OperationInput(BaseModel): + correlation_id: str + + class _OperationTerminal(BaseModel): + correlation_id: str + status: str = "success" + + class _OperationHandler: + async def handle(self, correlation_id: str) -> _OperationTerminal: + return _OperationTerminal(correlation_id=correlation_id) + + mod_input = types.ModuleType("_test_opmatch_input") + mod_input._OperationInput = _OperationInput # type: ignore[attr-defined] + mod_handler = types.ModuleType("_test_opmatch_handler") + mod_handler._OperationHandler = _OperationHandler # type: ignore[attr-defined] + + sys.modules["_test_opmatch_input"] = mod_input + sys.modules["_test_opmatch_handler"] = mod_handler + + try: + # operation_match handler entry deliberately omits event_model — this is + # the canonical shape that produced event_model_module="" -> L816 crash. + contract_yaml = ( + "name: test_operation_match_boot\n" + "contract_version: {major: 1, minor: 0, patch: 0}\n" + "node_type: orchestrator\n" + "description: operation_match boot test\n" + "initial_command: cmd.op.start.v1\n" + "terminal_event: evt.op.done.v1\n" + "event_bus:\n" + " subscribe_topics:\n" + " - cmd.op.start.v1\n" + " publish_topics:\n" + " - evt.op.done.v1\n" + "input_model:\n" + " module: _test_opmatch_input\n" + " class: _OperationInput\n" + "handler_routing:\n" + " routing_strategy: operation_match\n" + " handlers:\n" + " - operation: do_op\n" + " handler:\n" + " name: _OperationHandler\n" + " module: _test_opmatch_handler\n" + ) + workflow = tmp_path / "operation_match_boot.yaml" + workflow.write_text(contract_yaml) + + runtime = RuntimeLocal( + workflow_path=workflow, + state_root=tmp_path / "state", + timeout=10, + ) + result = await runtime.run_async() + + # The node must not FAIL from the empty-module ValueError. Before the fix + # this asserts COMPLETED but receives FAILED. + assert result != EnumWorkflowResult.FAILED, ( + "operation_match contract failed to boot — likely the empty " + "event_model_module ValueError at the wiring loop (OMN-13141)" + ) + assert result == EnumWorkflowResult.COMPLETED + + # No FAILED state persisted from an import crash. + import json + + state_file = tmp_path / "state" / "workflow_result.json" + assert state_file.exists() + state_data = json.loads(state_file.read_text()) + assert state_data["result"] == "completed" + finally: + sys.modules.pop("_test_opmatch_input", None) + sys.modules.pop("_test_opmatch_handler", None) + + +# --------------------------------------------------------------------------- +# Test 2 — operation_match wiring does not raise ValueError: Empty module name +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_operation_match_wiring_does_not_import_empty_module( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The wiring loop must never call importlib.import_module(""). + + Guards the exact failure mode: a spy on importlib.import_module asserts the + empty string is never passed (operation_match entries have no event model to + import). The handler module is still imported, so the spy sees non-empty + names only. + """ + import importlib + + class _OpHandler: + async def handle(self, correlation_id: str) -> None: + return None + + mod_handler = types.ModuleType("_test_opmatch_spy_handler") + mod_handler._OpHandler = _OpHandler # type: ignore[attr-defined] + sys.modules["_test_opmatch_spy_handler"] = mod_handler + + real_import_module = importlib.import_module + empty_calls: list[str] = [] + + def _spy_import_module(name: str, *args: Any, **kwargs: Any) -> Any: + if name == "": + empty_calls.append(name) + return real_import_module(name, *args, **kwargs) + + monkeypatch.setattr( + "omnibase_infra.runtime.runtime_local.importlib.import_module", + _spy_import_module, + ) + + try: + contract_yaml = ( + "name: test_operation_match_spy\n" + "contract_version: {major: 1, minor: 0, patch: 0}\n" + "node_type: orchestrator\n" + "description: operation_match spy test\n" + "terminal_event: evt.op.done.v1\n" + "event_bus:\n" + " subscribe_topics:\n" + " - cmd.op.start.v1\n" + " publish_topics:\n" + " - evt.op.done.v1\n" + "handler_routing:\n" + " routing_strategy: operation_match\n" + " handlers:\n" + " - operation: do_op\n" + " handler:\n" + " name: _OpHandler\n" + " module: _test_opmatch_spy_handler\n" + ) + workflow = tmp_path / "operation_match_spy.yaml" + workflow.write_text(contract_yaml) + + runtime = RuntimeLocal( + workflow_path=workflow, + state_root=tmp_path / "state", + timeout=10, + ) + await runtime.run_async() + + assert empty_calls == [], ( + "importlib.import_module('') was called during operation_match " + "wiring — empty event_model_module must be skipped (OMN-13141)" + ) + finally: + sys.modules.pop("_test_opmatch_spy_handler", None) + + +# --------------------------------------------------------------------------- +# Test 3 — payload_type_match still imports its event model (regression guard) +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_payload_type_match_still_imports_event_model(tmp_path: Path) -> None: + """payload_type_match path is unchanged — its event model is still imported. + + A payload_type_match entry pointing at a non-importable event_model module + must still FAIL (the import error is real). This guards against the fix + over-broadly skipping imports for the payload strategy. + """ + + class _PtInput(BaseModel): + correlation_id: str + + class _PtHandler: + async def handle(self, correlation_id: str) -> None: + return None + + mod_handler = types.ModuleType("_test_ptmatch_handler") + mod_handler._PtHandler = _PtHandler # type: ignore[attr-defined] + sys.modules["_test_ptmatch_handler"] = mod_handler + + try: + contract_yaml = ( + "name: test_payload_type_match_bad_model\n" + "contract_version: {major: 1, minor: 0, patch: 0}\n" + "node_type: orchestrator\n" + "description: payload_type_match bad event model\n" + "terminal_event: evt.pt.done.v1\n" + "event_bus:\n" + " subscribe_topics:\n" + " - cmd.pt.start.v1\n" + " publish_topics:\n" + " - evt.pt.done.v1\n" + "handler_routing:\n" + " routing_strategy: payload_type_match\n" + " handlers:\n" + " - event_model:\n" + " name: NotThere\n" + " module: _test_nonexistent_event_model_module\n" + " handler:\n" + " name: _PtHandler\n" + " module: _test_ptmatch_handler\n" + ) + workflow = tmp_path / "payload_type_match_bad.yaml" + workflow.write_text(contract_yaml) + + runtime = RuntimeLocal( + workflow_path=workflow, + state_root=tmp_path / "state", + timeout=10, + ) + result = await runtime.run_async() + + # The event model import genuinely fails -> FAILED is correct here. + assert result == EnumWorkflowResult.FAILED + finally: + sys.modules.pop("_test_ptmatch_handler", None) diff --git a/tests/unit/runtime/test_runtime_local_execution.py b/tests/unit/runtime/test_runtime_local_execution.py index 276865f218..88c438fe45 100644 --- a/tests/unit/runtime/test_runtime_local_execution.py +++ b/tests/unit/runtime/test_runtime_local_execution.py @@ -1369,12 +1369,13 @@ def _make_handler_entry( def test_validate_routing_map_based_happy_path() -> None: """Map-based validation passes when all handlers have input topics in subscribe_topics.""" routing = { + "routing_strategy": "payload_type_match", "handlers": [ _make_handler_entry( "EventA", handler_name="HandlerA", output_events=["EventB"] ), _make_handler_entry("EventB", handler_name="HandlerB"), - ] + ], } errors = RuntimeLocal._validate_routing( routing, @@ -1388,6 +1389,7 @@ def test_validate_routing_map_based_happy_path() -> None: def test_validate_routing_terminal_reducer_no_padding() -> None: """Terminal reducer with explicit subscribe_topic requires no padding of subscribe_topics.""" routing = { + "routing_strategy": "payload_type_match", "handlers": [ _make_handler_entry( "EventA", @@ -1399,7 +1401,7 @@ def test_validate_routing_terminal_reducer_no_padding() -> None: handler_name="HandlerB", subscribe_topic="topic.b.v1", ), - ] + ], } # Only one subscribe_topic declared (for HandlerA); HandlerB uses explicit field. errors = RuntimeLocal._validate_routing( @@ -1414,11 +1416,12 @@ def test_validate_routing_terminal_reducer_no_padding() -> None: def test_validate_routing_terminal_reducer_no_input_topic() -> None: """Terminal reducer with no subscribe_topic and no positional slot is valid (no bus wiring).""" routing = { + "routing_strategy": "payload_type_match", "handlers": [ _make_handler_entry("EventA", handler_name="HandlerA"), # Handler at index 1, but subscribe_topics only has 1 entry — no slot. _make_handler_entry("EventB", handler_name="TerminalReducer"), - ] + ], } errors = RuntimeLocal._validate_routing( routing, @@ -1432,13 +1435,14 @@ def test_validate_routing_terminal_reducer_no_input_topic() -> None: def test_validate_routing_explicit_subscribe_topic_not_in_list() -> None: """Explicit subscribe_topic that is not in subscribe_topics is an error.""" routing = { + "routing_strategy": "payload_type_match", "handlers": [ _make_handler_entry( "EventA", handler_name="HandlerA", subscribe_topic="topic.unknown.v1", ), - ] + ], } errors = RuntimeLocal._validate_routing( routing, @@ -1472,15 +1476,16 @@ def test_validate_routing_orphan_output_still_detected() -> None: @pytest.mark.unit def test_validate_routing_missing_required_fields() -> None: - """Missing event_model.name / handler.module are still caught.""" + """Missing event_model.name / handler.module are still caught for payload_type_match.""" routing = { + "routing_strategy": "payload_type_match", "handlers": [ { "event_model": {"module": "mod.events"}, "handler": {"name": "HandlerA"}, "output_events": [], } - ] + ], } errors = RuntimeLocal._validate_routing( routing, diff --git a/tests/unit/runtime/test_runtime_local_operation_match_typed_handler.py b/tests/unit/runtime/test_runtime_local_operation_match_typed_handler.py new file mode 100644 index 0000000000..fcd55dd170 --- /dev/null +++ b/tests/unit/runtime/test_runtime_local_operation_match_typed_handler.py @@ -0,0 +1,194 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Real-dispatch-path coverage for typed single-model-parameter handlers reached +via ``operation_match`` routing (OMN-8724). + +``operation_match`` routing entries declare no event model, so the runtime wires +the adapter with ``input_model_cls=None`` and forwards the raw decoded dict +(OMN-13141). When such a handler declares a single positional parameter annotated +with a concrete ``BaseModel`` subclass — e.g. ``handle(self, request: +GoldenChainSweepRequest)`` — the adapter must validate the dict into that type +before calling. Before OMN-8724 it forwarded the bare dict and the handler +crashed on the first attribute access (``AttributeError: 'dict' object has no +attribute ''``) the moment it ran through ``onex node`` / RuntimeLocal, +even though the module ``__main__`` entrypoint (which builds the model itself) +worked. + +These tests boot the real ``RuntimeLocal._run_event_driven`` path end-to-end — +not the handler in isolation — so they would have caught the dispatch-boundary +crash. Handler-isolation tests pass while the live ``onex node`` path fails; +this is the regression guard for the live path. +""" + +from __future__ import annotations + +import json +import sys +import types +from pathlib import Path + +import pytest +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_core.enums.enum_workflow_result import EnumWorkflowResult +from omnibase_infra.runtime.runtime_local import RuntimeLocal +from omnibase_infra.runtime.runtime_local_adapter import _invoke_handle_method + +# --------------------------------------------------------------------------- +# Typed request/result models + a single-model-parameter handler that mirrors +# NodeGoldenChainSweep.handle(self, request: GoldenChainSweepRequest). +# --------------------------------------------------------------------------- + + +class _TypedRequest(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + correlation_id: str = "" + items: list[str] = Field(default_factory=list) + + +class _TypedResult(BaseModel): + model_config = ConfigDict(extra="forbid") + + correlation_id: str = "" + item_count: int = 0 + status: str = "success" + + +class _TypedRequestHandler: + """Sync handler whose sole positional parameter is a concrete BaseModel. + + Accesses an attribute on the request immediately — the exact shape that + crashed in node_golden_chain_sweep when handed a raw dict. + """ + + def handle(self, request: _TypedRequest) -> _TypedResult: + return _TypedResult( + correlation_id=request.correlation_id, + item_count=len(request.items), + ) + + +# --------------------------------------------------------------------------- +# Unit-level: _invoke_handle_method coerces a dict into the annotated model. +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_invoke_coerces_dict_into_single_model_parameter() -> None: + """A raw dict payload is validated into the handler's annotated BaseModel.""" + handler = _TypedRequestHandler() + payload = {"correlation_id": "cid-coerce", "items": ["a", "b", "c"]} + + result = _invoke_handle_method(handler.handle, payload) + + assert isinstance(result, _TypedResult) + assert result.correlation_id == "cid-coerce" + assert result.item_count == 3 + + +@pytest.mark.unit +def test_invoke_does_not_double_wrap_model_payload() -> None: + """When the payload is already the model instance, it is passed through.""" + handler = _TypedRequestHandler() + request = _TypedRequest(correlation_id="cid-model", items=["x"]) + + result = _invoke_handle_method(handler.handle, request) + + assert isinstance(result, _TypedResult) + assert result.correlation_id == "cid-model" + assert result.item_count == 1 + + +@pytest.mark.unit +def test_invoke_dict_coercion_raises_on_invalid_payload() -> None: + """An invalid dict surfaces a pydantic ValidationError, not AttributeError. + + Fail-fast: bad data must fail validation at the boundary, not silently slip + a malformed object into the handler. + """ + from pydantic import ValidationError + + handler = _TypedRequestHandler() + # extra="forbid" rejects unknown keys → ValidationError (not AttributeError). + with pytest.raises(ValidationError): + _invoke_handle_method(handler.handle, {"unexpected_field": 1}) + + +# --------------------------------------------------------------------------- +# Real-dispatch-path: boot an operation_match contract whose handler takes a +# single typed-model parameter, through RuntimeLocal.run_async(). +# --------------------------------------------------------------------------- + + +_CMD_TOPIC = "onex.cmd.test.typed-operation-match-start.v1" +_EVT_TOPIC = "onex.evt.test.typed-operation-match-completed.v1" + + +def _typed_operation_match_contract() -> str: + return ( + "name: test_typed_operation_match\n" + "contract_version: {major: 1, minor: 0, patch: 0}\n" + "node_type: compute\n" + "description: operation_match handler with a single typed-model parameter\n" + "terminal_event: " + _EVT_TOPIC + "\n" + "event_bus:\n" + " subscribe_topics:\n" + " - " + _CMD_TOPIC + "\n" + " publish_topics:\n" + " - " + _EVT_TOPIC + "\n" + "handler_routing:\n" + " routing_strategy: operation_match\n" + " handlers:\n" + " - operation: run_typed\n" + " handler:\n" + " name: _TypedRequestHandler\n" + " module: _test_typed_op_match_handler\n" + ) + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_operation_match_typed_handler_boots_via_run_async( + tmp_path: Path, +) -> None: + """A typed single-model-parameter handler completes via the real dispatch path. + + Regression guard for OMN-8724: before the fix, the adapter forwarded the raw + decoded dict to ``handle(self, request: _TypedRequest)`` and the run ended + FAILED with ``AttributeError: 'dict' object has no attribute ...``. The fix + validates the dict into ``_TypedRequest`` first, so the run COMPLETES and the + terminal event carries the handler's result. + """ + mod = types.ModuleType("_test_typed_op_match_handler") + mod._TypedRequestHandler = _TypedRequestHandler # type: ignore[attr-defined] + sys.modules["_test_typed_op_match_handler"] = mod + + try: + contract = tmp_path / "contract.yaml" + contract.write_text(_typed_operation_match_contract()) + + runtime = RuntimeLocal( + workflow_path=contract, + state_root=tmp_path / "state", + timeout=10, + ) + result = await runtime.run_async() + + assert result == EnumWorkflowResult.COMPLETED, ( + f"typed operation_match handler did not complete: " + f"result={result.value} last_error={runtime.last_error!r}" + ) + assert runtime.last_error is None + + state_data = json.loads( + (tmp_path / "state" / "workflow_result.json").read_text() + ) + assert state_data["result"] == "completed" + assert state_data["exit_code"] == 0 + # Terminal payload carries the handler's typed result, proving the dict + # was validated into _TypedRequest and processed. + assert state_data["terminal_payload"]["status"] == "success" + finally: + sys.modules.pop("_test_typed_op_match_handler", None) diff --git a/tests/unit/runtime/test_runtime_local_terminal_reducer.py b/tests/unit/runtime/test_runtime_local_terminal_reducer.py index 56261252be..4fc2bb4f73 100644 --- a/tests/unit/runtime/test_runtime_local_terminal_reducer.py +++ b/tests/unit/runtime/test_runtime_local_terminal_reducer.py @@ -41,6 +41,7 @@ def test_terminal_reducer_no_padding_validates_cleanly() -> None: is valid for a terminal reducer (gets no bus subscription). """ routing = { + "routing_strategy": "payload_type_match", "handlers": [ { "event_model": {"name": "EventA", "module": "mod.events"}, @@ -52,7 +53,7 @@ def test_terminal_reducer_no_padding_validates_cleanly() -> None: "handler": {"name": "TerminalReducer", "module": "mod.handlers"}, "output_events": [], }, - ] + ], } errors = RuntimeLocal._validate_routing( routing, diff --git a/tests/unit/runtime/test_runtime_profile.py b/tests/unit/runtime/test_runtime_profile.py index 8e20b45553..d4b75452b8 100644 --- a/tests/unit/runtime/test_runtime_profile.py +++ b/tests/unit/runtime/test_runtime_profile.py @@ -4,7 +4,11 @@ from __future__ import annotations -from omnibase_infra.runtime.runtime_profile import load_runtime_profile +from omnibase_core.constants.constants_runtime_profiles import ( + CONSUMER_ATTACHED_RUNTIME_PROFILES, + REGISTERED_RUNTIME_PROFILES, +) +from omnibase_infra.runtime.runtime_profile import _PROFILES, load_runtime_profile def test_runtime_lane_profiles_preserve_identity() -> None: @@ -15,3 +19,21 @@ def test_runtime_lane_profiles_preserve_identity() -> None: def test_unknown_runtime_profile_still_falls_back_to_default() -> None: assert load_runtime_profile("unknown-lane").name == "default" + + +def test_profiles_match_core_registry() -> None: + """OMN-12957: infra _PROFILES keys are the canonical core registry. + + Core owns the profile name set so contract validation can enforce + ``runtime_profiles ⊆ REGISTERED_RUNTIME_PROFILES`` without a core→infra + dependency. If this drifts, a contract could name a profile core blesses + but infra cannot boot (or vice versa) — a silent-orphan hazard. The module + import already asserts this; the test makes the contract explicit. + """ + assert frozenset(_PROFILES) == REGISTERED_RUNTIME_PROFILES + + +def test_consumer_attached_profiles_actually_load() -> None: + """Every consumer-attached profile must be a real, loadable runtime lane.""" + for profile_name in CONSUMER_ATTACHED_RUNTIME_PROFILES: + assert load_runtime_profile(profile_name).name == profile_name diff --git a/tests/unit/runtime/test_service_pattern_b_broker.py b/tests/unit/runtime/test_service_pattern_b_broker.py index ebdd75a0bc..3b4e05a159 100644 --- a/tests/unit/runtime/test_service_pattern_b_broker.py +++ b/tests/unit/runtime/test_service_pattern_b_broker.py @@ -116,6 +116,10 @@ def __init__( self.topics_calls = 0 self.set_topics_calls: list[tuple[str, ...]] = [] self.seeked_to_end = False + # True once the consume leg has pinned its read position to the current + # end — either the legacy lazy ``seek_to_end`` or, post-OMN-13118, the + # synchronous ``end_offsets`` + ``seek`` pinning. + self.positioned = False self.started = False self.stopped = False type(self).created.append(self) @@ -155,6 +159,18 @@ def assignment(self) -> set[object]: async def seek_to_end(self, *_assignment: object) -> None: await asyncio.sleep(0) self.seeked_to_end = True + self.positioned = True + + async def end_offsets(self, partitions: object) -> dict[object, int]: + # Synchronous broker round-trip returning the current HWM per partition + # (queue-backed fake delivers regardless of offset, so 0 is faithful for + # the "next message" offset here). Does not change position. + await asyncio.sleep(0) + return dict.fromkeys(partitions, 0) + + def seek(self, _partition: object, _offset: int) -> None: + # Synchronous pin (OMN-13118): the position is fixed at open() time. + self.positioned = True async def getone(self) -> SimpleNamespace: return await self.messages.get() @@ -233,12 +249,28 @@ async def publish( _headers: object | None = None, ) -> None: assert self._consumers - consumer = self._consumers[-1] - if self._assert_seeked: - assert consumer.seeked_to_end is True - command_envelope = ModelEventEnvelope[object].model_validate_json(value) terminal_topic = self._terminal_topic or self._route.terminal_event or "unknown" + + # OMN-13128: the broker opens one INDEPENDENT consumer per terminal topic. + # Deliver the terminal to the consumer that actually holds the terminal + # topic's partition (mirrors the real broker producing to a topic and the + # subscribed consumer reading it). Fall back to the last-created consumer + # for the single-terminal-topic routes where there is exactly one. + target = next( + ( + consumer + for consumer in self._consumers + if any( + getattr(partition, "topic", "") == terminal_topic + for partition in consumer.assigned_partitions + ) + ), + self._consumers[-1], + ) + if self._assert_seeked: + assert all(consumer.positioned is True for consumer in self._consumers) + terminal_envelope = ModelEventEnvelope[object]( payload=self._terminal_payload, correlation_id=command_envelope.correlation_id, @@ -246,7 +278,7 @@ async def publish( event_type=terminal_topic, source_tool="session_orchestrator", ) - await consumer.messages.put( + await target.messages.put( SimpleNamespace( topic=terminal_topic, value=terminal_envelope.model_dump_json().encode(), @@ -391,7 +423,7 @@ async def worker(message: ModelEventMessage) -> None: @pytest.mark.asyncio -async def test_service_pattern_b_broker_kafka_waiter_seeks_before_dispatch( +async def test_service_pattern_b_broker_kafka_waiter_pins_end_offset_before_dispatch( monkeypatch: pytest.MonkeyPatch, ) -> None: route = _route() @@ -481,43 +513,83 @@ async def test_service_pattern_b_broker_kafka_waiter_consumes_failure_terminal( resolved_route, result = await broker.dispatch_request(command) assert resolved_route == route - assert created_consumers[0].topics == () - assert created_consumers[0].kwargs["group_id"] is None - assert { - getattr(partition, "topic", "") - for partition in created_consumers[0].assigned_partitions - } == set(route.terminal_events) + # OMN-13128: one INDEPENDENT consumer per terminal topic, never one consumer + # holding both subscriptions. The broker opens completed then failed, so two + # consumers are created, each assigned the partitions of EXACTLY ONE topic. + assert len(created_consumers) == len(route.terminal_events) + assert all(consumer.topics == () for consumer in created_consumers) + assert all(consumer.kwargs["group_id"] is None for consumer in created_consumers) + per_consumer_topics = [ + {getattr(partition, "topic", "") for partition in consumer.assigned_partitions} + for consumer in created_consumers + ] + # Each consumer's assignment is confined to a single terminal topic, and the + # union across the independent consumers covers both terminal topics. + assert all(len(topics) == 1 for topics in per_consumer_topics) + assert set().union(*per_consumer_topics) == set(route.terminal_events) assert result.status == "failed" assert result.error_message == "routing contract missing" - assert created_consumers[0].stopped is True + assert all(consumer.stopped is True for consumer in created_consumers) @pytest.mark.asyncio -async def test_service_pattern_b_broker_kafka_waiter_requires_all_terminal_topics( +async def test_service_pattern_b_broker_correlates_completed_when_failed_topic_partitionless( monkeypatch: pytest.MonkeyPatch, ) -> None: + """OMN-13128: a never-produced FAILED topic must not block the COMPLETED win. + + The runtime opens ONE independent consumer per terminal topic. When the FAILED + reply topic has never been produced to (zero partitions advertised — the + Redpanda auto-create-on-produce steady state, #1970), its consumer assigns the + empty set and its poll sleeps out the timeout, while the COMPLETED consumer — + its own independent subscription, positioned pre-publish — correlates the + terminal and wins the race. The prior single-consumer-both-topics shape could + not express this: it assigned both topics to one consumer and required BOTH to + surface partitions before it would assign at all. + """ route = _route_with_failure_terminal() - _install_fake_aiokafka_consumer(monkeypatch) + completed_topic, failed_topic = route.terminal_events + created_consumers = _install_fake_aiokafka_consumer(monkeypatch) + # COMPLETED has a partition (produced to); FAILED never has (partition-less). _FakeAIOKafkaConsumer.topic_partitions_by_topic = { - route.terminal_events[0]: {0}, - route.terminal_events[1]: set(), + completed_topic: {0}, + failed_topic: set(), } - consumer = _FakeAIOKafkaConsumer() - await consumer.start() + _FakeAIOKafkaConsumer.require_metadata_refresh = True + broker = RuntimePatternBBroker( - _FakeKafkaTransport(route, [consumer]), + _FakeKafkaTransport(route, created_consumers, terminal_topic=completed_topic), command_topic="onex.cmd.omnibase-infra.pattern-b-dispatch.v1", routes={"session_orchestrator": route}, ) + command = ModelDispatchBusCommand( + command_name="session_orchestrator", + requester="codex", + payload={"dry_run": True}, + response_topic="onex.evt.pattern-b.dispatch-completed.v1", + timeout_seconds=1, + ) - with pytest.raises(TimeoutError): - await broker._assign_terminal_topic_partitions( - consumer, - route.terminal_events, - timeout_seconds=0, - ) + resolved_route, result = await broker.dispatch_request(command) - assert consumer.assigned_partitions == set() + assert resolved_route == route + # Two independent consumers were opened, one per terminal topic. + assert len(created_consumers) == len(route.terminal_events) + completed_consumer = next( + c + for c in created_consumers + if any( + getattr(p, "topic", "") == completed_topic for p in c.assigned_partitions + ) + ) + failed_consumer = next(c for c in created_consumers if c is not completed_consumer) + # The COMPLETED consumer holds its real partition; the partition-less FAILED + # consumer assigns the empty set and never blocks the COMPLETED win. + assert completed_consumer.assigned_partitions + assert not failed_consumer.assigned_partitions + assert result.status == "completed" + assert result.payload == {"status": "complete", "dispatch_count": 5} + assert all(consumer.stopped is True for consumer in created_consumers) @pytest.mark.asyncio @@ -820,8 +892,10 @@ def __init__( monkeypatch.setattr( "omnibase_infra.runtime.runtime_host_process.discover_runtime_local_ingress_routes", - lambda packages: captured.setdefault("packages", packages) - and {"session_orchestrator": route}, + lambda packages: ( + captured.setdefault("packages", packages) + and {"session_orchestrator": route} + ), ) monkeypatch.setattr( "omnibase_infra.runtime.runtime_host_process.RuntimePatternBBroker", @@ -936,8 +1010,10 @@ def __init__( monkeypatch.setenv("ONEX_ACTIVE_RUNTIME_PACKAGES", "omnibase_infra") monkeypatch.setattr( "omnibase_infra.runtime.runtime_host_process.discover_runtime_local_ingress_routes", - lambda packages: captured.setdefault("packages", packages) - and {"session_orchestrator": route}, + lambda packages: ( + captured.setdefault("packages", packages) + and {"session_orchestrator": route} + ), ) monkeypatch.setattr( "omnibase_infra.runtime.runtime_host_process.RuntimePatternBBroker", diff --git a/tests/unit/runtime/test_terminal_correlator_resolve_before_await.py b/tests/unit/runtime/test_terminal_correlator_resolve_before_await.py new file mode 100644 index 0000000000..eeab4004a3 --- /dev/null +++ b/tests/unit/runtime/test_terminal_correlator_resolve_before_await.py @@ -0,0 +1,304 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Focused regression for the P_RESOLVE_BEFORE_AWAIT correlator race (OMN-13118). + +The live wedge (and the defect this test pins) is a delivery-ordering race in +``LongLivedTerminalCorrelator``: the always-on poll loop runs continuously, so it +typically consumes and matches the correlated terminal BEFORE the runner's +``wait()`` attaches. The publish -> generation -> wait latency is large (~28s +observed live), so by the time the runner blocks on its future the terminal has +already been delivered. + +Before the fix, ``_deliver`` popped the future on match. When delivery beat the +awaiter, the resolved future was removed from the registry, so the late ``_wait`` +found nothing, created a fresh (unresolved) future, and timed out at the trial +cap -> ``None`` -> degenerate generation-failure row -> zero rows. + +The fix: ``_deliver`` resolves the future IN PLACE (``get``, not ``pop``); the +awaiter (``_wait``) owns removal and pops on exit (``finally``). A future resolved +before the awaiter attaches therefore survives in the registry until ``_wait`` +reads it. + +These tests prove the resolve-before-await ordering deterministically: register +the cid, append the terminal, and wait (in the test) until the registry future is +``done()`` — i.e. ``_deliver`` has already resolved it IN PLACE — and ONLY THEN +call ``wait()``. The fix means ``wait()`` returns the body; the pre-fix +pop-on-match would have returned ``None``/timed out. +""" + +from __future__ import annotations + +import json +import threading +import time +from types import SimpleNamespace +from typing import Any, ClassVar + +import pytest + +from omnibase_infra.runtime.service_terminal_event_consumer import ( + LongLivedTerminalCorrelator, +) + +pytestmark = [pytest.mark.unit] + +_TERMINAL_TOPIC = "onex.evt.omnimarket.node-generation-completed.v1" +_HANDLER = "HandlerContextRoiRunner" + + +class _PartitionLog: + """Append-only partition log for one terminal topic with a high-water-mark.""" + + def __init__(self) -> None: + self._records: list[tuple[str, bytes]] = [] + self._lock = threading.Lock() + + @property + def hwm(self) -> int: + with self._lock: + return len(self._records) + + def append(self, correlation_id: str, value: bytes) -> None: + with self._lock: + self._records.append((correlation_id, value)) + + def read_at(self, offset: int) -> tuple[str, bytes] | None: + with self._lock: + if offset < len(self._records): + return self._records[offset] + return None + + +class _FakeClient: + def __init__(self) -> None: + self.tracked: set[str] = set() + + async def _refresh(self) -> bool: + return True + + def set_topics(self, topics: list[str]) -> Any: + self.tracked = set(topics) + return self._refresh() + + def force_metadata_update(self) -> Any: + return self._refresh() + + +class _FakeConsumer: + """Single long-lived ``AIOKafkaConsumer`` stand-in over one terminal topic.""" + + log: ClassVar[_PartitionLog | None] = None + + def __init__(self, *args: object, **kwargs: object) -> None: + self._client = _FakeClient() + self._assigned: set[Any] = set() + self._cursor = 0 + + async def start(self) -> None: + return None + + def partitions_for_topic(self, topic: str) -> set[int]: + self._client.tracked.add(topic) + return {0} + + def assign(self, partitions: Any) -> None: + self._assigned = set(partitions) + + def assignment(self) -> set[Any]: + return set(self._assigned) + + async def seek_to_end(self, *assignment: object) -> None: + return None + + async def end_offsets(self, partitions: Any) -> dict[Any, int]: + assert type(self).log is not None + return dict.fromkeys(partitions, type(self).log.hwm) + + def seek(self, partition: Any, offset: int) -> None: + self._cursor = offset + + async def getone(self) -> SimpleNamespace: + import asyncio + + assert type(self).log is not None + while True: + record = type(self).log.read_at(self._cursor) + if record is not None: + self._cursor += 1 + _cid, value = record + return SimpleNamespace(topic=_TERMINAL_TOPIC, value=value) + await asyncio.sleep(0.002) + + async def stop(self) -> None: + return None + + +class _KafkaLikeBus: + config = SimpleNamespace( + session_timeout_ms=45000, + heartbeat_interval_ms=15000, + max_poll_interval_ms=1800000, + reconnect_backoff_ms=2000, + ) + _bootstrap_servers = "omn-13118-resolve-before-await-broker" + + def _build_auth_kwargs(self) -> dict[str, object]: + return {} + + +def _terminal_envelope_json(correlation_id: str) -> bytes: + return json.dumps( + { + "correlation_id": correlation_id, + "payload": { + "attempt_count": 1, + "contract_passed": True, + "model_id": "Qwen3.6-35B-A3B", + "provider": "local", + "prompt_tokens": 128, + "completion_tokens": 64, + }, + } + ).encode("utf-8") + + +def _await_future_resolved( + correlator: LongLivedTerminalCorrelator, + correlation_id: str, + timeout_seconds: float = 5.0, +) -> bool: + """Block until the registry future for ``correlation_id`` is ``done()``. + + Returning True proves ``_deliver`` matched the terminal and resolved the + future IN PLACE *before* any ``wait()`` attached — the exact resolve-before- + await ordering. The future is read on the worker loop (where it lives) via a + thread-safe submit to avoid touching loop state from the test thread. + """ + import asyncio + + deadline = time.monotonic() + timeout_seconds + + async def _is_done() -> bool: + future = correlator._pending.get(correlation_id) + return future is not None and future.done() + + while time.monotonic() < deadline: + resolved = asyncio.run_coroutine_threadsafe( + _is_done(), correlator._loop + ).result(timeout=1.0) + if resolved: + return True + time.sleep(0.005) + return False + + +@pytest.mark.timeout(30) +def test_wait_returns_body_when_terminal_resolved_before_await( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """register -> deliver (resolved IN PLACE) -> THEN wait must return the body. + + This is the P_RESOLVE_BEFORE_AWAIT fix: the poll loop delivers and matches the + terminal before ``wait()`` attaches. The test makes the ordering deterministic + by blocking on the registry future being ``done()`` (proving ``_deliver`` + resolved it in place) BEFORE calling ``wait()``. + + Pre-fix (``_deliver`` popped on match): the resolved future was removed, the + late ``wait()`` created a fresh future and timed out -> ``None``. The + resolve-in-place fix keeps the resolved future in the registry until ``wait()`` + reads it, then ``_wait`` pops it on exit. + """ + log = _PartitionLog() + _FakeConsumer.log = log + monkeypatch.setattr( + "omnibase_infra.runtime.service_pattern_b_broker.AIOKafkaConsumer", + _FakeConsumer, + ) + + correlator = LongLivedTerminalCorrelator( + event_bus=_KafkaLikeBus(), handler_name=_HANDLER + ) + cid = "omn-13118-resolve-before-await-0001" + try: + # 1. Subscribe + register the cid BEFORE the terminal exists (pre-publish). + correlator.ensure_topic(_TERMINAL_TOPIC) + correlator.register(cid) + + # 2. Deliver the terminal. The always-on poll loop consumes + matches it. + log.append(cid, _terminal_envelope_json(cid)) + + # 3. Block (in the test) until the registry future is resolved IN PLACE by + # _deliver — i.e. delivery has provably BEATEN the awaiter. This is the + # late-awaiter ordering the fix targets. + assert _await_future_resolved(correlator, cid), ( + "the poll loop never resolved the registry future for the matched " + "terminal — _deliver did not match/resolve in place" + ) + + # 4. ONLY NOW attach the awaiter. With the resolve-in-place fix the body + # is still in the registry; pre-fix pop-on-match would return None. + result = correlator.wait(cid, timeout_seconds=2.0) + assert result is not None, ( + "wait() returned None for a terminal that was delivered and resolved " + "BEFORE the awaiter attached — the P_RESOLVE_BEFORE_AWAIT race " + "(_deliver must resolve in place, not pop on match)" + ) + assert str(result.get("correlation_id")) == cid + + # 5. The awaiter owns removal: the cid is popped from the registry on exit. + import asyncio + + still_pending = asyncio.run_coroutine_threadsafe( + _pending_contains(correlator, cid), correlator._loop + ).result(timeout=1.0) + assert not still_pending, ( + "_wait must pop the cid on exit so a resolved-and-read future does not " + "linger in the registry" + ) + finally: + correlator.close() + + +@pytest.mark.timeout(30) +def test_wait_still_returns_body_when_terminal_arrives_during_await( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Sibling ordering: terminal arrives DURING the wait — body still returned. + + Confirms the fix does not regress the in-window delivery path: the awaiter + attaches first, then the terminal lands and the poll loop resolves the + already-attached future. + """ + log = _PartitionLog() + _FakeConsumer.log = log + monkeypatch.setattr( + "omnibase_infra.runtime.service_pattern_b_broker.AIOKafkaConsumer", + _FakeConsumer, + ) + + correlator = LongLivedTerminalCorrelator( + event_bus=_KafkaLikeBus(), handler_name=_HANDLER + ) + cid = "omn-13118-resolve-during-await-0001" + try: + correlator.ensure_topic(_TERMINAL_TOPIC) + correlator.register(cid) + + def _publish_during_wait() -> None: + time.sleep(0.2) + log.append(cid, _terminal_envelope_json(cid)) + + threading.Thread(target=_publish_during_wait, daemon=True).start() + result = correlator.wait(cid, timeout_seconds=5.0) + assert result is not None, ( + "wait() returned None for a terminal that arrived during the await" + ) + assert str(result.get("correlation_id")) == cid + finally: + correlator.close() + + +async def _pending_contains( + correlator: LongLivedTerminalCorrelator, correlation_id: str +) -> bool: + return correlation_id in correlator._pending diff --git a/tests/unit/runtime/test_validate_routing_operation_match.py b/tests/unit/runtime/test_validate_routing_operation_match.py new file mode 100644 index 0000000000..ca015cedc2 --- /dev/null +++ b/tests/unit/runtime/test_validate_routing_operation_match.py @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Unit tests for OMN-13137: _validate_routing strategy-aware event_model checks. + +Root cause: _validate_routing unconditionally required event_model.name/module for +every handler entry. But event_model is only meaningful for routing_strategy: +payload_type_match — operation_match routes by the `operation` field and does not +use event_model. 230/295 omnimarket contracts are operation_match without +event_model and are correct as authored. + +Fix: read routing_strategy from the routing map and branch validation: + - payload_type_match → require event_model.{name, module} + - operation_match (and anything else) → require operation; skip event_model checks + +These tests prove the fix and serve as the boot gate (test 4 loads the real +node_integration_sweep_orchestrator handler_routing block and asserts clean +validation). +""" + +from __future__ import annotations + +import pytest + +from omnibase_infra.runtime.runtime_local import RuntimeLocal + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_OPERATION_MATCH_CLEAN_ROUTING: dict[str, object] = { + "routing_strategy": "operation_match", + "handlers": [ + { + "operation": "sweep", + "handler": { + "name": "HandlerIntegrationSweepOrchestrator", + "module": ( + "omnimarket.nodes.node_integration_sweep_orchestrator" + ".handlers.handler_integration_sweep_orchestrator" + ), + }, + }, + { + "operation": "surface_probes", + "handler": { + "name": "surface_probes", + "module": ( + "omnimarket.nodes.node_integration_sweep_orchestrator" + ".handlers.surface_probes" + ), + }, + }, + ], +} + +# --------------------------------------------------------------------------- +# Test 1 — operation_match without event_model must produce zero event_model errors +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_operation_match_no_event_model_produces_no_event_model_errors() -> None: + """operation_match entry with no event_model → _validate_routing returns no event_model errors. + + Before the fix this produced two errors per handler: + handlers[0].event_model.name is missing + handlers[0].event_model.module is missing + """ + routing: dict[str, object] = { + "routing_strategy": "operation_match", + "handlers": [ + { + "operation": "do_thing", + "handler": {"name": "HandlerDoThing", "module": "mod.handlers"}, + } + ], + } + errors = RuntimeLocal._validate_routing( + routing, + subscribe_topics=[], + publish_topics=[], + ) + event_model_errors = [e for e in errors if "event_model" in e] + assert event_model_errors == [], ( + f"Expected no event_model errors for operation_match, got: {event_model_errors}" + ) + + +# --------------------------------------------------------------------------- +# Test 2 — operation_match missing `operation` field must error +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_operation_match_missing_operation_produces_error() -> None: + """operation_match entry missing `operation` → error reported.""" + routing: dict[str, object] = { + "routing_strategy": "operation_match", + "handlers": [ + { + # no 'operation' key + "handler": {"name": "HandlerFoo", "module": "mod.handlers"}, + } + ], + } + errors = RuntimeLocal._validate_routing( + routing, + subscribe_topics=[], + publish_topics=[], + ) + operation_errors = [e for e in errors if "operation" in e] + assert operation_errors, ( + "Expected an error about missing 'operation' field, but got none. " + f"All errors: {errors}" + ) + + +# --------------------------------------------------------------------------- +# Test 3 — payload_type_match missing event_model still errors (unchanged) +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_payload_type_match_missing_event_model_still_errors() -> None: + """payload_type_match entry missing event_model → errors unchanged (regression guard).""" + routing: dict[str, object] = { + "routing_strategy": "payload_type_match", + "handlers": [ + { + # no 'event_model' key + "handler": {"name": "HandlerFoo", "module": "mod.handlers"}, + } + ], + } + errors = RuntimeLocal._validate_routing( + routing, + subscribe_topics=[], + publish_topics=[], + ) + event_model_errors = [e for e in errors if "event_model" in e] + assert len(event_model_errors) == 2, ( + "Expected exactly 2 event_model errors (name + module) for payload_type_match, " + f"got: {event_model_errors}" + ) + + +# --------------------------------------------------------------------------- +# Test 4 — real node_integration_sweep_orchestrator handler_routing validates clean +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_node_integration_sweep_orchestrator_routing_validates_clean() -> None: + """Real node_integration_sweep_orchestrator contract handler_routing → zero errors. + + Boot gate: this is the canonical operation_match contract that was previously + failing with spurious event_model errors. After OMN-13137, _validate_routing + must return an empty list for this routing block. + """ + errors = RuntimeLocal._validate_routing( + _OPERATION_MATCH_CLEAN_ROUTING, # type: ignore[arg-type] + subscribe_topics=[], + publish_topics=["onex.evt.omnimarket.integration-sweep-completed.v1"], + ) + assert errors == [], ( + f"node_integration_sweep_orchestrator routing unexpectedly failed: {errors}" + ) diff --git a/tests/unit/scripts/test_check_sibling_lock_pins.py b/tests/unit/scripts/test_check_sibling_lock_pins.py new file mode 100644 index 0000000000..92c6485f11 --- /dev/null +++ b/tests/unit/scripts/test_check_sibling_lock_pins.py @@ -0,0 +1,305 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Unit tests for scripts/runtime_build/check_sibling_lock_pins.py [OMN-12977]. + +Recurrence guard for the 2026-06-11 stability bootstrap crash: the workspace-mode +image build vendored omnibase_infra 0.37.0-dev (pre-OMN-12501 guard) while +omnimarket dev's uv.lock pinned omnibase-infra 0.38.1 @ e2dbdc95. The build +ignored the lock and shipped a 13-day-stale sibling. + +These tests pin the lock-parsing + pin-comparison contract so the preflight can +fail-fast on any future expected-vs-actual mismatch. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +SCRIPT_PATH = ( + Path(__file__).resolve().parents[3] + / "scripts" + / "runtime_build" + / "check_sibling_lock_pins.py" +) + +sys.path.insert(0, str(SCRIPT_PATH.parent)) + +from check_sibling_lock_pins import ( + ModelActualPin, + ModelLockPin, + ModelPinComparison, + check_pins, + compare_pins, + parse_lock_pins, +) + +# A minimal but faithful slice of omnimarket dev's uv.lock as it stood on +# 2026-06-11: omnibase-infra/omnibase-core are git-pinned with rev; omnibase-spi +# is a registry (PyPI) pin with no git rev. +LOCK_FIXTURE = """\ +[[package]] +name = "omnibase-core" +version = "0.44.0" +source = { git = "https://github.com/OmniNode-ai/omnibase_core.git?rev=c97c2c9a45c5fb0def5fb7dacfd5f01278bb9f55#c97c2c9a45c5fb0def5fb7dacfd5f01278bb9f55" } +dependencies = [ + { name = "pydantic" }, +] + +[[package]] +name = "omnibase-infra" +version = "0.38.1" +source = { git = "https://github.com/OmniNode-ai/omnibase_infra.git?rev=e2dbdc950540df8bc59ca4370b2d4a0f5b8d6c59#e2dbdc950540df8bc59ca4370b2d4a0f5b8d6c59" } +dependencies = [ + { name = "a2a-sdk" }, +] + +[[package]] +name = "omnibase-spi" +version = "0.20.6" +source = { registry = "https://pypi.org/simple" } + +[[package]] +name = "omnimarket" +version = "0.4.3" +source = { editable = "." } +dependencies = [ + { name = "omnibase-infra", git = "https://github.com/OmniNode-ai/omnibase_infra.git?rev=e2dbdc950540df8bc59ca4370b2d4a0f5b8d6c59" }, + { name = "omnibase-compat", git = "https://github.com/OmniNode-ai/omnibase_compat.git?rev=4d887307aae34d9d40d389ba91070cb411ce3df5" }, +] + +[[package]] +name = "unrelated-package" +version = "9.9.9" +source = { registry = "https://pypi.org/simple" } +""" + + +@pytest.mark.unit +class TestParseLockPins: + def test_extracts_git_pinned_infra_version_and_rev(self) -> None: + pins = parse_lock_pins(LOCK_FIXTURE, packages=["omnibase-infra"]) + assert "omnibase-infra" in pins + pin = pins["omnibase-infra"] + assert pin.version == "0.38.1" + assert pin.git_rev == "e2dbdc950540df8bc59ca4370b2d4a0f5b8d6c59" + + def test_extracts_git_pinned_core_version_and_rev(self) -> None: + pins = parse_lock_pins(LOCK_FIXTURE, packages=["omnibase-core"]) + pin = pins["omnibase-core"] + assert pin.version == "0.44.0" + assert pin.git_rev == "c97c2c9a45c5fb0def5fb7dacfd5f01278bb9f55" + + def test_registry_pin_has_no_git_rev(self) -> None: + pins = parse_lock_pins(LOCK_FIXTURE, packages=["omnibase-spi"]) + pin = pins["omnibase-spi"] + assert pin.version == "0.20.6" + assert pin.git_rev is None + + def test_only_requested_packages_returned(self) -> None: + pins = parse_lock_pins( + LOCK_FIXTURE, packages=["omnibase-infra", "omnibase-core"] + ) + assert set(pins) == {"omnibase-infra", "omnibase-core"} + + def test_editable_package_with_git_deps_has_no_own_rev(self) -> None: + # Regression: omnimarket is source = { editable = "." } but its + # dependencies carry ?rev= for git deps. A block-wide rev search would + # wrongly attribute a dependency's rev to omnimarket. Its own rev is None. + pins = parse_lock_pins(LOCK_FIXTURE, packages=["omnimarket"]) + assert pins["omnimarket"].version == "0.4.3" + assert pins["omnimarket"].git_rev is None + + def test_missing_package_raises(self) -> None: + # Fail-fast: a requested foundation package absent from the lock is a + # build-composition error, not something to silently skip. + with pytest.raises(KeyError): + parse_lock_pins(LOCK_FIXTURE, packages=["omnibase-infra", "does-not-exist"]) + + +@pytest.mark.unit +class TestComparePins: + def test_version_and_rev_match_is_ok(self) -> None: + expected = ModelLockPin( + package="omnibase-infra", + version="0.38.1", + git_rev="e2dbdc950540df8bc59ca4370b2d4a0f5b8d6c59", + ) + actual = ModelActualPin( + package="omnibase-infra", + version="0.38.1", + git_sha="e2dbdc950540df8bc59ca4370b2d4a0f5b8d6c59", + ) + result = compare_pins(expected, actual) + assert isinstance(result, ModelPinComparison) + assert result.matches is True + assert result.mismatch_reason == "" + + def test_stale_version_is_mismatch(self) -> None: + # The exact 2026-06-11 failure: vendored 0.37.0 vs locked 0.38.1. + expected = ModelLockPin( + package="omnibase-infra", + version="0.38.1", + git_rev="e2dbdc950540df8bc59ca4370b2d4a0f5b8d6c59", + ) + actual = ModelActualPin( + package="omnibase-infra", + version="0.37.0", + git_sha="2c1d672f00000000000000000000000000000000", + ) + result = compare_pins(expected, actual) + assert result.matches is False + assert "0.38.1" in result.mismatch_reason + assert "0.37.0" in result.mismatch_reason + + def test_version_match_but_sha_drift_is_mismatch(self) -> None: + expected = ModelLockPin( + package="omnibase-infra", + version="0.38.1", + git_rev="e2dbdc950540df8bc59ca4370b2d4a0f5b8d6c59", + ) + actual = ModelActualPin( + package="omnibase-infra", + version="0.38.1", + git_sha="ffffffffffffffffffffffffffffffffffffffff", + ) + result = compare_pins(expected, actual) + assert result.matches is False + assert "ffffffff" in result.mismatch_reason + + def test_registry_pin_only_checks_version(self) -> None: + # spi is a registry pin: no git rev to compare, version equality decides. + expected = ModelLockPin(package="omnibase-spi", version="0.20.6", git_rev=None) + actual = ModelActualPin( + package="omnibase-spi", + version="0.20.6", + git_sha="anything-here-is-ignored", + ) + result = compare_pins(expected, actual) + assert result.matches is True + + def test_comparison_serializes_for_provenance(self) -> None: + expected = ModelLockPin( + package="omnibase-infra", + version="0.38.1", + git_rev="e2dbdc950540df8bc59ca4370b2d4a0f5b8d6c59", + ) + actual = ModelActualPin( + package="omnibase-infra", version="0.37.0", git_sha="2c1d672f" + ) + result = compare_pins(expected, actual) + payload = json.loads(result.model_dump_json()) + assert payload["package"] == "omnibase-infra" + assert payload["expected_version"] == "0.38.1" + assert payload["actual_version"] == "0.37.0" + assert payload["matches"] is False + assert payload["drift_direction"] == "backward" + + +@pytest.mark.unit +class TestDriftDirection: + def test_stale_clone_is_backward(self) -> None: + # The exact 2026-06-11 crash: clone 0.37.0 older than locked 0.38.1. + expected = ModelLockPin(package="omnibase-infra", version="0.38.1") + actual = ModelActualPin( + package="omnibase-infra", version="0.37.0", git_sha="2c1d672f" + ) + assert compare_pins(expected, actual).drift_direction == "backward" + + def test_newer_clone_is_forward(self) -> None: + # Routine state: clones lead the lock until the bot bumps it. + expected = ModelLockPin(package="omnibase-infra", version="0.38.1") + actual = ModelActualPin( + package="omnibase-infra", version="0.38.3", git_sha="b7c93a8e" + ) + assert compare_pins(expected, actual).drift_direction == "forward" + + def test_match_is_none(self) -> None: + expected = ModelLockPin(package="omnibase-spi", version="0.20.6") + actual = ModelActualPin(package="omnibase-spi", version="0.20.6", git_sha="x") + assert compare_pins(expected, actual).drift_direction == "none" + + def test_non_numeric_version_is_unknown(self) -> None: + expected = ModelLockPin(package="omnimarket", version="0.4.3") + actual = ModelActualPin(package="omnimarket", version="0.4.3-dev", git_sha="x") + assert compare_pins(expected, actual).drift_direction == "unknown" + + +@pytest.mark.unit +class TestCheckPins: + def _write_lock(self, tmp_path: Path) -> Path: + lock = tmp_path / "uv.lock" + lock.write_text(LOCK_FIXTURE, encoding="utf-8") + return lock + + def _make_clone(self, tmp_path: Path, name: str, version: str) -> Path: + import subprocess + + root = tmp_path / name + root.mkdir() + (root / "pyproject.toml").write_text( + f'[project]\nname = "{name}"\nversion = "{version}"\n', + encoding="utf-8", + ) + subprocess.run(["git", "init", "-q", str(root)], check=True) + subprocess.run(["git", "-C", str(root), "add", "-A"], check=True) + subprocess.run( + [ + "git", + "-C", + str(root), + "-c", + "user.email=t@t", + "-c", + "user.name=t", + "commit", + "-qm", + "init", + ], + check=True, + ) + return root + + def test_missing_lock_returns_2(self, tmp_path: Path) -> None: + rc = check_pins( + tmp_path / "nope.lock", + {"omnibase-infra": tmp_path}, + output_path=None, + ) + assert rc == 2 + + def test_version_match_passes(self, tmp_path: Path) -> None: + lock = self._write_lock(tmp_path) + # spi is the registry pin (version-only): a matching clone passes. + clone = self._make_clone(tmp_path, "omnibase-spi", "0.20.6") + out = tmp_path / "out.json" + rc = check_pins(lock, {"omnibase-spi": clone}, output_path=out) + assert rc == 0 + payload = json.loads(out.read_text()) + assert payload["drift_count"] == 0 + assert payload["allow_drift"] is False + + def test_backward_drift_aborts(self, tmp_path: Path) -> None: + lock = self._write_lock(tmp_path) + clone = self._make_clone(tmp_path, "omnibase-spi", "0.19.0") + out = tmp_path / "out.json" + rc = check_pins(lock, {"omnibase-spi": clone}, output_path=out) + assert rc == 1 + payload = json.loads(out.read_text()) + assert payload["drift_count"] == 1 + assert payload["comparisons"][0]["drift_direction"] == "backward" + + def test_allow_drift_records_and_proceeds(self, tmp_path: Path) -> None: + lock = self._write_lock(tmp_path) + clone = self._make_clone(tmp_path, "omnibase-spi", "0.19.0") + out = tmp_path / "out.json" + rc = check_pins( + lock, {"omnibase-spi": clone}, output_path=out, allow_drift=True + ) + assert rc == 0 + payload = json.loads(out.read_text()) + assert payload["allow_drift"] is True + assert payload["drift_count"] == 1 diff --git a/tests/unit/scripts/test_disk_gc_dry_run.py b/tests/unit/scripts/test_disk_gc_dry_run.py new file mode 100644 index 0000000000..6341b545a8 --- /dev/null +++ b/tests/unit/scripts/test_disk_gc_dry_run.py @@ -0,0 +1,159 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Dry-run safety smoke tests for the GC shell scripts (OMN-13008). + +The single most important property: in DEFAULT (dry-run) mode the scripts must +NEVER call a destructive docker/git subcommand. We prove this by shimming `docker`, +`git`, and `rpk` on PATH with a recorder that logs every invocation, then asserting +no destructive verb (rmi, rm, prune, produce, 'worktree remove') was issued. +""" + +from __future__ import annotations + +import os +import stat +import subprocess +from pathlib import Path + +import pytest + +_REPO = Path(__file__).resolve().parents[3] +_SCRIPTS = _REPO / "scripts" + +DESTRUCTIVE_TOKENS = ("rmi", "prune", "produce", "remove", "rm ") + + +def _make_shim(bin_dir: Path, name: str, calllog: Path) -> None: + """Create an executable shim that records its argv and exits 0 with empty output.""" + shim = bin_dir / name + shim.write_text( + "#!/usr/bin/env bash\n" + f'echo "{name} $*" >> "{calllog}"\n' + # image ls / ps queries must return empty so planner produces an empty plan. + "exit 0\n" + ) + shim.chmod(shim.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + + +def _run( + script: str, args: list[str], tmp_path: Path +) -> tuple[subprocess.CompletedProcess[str], str]: + bin_dir = tmp_path / "bin" + bin_dir.mkdir(exist_ok=True) + calllog = tmp_path / "calls.log" + calllog.write_text("") + for tool in ("docker", "git", "rpk"): + _make_shim(bin_dir, tool, calllog) + + env = dict(os.environ) + env["PATH"] = f"{bin_dir}:{env['PATH']}" + env["HOME"] = str(tmp_path) # redirect log files into tmp + # Ensure no ambient broker leaks in — fail-fast publish path must stay quiet. + env.pop("KAFKA_BOOTSTRAP_SERVERS", None) + proc = subprocess.run( + ["bash", str(_SCRIPTS / script), *args], + capture_output=True, + text=True, + timeout=60, + env=env, + check=False, + ) + return proc, calllog.read_text() + + +@pytest.mark.unit +class TestGcDryRunSafety: + def test_disk_gc_default_is_dry_run_no_destructive_calls( + self, tmp_path: Path + ) -> None: + proc, calls = _run("disk-gc.sh", [], tmp_path) + assert proc.returncode == 0, proc.stderr + for tok in DESTRUCTIVE_TOKENS: + assert tok not in calls, ( + f"dry-run issued destructive docker/git op: {tok!r}\n{calls}" + ) + + def test_watermark_dry_run_does_not_produce_to_bus(self, tmp_path: Path) -> None: + # Force a breach against a tiny mount-independent threshold so the event + # path runs, but --dry-run must NOT call `rpk ... produce`. + proc, calls = _run( + "disk-watermark-check.sh", + ["--mount", "/", "--warn", "0", "--crit", "0", "--dry-run"], + tmp_path, + ) + # Breach with crit=0 → exit 20, but dry-run means no publish. + assert proc.returncode == 20, proc.stderr + assert "produce" not in calls, f"dry-run published to bus:\n{calls}" + assert '"severity": "critical"' in proc.stdout + + def test_watermark_breach_without_broker_does_not_publish( + self, tmp_path: Path + ) -> None: + # Real (non-dry-run) breach but KAFKA_BOOTSTRAP_SERVERS unset → fail-fast: + # the script must NOT fall back to a default broker and must NOT call produce. + proc, calls = _run( + "disk-watermark-check.sh", + ["--mount", "/", "--warn", "0", "--crit", "0"], + tmp_path, + ) + assert proc.returncode == 20, proc.stderr + assert "produce" not in calls, f"published with no broker configured:\n{calls}" + + def test_watermark_under_threshold_is_quiet(self, tmp_path: Path) -> None: + proc, calls = _run( + "disk-watermark-check.sh", + ["--mount", "/", "--warn", "100", "--crit", "100"], + tmp_path, + ) + # warn=100 can only breach at exactly 100%; on a normal test host it stays quiet. + if proc.returncode == 0: + assert "produce" not in calls + + def test_worktree_gc_default_dry_run_passes_no_execute( + self, tmp_path: Path + ) -> None: + # Point at an empty worktrees root and a stub prune script that records args. + wt_root = tmp_path / "wt" + wt_root.mkdir() + prune = tmp_path / "prune-worktrees.sh" + prune.write_text( + "#!/usr/bin/env bash\n" + f'echo "prune $*" >> "{tmp_path / "prune-calls.log"}"\n' + "exit 0\n" + ) + prune.chmod(0o755) + proc, _ = _run( + "worktree-gc.sh", + ["--worktrees-root", str(wt_root), "--prune-script", str(prune)], + tmp_path, + ) + assert proc.returncode == 0, proc.stderr + prune_calls = (tmp_path / "prune-calls.log").read_text() + # Default mode must NOT pass --execute to the prune script. + assert "--execute" not in prune_calls, prune_calls + + def test_worktree_gc_execute_forwards_execute_flag(self, tmp_path: Path) -> None: + wt_root = tmp_path / "wt" + wt_root.mkdir() + prune = tmp_path / "prune-worktrees.sh" + prune.write_text( + "#!/usr/bin/env bash\n" + f'echo "prune $*" >> "{tmp_path / "prune-calls.log"}"\n' + "exit 0\n" + ) + prune.chmod(0o755) + proc, _ = _run( + "worktree-gc.sh", + [ + "--execute", + "--worktrees-root", + str(wt_root), + "--prune-script", + str(prune), + ], + tmp_path, + ) + assert proc.returncode == 0, proc.stderr + prune_calls = (tmp_path / "prune-calls.log").read_text() + assert "--execute" in prune_calls, prune_calls diff --git a/tests/unit/scripts/test_disk_gc_plan.py b/tests/unit/scripts/test_disk_gc_plan.py new file mode 100644 index 0000000000..e925d68e5f --- /dev/null +++ b/tests/unit/scripts/test_disk_gc_plan.py @@ -0,0 +1,161 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Safety tests for the .201 disk-GC removal planner (OMN-13008). + +A GC bug that deletes the wrong thing is worse than no GC. These tests pin the +removal-plan invariants of scripts/disk_gc_plan.py WITHOUT touching docker: +the planner is pure (inventory in, plan out), so we can prove it never reaps a +kept repo, a kept tag, an in-use image, or a too-young artifact. +""" + +from __future__ import annotations + +import importlib.util +from datetime import UTC, datetime, timezone +from pathlib import Path + +import pytest + +_SCRIPTS = Path(__file__).resolve().parents[3] / "scripts" +_spec = importlib.util.spec_from_file_location( + "disk_gc_plan", _SCRIPTS / "disk_gc_plan.py" +) +assert _spec and _spec.loader +disk_gc_plan = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(disk_gc_plan) + +NOW = datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC) + + +def _created(days_ago: float) -> str: + """A docker CreatedAt string `days_ago` days before NOW.""" + ts = NOW.timestamp() - days_ago * 86400.0 + dt = datetime.fromtimestamp(ts, tz=UTC) + # docker prints e.g. "2026-05-29 14:03:11 +0000 UTC" + return dt.strftime("%Y-%m-%d %H:%M:%S +0000 UTC") + + +KEEP_LIST = { + "keep_image_repos": ["omninode-runtime", "postgres"], + "keep_image_tags": ["latest", "rollback"], + "protect_running": True, + "superseded_image_keep_generations": 2, + "min_age_days": 3, +} + + +def _img(image_id: str, repo: str, tag: str, days_ago: float) -> dict[str, str]: + return { + "ID": image_id, + "Repository": repo, + "Tag": tag, + "CreatedAt": _created(days_ago), + } + + +@pytest.mark.unit +class TestDiskGcPlanSafety: + def test_dangling_old_image_is_removed(self) -> None: + images = [_img("sha-dangle", "", "", days_ago=10)] + plan = disk_gc_plan.build_plan(KEEP_LIST, images, [], set(), now=NOW) + assert "sha-dangle" in plan["remove_image_ids"] + + def test_young_dangling_image_is_kept(self) -> None: + images = [_img("sha-young", "", "", days_ago=1)] + plan = disk_gc_plan.build_plan(KEEP_LIST, images, [], set(), now=NOW) + assert plan["remove_image_ids"] == [] + + def test_keep_repo_never_removed_regardless_of_age(self) -> None: + # An old postgres image (keep repo) must never be reaped. + images = [_img("sha-pg", "postgres", "16", days_ago=400)] + plan = disk_gc_plan.build_plan(KEEP_LIST, images, [], set(), now=NOW) + assert "sha-pg" not in plan["remove_image_ids"] + + def test_keep_tag_never_removed(self) -> None: + images = [_img("sha-latest", "omninode-runtime", "latest", days_ago=400)] + plan = disk_gc_plan.build_plan(KEEP_LIST, images, [], set(), now=NOW) + assert "sha-latest" not in plan["remove_image_ids"] + + def test_in_use_image_protected_when_protect_running(self) -> None: + images = [_img("sha-live", "omninode-runtime", "v0.40.0", days_ago=30)] + inuse = {"omninode-runtime:v0.40.0"} + plan = disk_gc_plan.build_plan(KEEP_LIST, images, [], inuse, now=NOW) + assert "sha-live" not in plan["remove_image_ids"] + + def test_superseded_keeps_newest_n_generations(self) -> None: + # Five old generations of a kept repo, none in use, none keep-tagged. + # Keep newest 2, remove the 3 oldest. + images = [ + _img("g1", "omninode-runtime", "v0.41.0", days_ago=4), + _img("g2", "omninode-runtime", "v0.40.0", days_ago=10), + _img("g3", "omninode-runtime", "v0.39.0", days_ago=20), + _img("g4", "omninode-runtime", "v0.38.0", days_ago=30), + _img("g5", "omninode-runtime", "v0.37.0", days_ago=40), + ] + plan = disk_gc_plan.build_plan(KEEP_LIST, images, [], set(), now=NOW) + assert "g1" not in plan["remove_image_ids"] # newest + assert "g2" not in plan["remove_image_ids"] # 2nd newest + assert set(plan["remove_image_ids"]) == {"g3", "g4", "g5"} + + def test_untracked_third_party_repo_is_conservatively_kept(self) -> None: + # An old non-keep, non-dangling image we don't track: do NOT remove. + images = [_img("sha-other", "some/random-tool", "1.2.3", days_ago=99)] + plan = disk_gc_plan.build_plan(KEEP_LIST, images, [], set(), now=NOW) + assert plan["remove_image_ids"] == [] + + def test_stopped_old_container_removed_running_kept(self) -> None: + containers = [ + { + "ID": "c-old", + "State": "exited", + "Status": "Exited (0) 5 days ago", + "CreatedAt": _created(5), + }, + { + "ID": "c-run", + "State": "running", + "Status": "Up 5 days", + "CreatedAt": _created(5), + }, + { + "ID": "c-young", + "State": "exited", + "Status": "Exited (0) 1 hour ago", + "CreatedAt": _created(0.04), + }, + ] + plan = disk_gc_plan.build_plan(KEEP_LIST, [], containers, set(), now=NOW) + assert plan["remove_container_ids"] == ["c-old"] + + def test_empty_inventory_is_noop(self) -> None: + plan = disk_gc_plan.build_plan(KEEP_LIST, [], [], set(), now=NOW) + assert plan["remove_image_ids"] == [] + assert plan["remove_container_ids"] == [] + + def test_min_age_days_surfaced_in_plan(self) -> None: + plan = disk_gc_plan.build_plan(KEEP_LIST, [], [], set(), now=NOW) + assert plan["min_age_days"] == 3 + + def test_keep_wins_when_same_id_appears_as_dangling_and_kept(self) -> None: + # The SAME image id surfaces twice: once as an old dangling row (→ remove + # path) and once tagged 'latest' (→ keep path). Keep must win; the id must + # NOT appear in remove_image_ids. + shared = "sha-shared" + images = [ + _img(shared, "", "", days_ago=10), + _img(shared, "omninode-runtime", "latest", days_ago=10), + ] + plan = disk_gc_plan.build_plan(KEEP_LIST, images, [], set(), now=NOW) + assert shared not in plan["remove_image_ids"] + assert shared in plan["kept_reasons"] + + def test_remove_image_ids_are_deduped(self) -> None: + # Same id as two old dangling rows must appear at most once in the plan. + dup = "sha-dup" + images = [ + _img(dup, "", "", days_ago=10), + _img(dup, "", "", days_ago=12), + ] + plan = disk_gc_plan.build_plan(KEEP_LIST, images, [], set(), now=NOW) + assert plan["remove_image_ids"].count(dup) == 1 diff --git a/tests/unit/scripts/test_disk_watermark_event.py b/tests/unit/scripts/test_disk_watermark_event.py new file mode 100644 index 0000000000..a7ba980c9a --- /dev/null +++ b/tests/unit/scripts/test_disk_watermark_event.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Tests for the disk-watermark bus event builder (OMN-13008).""" + +from __future__ import annotations + +import importlib.util +from datetime import UTC, datetime, timezone +from pathlib import Path + +import pytest + +_SCRIPTS = Path(__file__).resolve().parents[3] / "scripts" +_spec = importlib.util.spec_from_file_location( + "disk_watermark_event", _SCRIPTS / "disk_watermark_event.py" +) +assert _spec and _spec.loader +mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(mod) + +NOW = datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC) +TOPIC = "onex.evt.infra.disk-watermark.v1" + + +@pytest.mark.unit +class TestDiskWatermarkEvent: + def _build(self, *, used_pct: int, severity: str) -> dict[str, object]: + return mod.build_event( + mount="/data", + used_pct=used_pct, + avail_kb=123456, + severity=severity, + warn_pct=85, + crit_pct=90, + host="server201", + topic=TOPIC, + now=NOW, + ) + + def test_warning_event_shape(self) -> None: + ev = self._build(used_pct=87, severity="warning") + assert ev["severity"] == "warning" + assert ev["used_pct"] == 87 + assert ev["topic"] == TOPIC + assert ev["event_type"] == "disk-watermark" + assert ev["schema_version"] == "1.0.0" + assert ev["emitted_at"] == NOW.isoformat() + + def test_critical_event_shape(self) -> None: + ev = self._build(used_pct=95, severity="critical") + assert ev["severity"] == "critical" + assert ev["used_pct"] == 95 + + def test_alert_key_is_stable_dedupe_key(self) -> None: + a = self._build(used_pct=87, severity="warning") + b = self._build(used_pct=88, severity="warning") + # Same host/mount/severity collapses to one open ticket. + assert ( + a["alert_key"] == b["alert_key"] == "disk-watermark:server201:/data:warning" + ) + + def test_invalid_severity_rejected(self) -> None: + with pytest.raises(ValueError): + self._build(used_pct=50, severity="info") diff --git a/tests/unit/scripts/test_lane_census_dry_run.py b/tests/unit/scripts/test_lane_census_dry_run.py new file mode 100644 index 0000000000..10ead82ff1 --- /dev/null +++ b/tests/unit/scripts/test_lane_census_dry_run.py @@ -0,0 +1,166 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Dry-run + fail-fast safety tests for lane-census-check.sh (OMN-13011). + +Properties proven here: + 1. --dry-run NEVER publishes to the bus (no `rpk produce`). + 2. On drift, the script exits 30 (fail-fast, no warn-only mode). + 3. With no broker configured, a live run does NOT publish (fail-fast, no + localhost default) — it logs for manual replay and still exits 30 on drift. + 4. A clean lane (no drift) exits 0. + +We shim `docker`, `rpk`, and `hostname` on PATH with a recorder so the test is +hermetic and asserts on the exact subcommands the script issued. +""" + +from __future__ import annotations + +import os +import stat +import subprocess +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.unit + +_REPO = Path(__file__).resolve().parents[3] +_SCRIPT = _REPO / "scripts" / "lane-census-check.sh" + + +def _shim(bin_dir: Path, name: str, body: str, calllog: Path) -> None: + shim = bin_dir / name + shim.write_text(f'#!/usr/bin/env bash\necho "{name} $*" >> "{calllog}"\n{body}\n') + shim.chmod(shim.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + + +def _run( + args: list[str], + tmp_path: Path, + *, + ps_rows: str, + networks: str, + broker: str | None, +) -> tuple[subprocess.CompletedProcess[str], str]: + bin_dir = tmp_path / "bin" + bin_dir.mkdir(exist_ok=True) + calllog = tmp_path / "calls.log" + calllog.write_text("") + + # Write the canned inventory to files the shim cats — avoids any inline + # quoting/printf interpretation of the JSON rows. + ps_file = tmp_path / "ps.ndjson" + ps_file.write_text(ps_rows) + net_file = tmp_path / "networks.txt" + net_file.write_text(networks) + + # docker shim: ps -a -> rows; network ls -> networks; everything else empty. + docker_body = ( + 'case "$*" in\n' + f' *"ps -a"*) cat "{ps_file}" ;;\n' + f' *"network ls"*) cat "{net_file}" ;;\n' + " *) : ;;\n" + "esac\n" + "exit 0" + ) + _shim(bin_dir, "docker", docker_body, calllog) + _shim(bin_dir, "rpk", "exit 0", calllog) + _shim(bin_dir, "hostname", 'echo "omninode-pc"', calllog) + + env = dict(os.environ) + env["PATH"] = f"{bin_dir}:{env['PATH']}" + env["HOME"] = str(tmp_path) # log dir under tmp, never ~ + env.pop("KAFKA_BOOTSTRAP_SERVERS", None) + if broker: + env["KAFKA_BOOTSTRAP_SERVERS"] = broker + + proc = subprocess.run( + ["bash", str(_SCRIPT), *args], + capture_output=True, + text=True, + env=env, + timeout=60, + check=False, + ) + return proc, calllog.read_text() + + +# A prod lane with the runtime containers absent + network detached (tonight's +# outage), so every live run here MUST detect drift. +_PROD_OUTAGE_PS = ( + '{"Names":"omnibase-infra-prod-postgres","State":"running",' + '"Status":"Up 2 hours","Image":"postgres:16","Labels":"com.omninode.lane=prod"}\n' +) +_NO_NETWORKS = "bridge\nhost\n" + + +def test_dry_run_does_not_publish(tmp_path: Path) -> None: + proc, calls = _run( + ["--lane", "prod", "--dry-run"], + tmp_path, + ps_rows=_PROD_OUTAGE_PS, + networks=_NO_NETWORKS, + broker="redpanda:9092", + ) + assert proc.returncode == 30, proc.stderr + assert "produce" not in calls, f"dry-run published to bus:\n{calls}" + + +def test_drift_exits_30(tmp_path: Path) -> None: + proc, _ = _run( + ["--lane", "prod"], + tmp_path, + ps_rows=_PROD_OUTAGE_PS, + networks=_NO_NETWORKS, + broker="redpanda:9092", + ) + assert proc.returncode == 30, proc.stderr + + +def test_live_without_broker_does_not_publish(tmp_path: Path) -> None: + """No KAFKA_BOOTSTRAP_SERVERS => no publish (fail-fast, no localhost default).""" + proc, calls = _run( + ["--lane", "prod"], + tmp_path, + ps_rows=_PROD_OUTAGE_PS, + networks=_NO_NETWORKS, + broker=None, + ) + assert "produce" not in calls, f"published with no broker configured:\n{calls}" + assert proc.returncode == 30 + + +def test_clean_lane_exits_zero(tmp_path: Path) -> None: + """A fully-running prod lane with its network present exits 0 (no drift).""" + # Build a healthy prod inventory from the manifest. + import yaml + + manifest = yaml.safe_load( + (_REPO / "deploy" / "lane-census" / "lane-manifest.yaml").read_text() + ) + rows = [] + for svc in manifest["lanes"]["prod"]["services"]: + name = svc["name"] + if svc.get("kind") == "oneshot": + rows.append( + f'{{"Names":"{name}","State":"exited",' + '"Status":"Exited (0) 1 hour ago",' + '"Image":"x:1","Labels":"com.omninode.lane=prod"}' + ) + else: + rows.append( + f'{{"Names":"{name}","State":"running","Status":"Up 1 hour",' + '"Image":"x:1","Labels":"com.omninode.lane=prod"}' + ) + ps = "\n".join(rows) + "\n" + networks = "omnibase-infra-prod-network\nbridge\n" + proc, calls = _run( + ["--lane", "prod"], + tmp_path, + ps_rows=ps, + networks=networks, + broker="redpanda:9092", + ) + assert proc.returncode == 0, proc.stderr + assert "produce" not in calls, "clean lane must not publish" diff --git a/tests/unit/scripts/test_lane_census_event.py b/tests/unit/scripts/test_lane_census_event.py new file mode 100644 index 0000000000..2ceb4fe0fe --- /dev/null +++ b/tests/unit/scripts/test_lane_census_event.py @@ -0,0 +1,119 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Tests for the lane-census-drift bus event builder (OMN-13011).""" + +from __future__ import annotations + +import importlib.util +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import pytest + +pytestmark = pytest.mark.unit + +_REPO = Path(__file__).resolve().parents[3] +_EVENT_PATH = _REPO / "scripts" / "lane_census_event.py" + + +def _load() -> Any: + spec = importlib.util.spec_from_file_location("lane_census_event", _EVENT_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +EV = _load() + +_FIXED_NOW = datetime(2026, 6, 11, 22, 0, 0, tzinfo=UTC) + + +def _prod_outage_plan() -> dict[str, object]: + return { + "schema_version": "1.0.0", + "lanes_checked": ["prod"], + "has_drift": True, + "findings": [ + { + "lane": "prod", + "kind": "network_detached", + "container": "omnibase-infra-prod-network", + "detail": "lane network is not present", + "severity": "critical", + }, + { + "lane": "prod", + "kind": "container_absent", + "container": "omninode-prod-runtime", + "detail": "required service not running", + "severity": "critical", + }, + ], + } + + +def test_event_is_critical_when_any_finding_critical() -> None: + event = EV.build_event(host="omninode-pc", plan=_prod_outage_plan(), now=_FIXED_NOW) + assert event["severity"] == "critical" + assert event["event_type"] == "lane-census-drift" + assert event["topic"] == "onex.evt.infra.lane-census-drift.v1" + assert event["drift_count"] == 2 + assert event["emitted_at"] == "2026-06-11T22:00:00+00:00" + + +def test_event_is_warning_when_no_critical() -> None: + plan = { + "lanes_checked": ["judge"], + "findings": [ + { + "lane": "judge", + "kind": "image_tag_mismatch", + "container": "omninode-judge-runtime", + "detail": "stale tag", + "severity": "warning", + } + ], + } + event = EV.build_event(host="h", plan=plan, now=_FIXED_NOW) + assert event["severity"] == "warning" + + +def test_alert_key_is_stable_and_dedupes() -> None: + """Same finding set => same alert_key (one ticket per outage, not per tick).""" + plan = _prod_outage_plan() + k1 = EV.build_event(host="omninode-pc", plan=plan, now=_FIXED_NOW)["alert_key"] + k2 = EV.build_event(host="omninode-pc", plan=plan, now=datetime.now(UTC))[ + "alert_key" + ] + assert k1 == k2 + assert k1.startswith("lane-census-drift:omninode-pc:") + + +def test_alert_key_changes_with_finding_set() -> None: + plan_a = _prod_outage_plan() + plan_b = _prod_outage_plan() + plan_b["findings"] = plan_b["findings"][:1] # only the network finding + ka = EV.build_event(host="h", plan=plan_a, now=_FIXED_NOW)["alert_key"] + kb = EV.build_event(host="h", plan=plan_b, now=_FIXED_NOW)["alert_key"] + assert ka != kb + + +def test_ticket_title_names_lane_and_kinds() -> None: + title = EV.build_event(host="h", plan=_prod_outage_plan(), now=_FIXED_NOW)[ + "ticket_title" + ] + assert "prod" in title + assert "network_detached" in title + assert "container_absent" in title + + +def test_ticket_body_lists_every_finding() -> None: + body = EV.build_event(host="h", plan=_prod_outage_plan(), now=_FIXED_NOW)[ + "ticket_body" + ] + assert "omnibase-infra-prod-network" in body + assert "omninode-prod-runtime" in body + assert "OMN-13011" in body diff --git a/tests/unit/scripts/test_lane_census_manifest_parity.py b/tests/unit/scripts/test_lane_census_manifest_parity.py new file mode 100644 index 0000000000..73339d6ac8 --- /dev/null +++ b/tests/unit/scripts/test_lane_census_manifest_parity.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Manifest <-> compose parity ratchet for the lane census (OMN-13011). + +The lane manifest is the desired-state authority, but it can only police a lane +if it stays in lock-step with that lane's compose file. These tests assert that +every container_name declared in a lane's compose file appears in the manifest +(and vice-versa) so the manifest can never silently drift away from the lane it +is supposed to reconcile. A lane that gains/renames a container without updating +the manifest in the same PR fails CI here — that is the whole point. + +This also closes the OMN-12988 census gap where +docker/catalog/services/runtime-worker.yaml carried container_name: null: the +manifest sources concrete names from the compose lane files, not the catalog +stub. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +import yaml + +pytestmark = pytest.mark.unit + +_REPO = Path(__file__).resolve().parents[3] +_MANIFEST_PATH = _REPO / "deploy" / "lane-census" / "lane-manifest.yaml" + +# Lanes whose compose files have concrete container_name values we can diff. +# dev uses generated/non-prefixed names and is intentionally optional/loose, so +# it is parity-checked against its own service list only, not a compose scrape. +_COMPOSE_LANES = ("stability-test", "prod", "judge") + + +def _load_manifest() -> dict: + with open(_MANIFEST_PATH, encoding="utf-8") as fh: + return yaml.safe_load(fh) + + +def _compose_container_names(lane: str) -> set[str]: + compose_path = _REPO / "docker" / f"docker-compose.{lane}.yml" + raw = compose_path.read_text(encoding="utf-8") + # The compose files use custom !override / !!merge tags that safe_load cannot + # parse, so scrape container_name lines directly. + return set(re.findall(r"^\s*container_name:\s*(\S+)\s*$", raw, re.MULTILINE)) + + +@pytest.mark.parametrize("lane", _COMPOSE_LANES) +def test_every_compose_container_is_declared(lane: str) -> None: + """No compose container_name may be missing from the lane manifest.""" + manifest = _load_manifest() + declared = {s["name"] for s in manifest["lanes"][lane]["services"]} + compose_names = _compose_container_names(lane) + missing = compose_names - declared + assert not missing, ( + f"lane {lane!r}: compose declares containers absent from the lane " + f"manifest (update deploy/lane-census/lane-manifest.yaml in the same PR): " + f"{sorted(missing)}" + ) + + +@pytest.mark.parametrize("lane", _COMPOSE_LANES) +def test_no_manifest_phantom_containers(lane: str) -> None: + """No manifest service may reference a container the compose file lacks.""" + manifest = _load_manifest() + declared = {s["name"] for s in manifest["lanes"][lane]["services"]} + compose_names = _compose_container_names(lane) + phantom = declared - compose_names + assert not phantom, ( + f"lane {lane!r}: lane manifest declares containers the compose file does " + f"not define (stale manifest entry): {sorted(phantom)}" + ) + + +@pytest.mark.parametrize("lane", _COMPOSE_LANES) +def test_lane_network_matches_compose(lane: str) -> None: + """The manifest's declared network must exist in the compose networks block.""" + manifest = _load_manifest() + declared_network = manifest["lanes"][lane]["network"] + compose_path = _REPO / "docker" / f"docker-compose.{lane}.yml" + raw = compose_path.read_text(encoding="utf-8") + network_names = set( + re.findall(r"^\s*name:\s*(omnibase-infra-\S+-network)\s*$", raw, re.MULTILINE) + ) + assert declared_network in network_names, ( + f"lane {lane!r}: manifest network {declared_network!r} not found among " + f"compose networks {sorted(network_names)}" + ) + + +def test_no_service_declares_replicas_zero() -> None: + """A service may never declare replicas: 0 — that is the silent-drop surface.""" + manifest = _load_manifest() + for lane, spec in manifest["lanes"].items(): + for svc in spec["services"]: + if svc.get("kind", "service") == "service": + assert svc.get("replicas", 1) >= 1, ( + f"lane {lane!r} service {svc['name']!r} declares replicas 0 — " + f"forbidden (the WORKER_REPLICAS silent-zero regression)" + ) + + +def test_runtime_worker_declared_in_every_runtime_lane() -> None: + """The worker that silently dropped (OMN-12988) must be a required service.""" + manifest = _load_manifest() + for lane in ("stability-test", "prod"): + names = {s["name"] for s in manifest["lanes"][lane]["services"]} + worker = next((n for n in names if n.endswith("runtime-worker")), None) + assert worker is not None, f"lane {lane!r} missing a runtime-worker service" diff --git a/tests/unit/scripts/test_lane_census_plan.py b/tests/unit/scripts/test_lane_census_plan.py new file mode 100644 index 0000000000..d8a82a98a4 --- /dev/null +++ b/tests/unit/scripts/test_lane_census_plan.py @@ -0,0 +1,287 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Pure-planner tests for the lane-census reconciler (OMN-13011). + +The planner diffs the versioned desired-state lane manifest against a live docker +inventory and emits typed drift findings. These tests pin every drift kind plus +the headline red fixture: the 2026-06-11 prod outage (runtime containers absent + +broker network detached) MUST produce drift findings that would have ticketed +hours before a human noticed. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +from typing import Any + +import pytest + +pytestmark = pytest.mark.unit + +_REPO = Path(__file__).resolve().parents[3] +_PLAN_PATH = _REPO / "scripts" / "lane_census_plan.py" +_MANIFEST_PATH = _REPO / "deploy" / "lane-census" / "lane-manifest.yaml" + + +def _load_planner() -> Any: + spec = importlib.util.spec_from_file_location("lane_census_plan", _PLAN_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +PLAN = _load_planner() +MANIFEST = PLAN.load_manifest(_MANIFEST_PATH) + + +def _container( + name: str, + *, + lane: str, + state: str = "running", + status: str = "Up 3 hours", + image: str = "omninode-runtime:0.37.0", +) -> dict[str, str]: + return { + "Names": name, + "State": state, + "Status": status, + "Image": image, + "Labels": f"com.omninode.lane={lane},com.omninode.layer=runtime", + } + + +def _healthy_prod_containers() -> list[dict[str, str]]: + """All declared prod services running, migrations exited 0.""" + services = list(MANIFEST["lanes"]["prod"]["services"]) + rows: list[dict[str, str]] = [] + for svc in services: + if svc.get("kind") == "oneshot": + rows.append( + _container( + svc["name"], + lane="prod", + state="exited", + status="Exited (0) 2 hours ago", + ) + ) + else: + rows.append(_container(svc["name"], lane="prod")) + return rows + + +def _kinds(findings: list[dict[str, str]]) -> set[str]: + return {f["kind"] for f in findings} + + +def test_healthy_prod_lane_no_drift() -> None: + """A fully-running prod lane with its network present yields zero drift.""" + envelope = { + "lane": "prod", + "containers": _healthy_prod_containers(), + "networks": ["omnibase-infra-prod-network"], + "runtime_tag": None, + } + plan = PLAN.build_plan(envelope, MANIFEST) + assert plan["has_drift"] is False, plan["findings"] + assert plan["lanes_checked"] == ["prod"] + + +def test_tonight_prod_red_fixture_runtime_absent_and_network_detached() -> None: + """RED FIXTURE — the 2026-06-11 outage. + + prod runtime containers (main/effects/worker/projection-api) absent AND the + broker network detached. The reconciler MUST emit a network_detached finding + plus a container_absent finding for every missing runtime service — exactly + the ticket that should have fired hours before a human noticed. + """ + # Only the infra/migration layer is up; the four runtime services and the + # network are gone. + surviving = [ + _container("omnibase-infra-prod-postgres", lane="prod"), + _container("omnibase-infra-prod-redpanda", lane="prod"), + _container("omnibase-infra-prod-valkey", lane="prod"), + _container( + "omnibase-infra-prod-migration-gate", + lane="prod", + state="exited", + status="Exited (0) 4 hours ago", + ), + ] + envelope = { + "lane": "prod", + "containers": surviving, + "networks": [], # broker network detached + "runtime_tag": None, + } + plan = PLAN.build_plan(envelope, MANIFEST) + + assert plan["has_drift"] is True + kinds = _kinds(plan["findings"]) + assert "network_detached" in kinds, plan["findings"] + assert "container_absent" in kinds, plan["findings"] + + absent = { + f["container"] for f in plan["findings"] if f["kind"] == "container_absent" + } + # Every runtime service must be named as absent. + for runtime_svc in ( + "omninode-prod-runtime", + "omninode-prod-runtime-effects", + "omnimarket-prod-projection-api", + "omninode-prod-runtime-worker", + ): + assert runtime_svc in absent, f"{runtime_svc} not reported absent: {absent}" + + # network_detached + container_absent are all critical severity. + crit = [f for f in plan["findings"] if f["severity"] == "critical"] + assert crit, "tonight's outage must be critical severity" + + +def test_worker_replicas_zero_silent_drop_is_drift() -> None: + """The WORKER_REPLICAS silent-zero regression (OMN-12988/12990). + + A worker scaled to 0 produces no container; the planner must report it as + drift on the declared replicas:1 service. + """ + containers = _healthy_prod_containers() + containers = [c for c in containers if c["Names"] != "omninode-prod-runtime-worker"] + envelope = { + "lane": "prod", + "containers": containers, + "networks": ["omnibase-infra-prod-network"], + "runtime_tag": None, + } + plan = PLAN.build_plan(envelope, MANIFEST) + absent = { + f["container"] for f in plan["findings"] if f["kind"] == "container_absent" + } + assert "omninode-prod-runtime-worker" in absent + + +def test_oneshot_failed_exit_nonzero_is_critical_drift() -> None: + """A migration container that Exited non-zero is critical drift.""" + containers = _healthy_prod_containers() + for c in containers: + if c["Names"] == "omnibase-infra-prod-migration-gate": + c["state"] = "exited" + c["State"] = "exited" + c["Status"] = "Exited (1) 5 minutes ago" + envelope = { + "lane": "prod", + "containers": containers, + "networks": ["omnibase-infra-prod-network"], + "runtime_tag": None, + } + plan = PLAN.build_plan(envelope, MANIFEST) + failed = [f for f in plan["findings"] if f["kind"] == "oneshot_failed"] + assert failed and failed[0]["severity"] == "critical", plan["findings"] + + +def test_oneshot_stuck_running_is_warning_drift() -> None: + """A migration container still Running (never completed) is warning drift.""" + containers = _healthy_prod_containers() + for c in containers: + if c["Names"] == "omnibase-infra-prod-forward-migration": + c["State"] = "running" + c["Status"] = "Up 6 hours" + envelope = { + "lane": "prod", + "containers": containers, + "networks": ["omnibase-infra-prod-network"], + "runtime_tag": None, + } + plan = PLAN.build_plan(envelope, MANIFEST) + stuck = [f for f in plan["findings"] if f["kind"] == "oneshot_stuck"] + assert stuck and stuck[0]["severity"] == "warning" + + +def test_unexpected_lane_labeled_container_is_drift() -> None: + """A container labeled for the lane but not declared is unexpected_container.""" + containers = _healthy_prod_containers() + containers.append(_container("omninode-prod-rogue-shadow", lane="prod")) + envelope = { + "lane": "prod", + "containers": containers, + "networks": ["omnibase-infra-prod-network"], + "runtime_tag": None, + } + plan = PLAN.build_plan(envelope, MANIFEST) + unexpected = { + f["container"] for f in plan["findings"] if f["kind"] == "unexpected_container" + } + assert "omninode-prod-rogue-shadow" in unexpected + + +def test_image_tag_mismatch_is_drift() -> None: + """A running container whose tag fails the lane pattern is drift.""" + # Pin a strict pattern lane manifest in-memory. + manifest = PLAN.load_manifest(_MANIFEST_PATH) + manifest["lanes"]["prod"]["image_tag_pattern"] = r"0\.37\..+" + containers = _healthy_prod_containers() + for c in containers: + if c["Names"] == "omninode-prod-runtime": + c["Image"] = "omninode-runtime:0.30.0-stale" + envelope = { + "lane": "prod", + "containers": containers, + "networks": ["omnibase-infra-prod-network"], + "runtime_tag": None, + } + plan = PLAN.build_plan(envelope, manifest) + mism = [f for f in plan["findings"] if f["kind"] == "image_tag_mismatch"] + assert mism and mism[0]["container"] == "omninode-prod-runtime" + + +def test_optional_dev_lane_entirely_down_is_not_drift() -> None: + """The optional dev lane being fully down must NOT ticket (developer lane).""" + envelope = { + "lane": "dev", + "containers": [], + "networks": [], + "runtime_tag": None, + } + plan = PLAN.build_plan(envelope, MANIFEST) + assert plan["has_drift"] is False, plan["findings"] + + +def test_optional_dev_lane_partially_up_is_drift() -> None: + """A partially-up optional lane IS reconciled (one service up, one missing).""" + envelope = { + "lane": "dev", + "containers": [ + _container("omninode-runtime", lane="dev"), + # omninode-runtime-effects missing + ], + "networks": ["omnibase-infra_default"], + "runtime_tag": None, + } + plan = PLAN.build_plan(envelope, MANIFEST) + absent = { + f["container"] for f in plan["findings"] if f["kind"] == "container_absent" + } + assert "omninode-runtime-effects" in absent + + +def test_unknown_lane_raises() -> None: + with pytest.raises(ValueError): + PLAN.build_plan( + {"lane": "does-not-exist", "containers": [], "networks": []}, MANIFEST + ) + + +def test_all_lanes_default_excludes_nothing_required() -> None: + """With lane=None all manifest lanes are checked.""" + envelope = {"lane": None, "containers": [], "networks": [], "runtime_tag": None} + plan = PLAN.build_plan(envelope, MANIFEST) + assert set(plan["lanes_checked"]) == set(MANIFEST["lanes"].keys()) + + +def test_tag_parsing_handles_registry_host_port() -> None: + """`_tag_of` must not mistake a registry host:port for the tag.""" + assert PLAN._tag_of("registry.local:5000/omninode-runtime:0.37.0") == "0.37.0" + assert PLAN._tag_of("omninode-runtime") == "latest" + assert PLAN._tag_of("omninode-runtime@sha256:abc") == "latest" diff --git a/tests/unit/scripts/test_lane_census_timer_dropin.py b/tests/unit/scripts/test_lane_census_timer_dropin.py new file mode 100644 index 0000000000..18f4571a2e --- /dev/null +++ b/tests/unit/scripts/test_lane_census_timer_dropin.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Systemd coordination tests for the lane-census timer drop-in (OMN-13011). + +The ticket requires sharing the OMN-13008 timer unit rather than adding a second +one. These tests assert the coordination contract: + 1. The lane-census ExecStart is delivered as a drop-in for the SHARED + onex-disk-gc.service (not a new .timer / .service). + 2. The drop-in invokes scripts/lane-census-check.sh. + 3. The installer refuses to install a second timer and depends on the base unit. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.unit + +_REPO = Path(__file__).resolve().parents[3] +_LANE_CENSUS_DIR = _REPO / "deploy" / "lane-census" +_DROPIN = _LANE_CENSUS_DIR / "onex-disk-gc.service.d" / "20-lane-census.conf" +_INSTALLER = _LANE_CENSUS_DIR / "install-lane-census.sh" + + +def test_dropin_targets_shared_disk_gc_service() -> None: + """The drop-in lives under onex-disk-gc.service.d (shared unit), not a new unit.""" + assert _DROPIN.exists(), f"missing drop-in: {_DROPIN}" + assert _DROPIN.parent.name == "onex-disk-gc.service.d" + body = _DROPIN.read_text(encoding="utf-8") + assert "[Service]" in body + assert "ExecStart=" in body + + +def test_dropin_invokes_lane_census_check() -> None: + body = _DROPIN.read_text(encoding="utf-8") + assert "scripts/lane-census-check.sh" in body + + +def test_no_second_timer_unit_shipped() -> None: + """Coordination rule: do NOT add a second timer. No .timer file in this dir.""" + timers = list(_LANE_CENSUS_DIR.glob("*.timer")) + assert not timers, ( + f"lane-census must share the onex-disk-gc.timer, not add its own: {timers}" + ) + # And no standalone lane-census .service either. + services = [ + p + for p in _LANE_CENSUS_DIR.glob("*.service") + if p.name != "onex-disk-gc.service" + ] + assert not services, f"unexpected standalone service unit(s): {services}" + + +def test_installer_depends_on_base_unit() -> None: + """Installer fails fast if the base onex-disk-gc.service is not installed.""" + body = _INSTALLER.read_text(encoding="utf-8") + assert "onex-disk-gc.service" in body + assert "install-disk-gc.sh" in body # points operator at OMN-13008's installer + assert "daemon-reload" in body diff --git a/tests/unit/scripts/test_preflight_hotpatch_ledger.py b/tests/unit/scripts/test_preflight_hotpatch_ledger.py new file mode 100644 index 0000000000..287624b4ee --- /dev/null +++ b/tests/unit/scripts/test_preflight_hotpatch_ledger.py @@ -0,0 +1,464 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Tests for scripts/preflight_hotpatch_ledger.py (OMN-13014, retro B-1).""" + +from __future__ import annotations + +import importlib.util +import os +import stat +import subprocess +from pathlib import Path +from types import ModuleType + +import pytest +import yaml + +SCRIPT_PATH = ( + Path(__file__).resolve().parents[3] / "scripts" / "preflight_hotpatch_ledger.py" +) + + +def _load_module() -> ModuleType: + spec = importlib.util.spec_from_file_location( + "preflight_hotpatch_ledger", SCRIPT_PATH + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +MODULE = _load_module() + + +def _git(repo: Path, *args: str) -> str: + result = subprocess.run( + ["git", "-C", str(repo), *args], + capture_output=True, + text=True, + check=True, + env={ + **os.environ, + "GIT_AUTHOR_NAME": "t", + "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@t", + }, + ) + return result.stdout.strip() + + +def _make_repo(root: Path, name: str) -> Path: + repo = root / name + repo.mkdir(parents=True) + _git(repo, "init", "-b", "dev") + (repo / "f.txt").write_text("one\n") + _git(repo, "add", "f.txt") + _git(repo, "commit", "-m", "c1") + return repo + + +def _commit(repo: Path, content: str) -> str: + (repo / "f.txt").write_text(content) + _git(repo, "add", "f.txt") + _git(repo, "commit", "-m", content) + return _git(repo, "rev-parse", "HEAD") + + +def _write_ledger( + path: Path, + rows: list[dict[str, object]], + schema: int = 1, +) -> Path: + path.write_text(yaml.safe_dump({"schema": schema, "rows": rows})) + return path + + +def _row( + container: str, + repo: str, + commit: str, + *, + lane: str = "stability-test", + file: str = "/app/x.py", + prepatch: str = "/app/x.py.prepatch", +) -> dict[str, object]: + return { + "container": container, + "lane": lane, + "file": file, + "prepatch_path": prepatch, + "source_repo": repo, + "source_pr": f"OmniNode-ai/{repo}#1", + "merge_commit": commit, + "merged": True, + } + + +def _fake_docker(bin_dir: Path, prepatch_lines: list[str], rc: int = 0) -> str: + """Create a stub docker executable whose `exec` prints the given paths.""" + script = bin_dir / "docker" + body = "\n".join(prepatch_lines) + script.write_text( + f'#!/bin/sh\nif [ "$1" = "exec" ]; then\n' + f'cat <<"EOF"\n{body}\nEOF\nexit {rc}\nfi\nexit 0\n' + ) + script.chmod(script.stat().st_mode | stat.S_IEXEC) + return str(script) + + +@pytest.mark.unit +class TestAncestorGate: + def test_pass_when_merge_commit_in_build_ref(self, tmp_path: Path) -> None: + repo = _make_repo(tmp_path, "omnimarket") + sha = _commit(repo, "two") + ledger = _write_ledger( + tmp_path / "ledger.yaml", [_row("c1", "omnimarket", sha)] + ) + rc = MODULE.main( + [ + "--container", + "c1", + "--clones-root", + str(tmp_path), + "--ledger", + str(ledger), + "--build-ref", + f"omnimarket={_git(repo, 'rev-parse', 'HEAD')}", + "--skip-tripwire", + ] + ) + assert rc == 0 + + def test_fail_when_merge_commit_not_in_build_ref(self, tmp_path: Path) -> None: + repo = _make_repo(tmp_path, "omnimarket") + base = _git(repo, "rev-parse", "HEAD") + unmerged = _commit(repo, "two") # build ref = base, patch commit after it + ledger = _write_ledger( + tmp_path / "ledger.yaml", [_row("c1", "omnimarket", unmerged)] + ) + rc = MODULE.main( + [ + "--container", + "c1", + "--clones-root", + str(tmp_path), + "--ledger", + str(ledger), + "--build-ref", + f"omnimarket={base}", + "--skip-tripwire", + ] + ) + assert rc == 1 + + def test_unknown_commit_is_config_error(self, tmp_path: Path) -> None: + repo = _make_repo(tmp_path, "omnimarket") + ledger = _write_ledger( + tmp_path / "ledger.yaml", + [_row("c1", "omnimarket", "0" * 40)], + ) + rc = MODULE.main( + [ + "--container", + "c1", + "--clones-root", + str(tmp_path), + "--ledger", + str(ledger), + "--build-ref", + f"omnimarket={_git(repo, 'rev-parse', 'HEAD')}", + "--skip-tripwire", + ] + ) + assert rc == 2 + + def test_build_ref_defaults_to_clone_head(self, tmp_path: Path) -> None: + repo = _make_repo(tmp_path, "omnimarket") + sha = _commit(repo, "two") + ledger = _write_ledger( + tmp_path / "ledger.yaml", [_row("c1", "omnimarket", sha)] + ) + rc = MODULE.main( + [ + "--container", + "c1", + "--clones-root", + str(tmp_path), + "--ledger", + str(ledger), + "--skip-tripwire", + ] + ) + assert rc == 0 + + def test_lane_scope_selects_rows(self, tmp_path: Path) -> None: + repo = _make_repo(tmp_path, "omnimarket") + base = _git(repo, "rev-parse", "HEAD") + unmerged = _commit(repo, "two") + ledger = _write_ledger( + tmp_path / "ledger.yaml", + [ + _row("c1", "omnimarket", unmerged, lane="stability-test"), + _row("c2", "omnimarket", base, lane="prod"), + ], + ) + # prod lane only sees the merged row -> pass + rc = MODULE.main( + [ + "--lane", + "prod", + "--clones-root", + str(tmp_path), + "--ledger", + str(ledger), + "--build-ref", + f"omnimarket={base}", + "--skip-tripwire", + ] + ) + assert rc == 0 + # stability lane sees the unmerged row -> fail + rc = MODULE.main( + [ + "--lane", + "stability-test", + "--clones-root", + str(tmp_path), + "--ledger", + str(ledger), + "--build-ref", + f"omnimarket={base}", + "--skip-tripwire", + ] + ) + assert rc == 1 + + +@pytest.mark.unit +class TestLedgerLoading: + def test_missing_ledger_is_config_error(self, tmp_path: Path) -> None: + _make_repo(tmp_path, "omnimarket") + rc = MODULE.main( + [ + "--container", + "c1", + "--clones-root", + str(tmp_path), + "--ledger", + str(tmp_path / "nope.yaml"), + "--skip-tripwire", + ] + ) + assert rc == 2 + + def test_unsupported_schema_is_config_error(self, tmp_path: Path) -> None: + _make_repo(tmp_path, "omnimarket") + ledger = _write_ledger(tmp_path / "ledger.yaml", [], schema=99) + rc = MODULE.main( + [ + "--container", + "c1", + "--clones-root", + str(tmp_path), + "--ledger", + str(ledger), + "--skip-tripwire", + ] + ) + assert rc == 2 + + def test_empty_rows_pass(self, tmp_path: Path) -> None: + _make_repo(tmp_path, "omnimarket") + ledger = _write_ledger(tmp_path / "ledger.yaml", []) + rc = MODULE.main( + [ + "--container", + "c1", + "--clones-root", + str(tmp_path), + "--ledger", + str(ledger), + "--skip-tripwire", + ] + ) + assert rc == 0 + + +@pytest.mark.unit +class TestTripwire: + def _ledger_with_merged_row( + self, tmp_path: Path, prepatch: str + ) -> tuple[Path, str]: + repo = _make_repo(tmp_path, "omnimarket") + sha = _commit(repo, "two") + ledger = _write_ledger( + tmp_path / "ledger.yaml", + [_row("c1", "omnimarket", sha, prepatch=prepatch)], + ) + return ledger, sha + + def test_ledgered_prepatch_passes_pre_rebuild(self, tmp_path: Path) -> None: + ledger, _ = self._ledger_with_merged_row(tmp_path, "/app/x.py.prepatch") + docker = _fake_docker(tmp_path, ["/app/x.py.prepatch"]) + rc = MODULE.main( + [ + "--container", + "c1", + "--clones-root", + str(tmp_path), + "--ledger", + str(ledger), + "--docker-cmd", + docker, + ] + ) + assert rc == 0 + + def test_unledgered_prepatch_fails(self, tmp_path: Path) -> None: + ledger, _ = self._ledger_with_merged_row(tmp_path, "/app/x.py.prepatch") + docker = _fake_docker( + tmp_path, ["/app/x.py.prepatch", "/app/rogue.py.prepatch"] + ) + rc = MODULE.main( + [ + "--container", + "c1", + "--clones-root", + str(tmp_path), + "--ledger", + str(ledger), + "--docker-cmd", + docker, + ] + ) + assert rc == 1 + + def test_post_rebuild_fails_on_any_prepatch(self, tmp_path: Path) -> None: + ledger, _ = self._ledger_with_merged_row(tmp_path, "/app/x.py.prepatch") + docker = _fake_docker(tmp_path, ["/app/x.py.prepatch"]) + rc = MODULE.main( + [ + "--container", + "c1", + "--clones-root", + str(tmp_path), + "--ledger", + str(ledger), + "--docker-cmd", + docker, + "--post-rebuild", + ] + ) + assert rc == 1 + + def test_post_rebuild_passes_when_clean(self, tmp_path: Path) -> None: + ledger, _ = self._ledger_with_merged_row(tmp_path, "/app/x.py.prepatch") + docker = _fake_docker(tmp_path, []) + rc = MODULE.main( + [ + "--container", + "c1", + "--clones-root", + str(tmp_path), + "--ledger", + str(ledger), + "--docker-cmd", + docker, + "--post-rebuild", + ] + ) + assert rc == 0 + + def test_exec_failure_fails_gate(self, tmp_path: Path) -> None: + ledger, _ = self._ledger_with_merged_row(tmp_path, "/app/x.py.prepatch") + docker = _fake_docker(tmp_path, [], rc=1) + rc = MODULE.main( + [ + "--container", + "c1", + "--clones-root", + str(tmp_path), + "--ledger", + str(ledger), + "--docker-cmd", + docker, + ] + ) + assert rc == 1 + + +@pytest.mark.unit +class TestBypass: + def test_valid_receipt_bypasses( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv( + "HOTPATCH_PREFLIGHT_BYPASS", "# skip-token-allowed: receipt-abc-123" + ) + rc = MODULE.main( + [ + "--container", + "c1", + "--clones-root", + str(tmp_path), + "--ledger", + str(tmp_path / "absent.yaml"), + ] + ) + assert rc == 0 + + def test_malformed_bypass_is_hard_failure( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("HOTPATCH_PREFLIGHT_BYPASS", "because I said so") + rc = MODULE.main( + [ + "--container", + "c1", + "--clones-root", + str(tmp_path), + "--ledger", + str(tmp_path / "absent.yaml"), + ] + ) + assert rc == 2 + + def test_no_bypass_var_runs_gate( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # Without bypass, missing ledger is a config error (proves gate ran). + monkeypatch.delenv("HOTPATCH_PREFLIGHT_BYPASS", raising=False) + rc = MODULE.main( + [ + "--container", + "c1", + "--clones-root", + str(tmp_path), + "--ledger", + str(tmp_path / "absent.yaml"), + ] + ) + assert rc == 2 + + +@pytest.mark.unit +class TestBuildRefParsing: + def test_malformed_build_ref_is_config_error(self, tmp_path: Path) -> None: + _make_repo(tmp_path, "omnimarket") + ledger = _write_ledger(tmp_path / "ledger.yaml", []) + rc = MODULE.main( + [ + "--container", + "c1", + "--clones-root", + str(tmp_path), + "--ledger", + str(ledger), + "--build-ref", + "missing-equals", + "--skip-tripwire", + ] + ) + assert rc == 2 diff --git a/tests/unit/scripts/test_provision_infisical.py b/tests/unit/scripts/test_provision_infisical.py index f000adeda4..c9d8292771 100644 --- a/tests/unit/scripts/test_provision_infisical.py +++ b/tests/unit/scripts/test_provision_infisical.py @@ -325,7 +325,82 @@ def fake_post(url: str, **kwargs: Any) -> MagicMock: assert len(folder_calls) > 0 +# --------------------------------------------------------------------------- +# Tests: main() — fresh-provision readiness gate (OMN-12966) +# --------------------------------------------------------------------------- + + +class _GatePassedError(Exception): + """Raised from the mocked bootstrap call to signal the fresh-provision + readiness gate was passed (i.e. did NOT short-circuit with return 1).""" + + +@pytest.mark.unit +class TestMainFreshProvisionReadinessGate: + """The fresh-provision path (no credentials in env) must accept both the + enterprise {"status": "ok"} and community {"message": "Ok"} status payloads + at its readiness gate before attempting bootstrap. + + Regression for OMN-12966: the fresh-path gate only accepted {"status": + "ok"} and rejected the community-edition payload returned by the Infisical + instance deployed on .201, returning 1 before bootstrap could run. + """ + + def _empty_env(self, tmp_path: Path) -> Path: + env_file = tmp_path / ".env" + _write_env(env_file, "INFRA_HOST=192.168.86.201\n") # no INFISICAL_* creds + return env_file + + def _run_with_status(self, tmp_path: Path, status_body: dict[str, Any]) -> int: + env_file = self._empty_env(tmp_path) + status_resp = _make_httpx_response(200, status_body) + + # First httpx.Client (readiness probe) returns the status body. + # Second httpx.Client (bootstrap flow) raises _GatePassedError so the test + # proves the gate was passed without exercising the full bootstrap. + probe_client = MagicMock() + probe_client.__enter__ = MagicMock(return_value=probe_client) + probe_client.__exit__ = MagicMock(return_value=False) + probe_client.get.return_value = status_resp + + bootstrap_client = MagicMock() + bootstrap_client.__enter__ = MagicMock(side_effect=_GatePassedError()) + + with ( + patch.object(_provision_mod, "_ENV_FILE", env_file), + patch.object( + _provision_mod, "_ADMIN_TOKEN_FILE", tmp_path / ".infisical-admin-token" + ), + patch("sys.argv", ["provision-infisical.py", f"--env-file={env_file}"]), + patch("httpx.Client", side_effect=[probe_client, bootstrap_client]), + ): + try: + return int(_main()) + except _GatePassedError: + # Reaching the bootstrap client means the readiness gate passed. + return 0 + + def test_fresh_provision_community_edition_message_ok_passes_gate( + self, tmp_path: Path + ) -> None: + rc = self._run_with_status(tmp_path, {"message": "Ok"}) + assert rc == 0 # gate passed (bootstrap reached via _GatePassedError) + + def test_fresh_provision_enterprise_status_ok_passes_gate( + self, tmp_path: Path + ) -> None: + rc = self._run_with_status(tmp_path, {"status": "ok"}) + assert rc == 0 + + def test_fresh_provision_not_ready_returns_1(self, tmp_path: Path) -> None: + # Neither status=ok nor message=Ok: gate must reject with rc=1, never + # reach bootstrap. + rc = self._run_with_status(tmp_path, {"message": "initializing"}) + assert rc == 1 + + __all__: list[str] = [ "TestCreateInfisicalFoldersIdempotency", "TestMainAlreadyProvisioned", + "TestMainFreshProvisionReadinessGate", ] diff --git a/tests/unit/scripts/test_pull_all.py b/tests/unit/scripts/test_pull_all.py index 735f5886c4..fa278104b2 100644 --- a/tests/unit/scripts/test_pull_all.py +++ b/tests/unit/scripts/test_pull_all.py @@ -244,8 +244,8 @@ def test_script_refuses_dirty_repo_before_branch_switch( ).strip() assert current_branch == "main" - def test_script_handles_missing_repo(self) -> None: - """Run pull-all.sh with a nonexistent repo name against a temp dir.""" + def test_missing_repo_warns_and_exits_zero(self) -> None: + """Absent repo emits a WARN line and exits 0 (OMN-13055).""" with tempfile.TemporaryDirectory() as tmpdir: result = subprocess.run( ["bash", str(PULL_ALL), "nonexistent_repo_xyz"], @@ -254,8 +254,55 @@ def test_script_handles_missing_repo(self) -> None: env={**os.environ, "OMNI_HOME": tmpdir}, check=False, ) - assert result.returncode != 0 - assert "MISSING" in result.stdout + assert result.returncode == 0, ( + f"Expected exit 0 for absent repo; got {result.returncode}. " + f"stdout={result.stdout!r} stderr={result.stderr!r}" + ) + assert "MISSING" in result.stdout or "not cloned" in result.stdout, ( + f"Expected warning about absent repo; stdout={result.stdout!r}" + ) + + def test_missing_repo_mixed_with_present_exits_zero(self, tmp_path: Path) -> None: + """One absent + one OK repo: exit 0, absent repo warned (OMN-13055).""" + omni_home = tmp_path / "omni_home" + omni_home.mkdir() + fake_home = tmp_path / "home" + fake_home.mkdir() + + _make_omniclaude_source(omni_home) + + result = _run_pull_all( + omni_home, fake_home, repos=["omniclaude", "nonexistent_xyz"] + ) + assert result.returncode == 0, ( + f"Expected exit 0 when present repos all OK; " + f"stdout={result.stdout!r} stderr={result.stderr!r}" + ) + assert "MISSING" in result.stdout or "not cloned" in result.stdout, ( + f"Expected warning for absent repo; stdout={result.stdout!r}" + ) + # The present repo must have been processed successfully. + assert "OK" in result.stdout, ( + f"Expected OK for omniclaude; stdout={result.stdout!r}" + ) + + def test_failed_present_repo_still_exits_nonzero(self, tmp_path: Path) -> None: + """When a present repo fails (dirty), exit 1 even if other repos are absent.""" + omni_home = tmp_path / "omni_home" + omni_home.mkdir() + fake_home = tmp_path / "home" + fake_home.mkdir() + + omniclaude = _make_omniclaude_source(omni_home) + # Make omniclaude dirty so it fails. + (omniclaude / "dirty.txt").write_text("uncommitted\n") + + result = _run_pull_all(omni_home, fake_home, repos=["omniclaude", "absent_xyz"]) + assert result.returncode != 0, ( + f"Expected non-zero exit when a present repo fails; " + f"stdout={result.stdout!r} stderr={result.stderr!r}" + ) + assert "dirty worktree" in result.stdout def test_script_handles_bare_repo_no_crash(self) -> None: """Create a bare git repo and verify pull-all.sh does not crash.""" diff --git a/tests/unit/scripts/test_resolve_workspace_pins.py b/tests/unit/scripts/test_resolve_workspace_pins.py new file mode 100644 index 0000000000..87b0962e2a --- /dev/null +++ b/tests/unit/scripts/test_resolve_workspace_pins.py @@ -0,0 +1,220 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Tests for workspace-mode sibling lock-pin resolution + preflight (OMN-12989). + +Defect under test: the 2026-06-11 stability bootstrap crash was caused by a +workspace-mode rebuild that vendored omnibase_infra 0.37.0-dev (a 13-day-stale +worktree) even though omnimarket dev's uv.lock pins infra 0.38.1. The build +ignored the lock pin and shipped a downgraded sibling, which removed the +OMN-12501 Protocol-quarantine guard and turned a latent contract defect into a +fatal crash. + +These tests encode the ratchet contract: + (a) expected sibling pins are resolved from the consuming repo's uv.lock, + (b) a fail-fast preflight aborts on version regression / SHA mismatch, + (c) the comparison block is exported for build-provenance.json. +""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.unit + +REPO_ROOT = Path(__file__).resolve().parents[3] +MODULE_PATH = REPO_ROOT / "scripts" / "runtime_build" / "resolve_workspace_pins.py" + + +def _load_module(): + spec = importlib.util.spec_from_file_location( + "resolve_workspace_pins", str(MODULE_PATH) + ) + assert spec is not None + mod = importlib.util.module_from_spec(spec) + assert spec.loader is not None + # Register before exec so dataclass KW_ONLY detection can resolve the module. + sys.modules[spec.name] = mod + spec.loader.exec_module(mod) # type: ignore[attr-defined] + return mod + + +# Minimal uv.lock fragment mirroring omnimarket dev's pin structure (git-rev source). +_LOCK_TEXT = """ +version = 1 +requires-python = ">=3.12" + +[[package]] +name = "omnibase-compat" +version = "0.5.1" +source = { git = "https://github.com/OmniNode-ai/omnibase_compat.git?rev=4d887307aae34d9d40d389ba91070cb411ce3df5#4d887307aae34d9d40d389ba91070cb411ce3df5" } + +[[package]] +name = "omnibase-infra" +version = "0.38.1" +source = { git = "https://github.com/OmniNode-ai/omnibase_infra.git?rev=e2dbdc950540df8bc59ca4370b2d4a0f5b8d6c59#e2dbdc950540df8bc59ca4370b2d4a0f5b8d6c59" } + +[[package]] +name = "onex-change-control" +version = "0.9.0" +source = { git = "https://github.com/OmniNode-ai/onex_change_control.git?rev=4877d3c223517cb0c7e1eca462ba0f4d38916314" } + +[[package]] +name = "omnimarket" +version = "0.4.3" +source = { editable = "." } +""" + + +def test_module_exists() -> None: + assert MODULE_PATH.exists(), f"resolve_workspace_pins.py not found at {MODULE_PATH}" + + +def test_parse_lock_pins_extracts_version_and_rev(tmp_path: Path) -> None: + mod = _load_module() + lock = tmp_path / "uv.lock" + lock.write_text(_LOCK_TEXT, encoding="utf-8") + + pins = mod.parse_lock_pins(lock) + + assert pins["omnibase-infra"].version == "0.38.1" + assert pins["omnibase-infra"].rev == "e2dbdc950540df8bc59ca4370b2d4a0f5b8d6c59" + assert pins["omnibase-compat"].version == "0.5.1" + assert pins["omnibase-compat"].rev == "4d887307aae34d9d40d389ba91070cb411ce3df5" + assert pins["onex-change-control"].rev == "4877d3c223517cb0c7e1eca462ba0f4d38916314" + + +def test_parse_lock_pins_missing_lock_raises(tmp_path: Path) -> None: + mod = _load_module() + with pytest.raises(FileNotFoundError): + mod.parse_lock_pins(tmp_path / "does-not-exist.lock") + + +def test_compare_pin_exact_match_is_ok() -> None: + mod = _load_module() + result = mod.compare_pin( + package="omnibase-infra", + expected_version="0.38.1", + expected_rev="e2dbdc950540df8bc59ca4370b2d4a0f5b8d6c59", + actual_version="0.38.1", + actual_rev="e2dbdc950540df8bc59ca4370b2d4a0f5b8d6c59", + ) + assert result.match is True + assert result.status == "exact" + + +def test_compare_pin_forward_advance_is_ok() -> None: + """Canonical clone may be AT OR AHEAD of the lock pin (dev advances) — allowed.""" + mod = _load_module() + result = mod.compare_pin( + package="omnibase-infra", + expected_version="0.38.1", + expected_rev="e2dbdc950540df8bc59ca4370b2d4a0f5b8d6c59", + actual_version="0.38.3", + actual_rev="20a42cbf0000000000000000000000000000aaaa", + ) + assert result.match is True + assert result.status == "ahead" + + +def test_compare_pin_regression_is_violation() -> None: + """The exact 06-11 crash: vendored 0.37.0 against a 0.38.1 lock pin.""" + mod = _load_module() + result = mod.compare_pin( + package="omnibase-infra", + expected_version="0.38.1", + expected_rev="e2dbdc950540df8bc59ca4370b2d4a0f5b8d6c59", + actual_version="0.37.0", + actual_rev="2c1d672f0000000000000000000000000000bbbb", + ) + assert result.match is False + assert result.status == "regression" + + +def test_assert_pins_satisfied_raises_on_regression() -> None: + mod = _load_module() + comparisons = [ + mod.PinComparison( + package="omnibase-infra", + expected_version="0.38.1", + expected_rev="e2dbdc95", + actual_version="0.37.0", + actual_rev="2c1d672f", + match=False, + status="regression", + ) + ] + with pytest.raises(mod.WorkspacePinError) as exc: + mod.assert_pins_satisfied(comparisons) + msg = str(exc.value) + assert "omnibase-infra" in msg + assert "0.38.1" in msg # expected + assert "0.37.0" in msg # actual + + +def test_assert_pins_satisfied_passes_when_all_match() -> None: + mod = _load_module() + comparisons = [ + mod.PinComparison( + package="omnibase-infra", + expected_version="0.38.1", + expected_rev="e2dbdc95", + actual_version="0.38.3", + actual_rev="20a42cbf", + match=True, + status="ahead", + ) + ] + # Must not raise. + mod.assert_pins_satisfied(comparisons) + + +def test_read_repo_version_from_pyproject(tmp_path: Path) -> None: + mod = _load_module() + repo = tmp_path / "omnibase_infra" + repo.mkdir() + (repo / "pyproject.toml").write_text( + '[project]\nname = "omnibase-infra"\nversion = "0.38.3"\n', + encoding="utf-8", + ) + assert mod.read_repo_version(repo) == "0.38.3" + + +def test_read_repo_version_missing_pyproject_raises(tmp_path: Path) -> None: + mod = _load_module() + with pytest.raises(FileNotFoundError): + mod.read_repo_version(tmp_path / "no-repo") + + +def test_build_comparisons_detects_staged_regression(tmp_path: Path) -> None: + """End-to-end: a staged sibling tree older than the lock pin is flagged.""" + mod = _load_module() + + # Consuming repo (omnimarket) with the dev lock pinning infra 0.38.1. + omnimarket = tmp_path / "omnimarket" + omnimarket.mkdir() + (omnimarket / "uv.lock").write_text(_LOCK_TEXT, encoding="utf-8") + (omnimarket / "pyproject.toml").write_text( + '[project]\nname = "omnimarket"\nversion = "0.4.3"\n', encoding="utf-8" + ) + + # Staged infra sibling at a STALE 0.37.0 (the crash condition). + infra = tmp_path / "omnibase_infra" + infra.mkdir() + (infra / "pyproject.toml").write_text( + '[project]\nname = "omnibase-infra"\nversion = "0.37.0"\n', encoding="utf-8" + ) + + comparisons = mod.build_comparisons( + lock_path=omnimarket / "uv.lock", + siblings={"omnibase-infra": infra}, + ) + infra_cmp = next(c for c in comparisons if c.package == "omnibase-infra") + assert infra_cmp.status == "regression" + assert infra_cmp.match is False + + with pytest.raises(mod.WorkspacePinError): + mod.assert_pins_satisfied(comparisons) diff --git a/tests/unit/scripts/validation/test_check_topic_completeness.py b/tests/unit/scripts/validation/test_check_topic_completeness.py index 26bc778be7..3b01c007d8 100644 --- a/tests/unit/scripts/validation/test_check_topic_completeness.py +++ b/tests/unit/scripts/validation/test_check_topic_completeness.py @@ -11,10 +11,13 @@ import pytest from scripts.validation.check_topic_completeness import ( + SuppressionConfigError, check_completeness, collect_registry_from_constants_file, collect_registry_from_contracts, + collect_registry_from_manifests, load_allowlist, + load_topic_suppressions, main, scan_source_for_topics, ) @@ -141,40 +144,77 @@ def test_handles_syntax_errors(self, tmp_path: Path) -> None: @pytest.mark.unit -class TestLoadAllowlist: - """Tests for allowlist loading.""" +class TestLoadTopicSuppressions: + """Tests for temporary suppression loading.""" - def test_loads_yaml_format(self, tmp_path: Path) -> None: + def test_loads_metadata_rich_suppression(self, tmp_path: Path) -> None: + suppressions = tmp_path / "suppressions.yaml" + suppressions.write_text( + textwrap.dedent("""\ + suppressions: + - topic: onex.evt.platform.test-topic.v1 + owner: "@infra" + reason: "Waiting for contract authority to land." + created_at: "2026-06-11" + expires_at: "2099-06-11" + replacement_authority: "src/omnibase_infra/nodes/example/contract.yaml" + """) + ) + topics = load_topic_suppressions(suppressions) + assert "onex.evt.platform.test-topic.v1" in topics + + def test_empty_legacy_allowlist_returns_empty(self, tmp_path: Path) -> None: + allowlist = tmp_path / "allowlist.yaml" + allowlist.write_text("allowlist: []\n") + topics = load_allowlist(allowlist) + assert topics == set() + + def test_legacy_allowlist_entries_are_rejected(self, tmp_path: Path) -> None: allowlist = tmp_path / "allowlist.yaml" allowlist.write_text( textwrap.dedent("""\ allowlist: - onex.evt.platform.test-topic.v1 - - onex.cmd.omniclaude.other-topic.v1 """) ) - topics = load_allowlist(allowlist) - assert "onex.evt.platform.test-topic.v1" in topics - assert "onex.cmd.omniclaude.other-topic.v1" in topics + with pytest.raises(SuppressionConfigError, match="legacy 'allowlist'"): + load_topic_suppressions(allowlist) - def test_loads_plain_text_format(self, tmp_path: Path) -> None: - allowlist = tmp_path / "allowlist.txt" - allowlist.write_text( + def test_missing_suppression_metadata_is_rejected(self, tmp_path: Path) -> None: + suppressions = tmp_path / "suppressions.yaml" + suppressions.write_text( textwrap.dedent("""\ - # Comment line - onex.evt.platform.test-topic.v1 - onex.cmd.omniclaude.other-topic.v1 + suppressions: + - topic: onex.evt.platform.test-topic.v1 + owner: "@infra" + reason: "Missing expiry and authority." + created_at: "2026-06-11" """) ) - topics = load_allowlist(allowlist) - assert "onex.evt.platform.test-topic.v1" in topics - assert "onex.cmd.omniclaude.other-topic.v1" in topics + with pytest.raises(SuppressionConfigError, match="missing required field"): + load_topic_suppressions(suppressions) def test_nonexistent_file_returns_empty(self, tmp_path: Path) -> None: - topics = load_allowlist(tmp_path / "nope.yaml") + topics = load_topic_suppressions(tmp_path / "nope.yaml") assert topics == set() +@pytest.mark.unit +class TestCollectRegistryFromManifests: + """Tests for topics.yaml manifest topic extraction.""" + + def test_root_topics_yaml_is_registry_authority(self, tmp_path: Path) -> None: + manifest = tmp_path / "topics.yaml" + manifest.write_text( + textwrap.dedent("""\ + topics: + - onex.evt.platform.manifest-topic.v1 + """) + ) + topics = collect_registry_from_manifests([tmp_path]) + assert "onex.evt.platform.manifest-topic.v1" in topics + + @pytest.mark.unit class TestCheckCompleteness: """Integration tests for the check_completeness function.""" @@ -201,19 +241,67 @@ def test_unregistered_topic_fails(self, tmp_path: Path) -> None: assert len(unregistered) == 1 assert unregistered[0][1] == "onex.evt.fake.unknown-topic.v1" - def test_allowlisted_topic_passes(self, tmp_path: Path) -> None: - # Source with a topic that is NOT in registry but IS in allowlist + def test_bare_allowlist_does_not_hide_unregistered_topic( + self, tmp_path: Path + ) -> None: + # Source with a topic that is NOT in registry but is in a legacy allowlist src = tmp_path / "module.py" src.write_text('MY_TOPIC = "onex.evt.fake.allowed-topic.v1"\n') allowlist = tmp_path / "allowlist.yaml" allowlist.write_text("allowlist:\n - onex.evt.fake.allowed-topic.v1\n") + with pytest.raises(SuppressionConfigError, match="legacy 'allowlist'"): + check_completeness(src_root=tmp_path, allowlist_path=allowlist) + + def test_unexpired_suppression_passes(self, tmp_path: Path) -> None: + # Source with a topic that is NOT in registry but has temporary metadata + src = tmp_path / "module.py" + src.write_text('MY_TOPIC = "onex.evt.fake.allowed-topic.v1"\n') + + suppressions = tmp_path / "suppressions.yaml" + suppressions.write_text( + textwrap.dedent("""\ + suppressions: + - topic: onex.evt.fake.allowed-topic.v1 + owner: "@infra" + reason: "Temporary fixture while contract authority lands." + created_at: "2026-06-11" + expires_at: "2099-06-11" + replacement_authority: "contracts/example/contract.yaml" + """) + ) + unregistered, _ = check_completeness( - src_root=tmp_path, allowlist_path=allowlist + src_root=tmp_path, allowlist_path=suppressions ) assert len(unregistered) == 0 + def test_expired_suppression_does_not_hide_unregistered_topic( + self, tmp_path: Path + ) -> None: + src = tmp_path / "module.py" + src.write_text('MY_TOPIC = "onex.evt.fake.expired-topic.v1"\n') + + suppressions = tmp_path / "suppressions.yaml" + suppressions.write_text( + textwrap.dedent("""\ + suppressions: + - topic: onex.evt.fake.expired-topic.v1 + owner: "@infra" + reason: "Expired fixture." + created_at: "2020-01-01" + expires_at: "2020-01-02" + replacement_authority: "contracts/example/contract.yaml" + """) + ) + + unregistered, _ = check_completeness( + src_root=tmp_path, allowlist_path=suppressions + ) + assert len(unregistered) == 1 + assert unregistered[0][1] == "onex.evt.fake.expired-topic.v1" + def test_contract_registered_topic_passes(self, tmp_path: Path) -> None: # Create contract with topic contracts = tmp_path / "contracts" @@ -240,6 +328,28 @@ def test_contract_registered_topic_passes(self, tmp_path: Path) -> None: assert len(unregistered) == 0 assert "onex.evt.platform.test-event.v1" in registry + def test_topics_yaml_registered_topic_passes(self, tmp_path: Path) -> None: + manifests = tmp_path / "manifests" + manifests.mkdir() + manifest = manifests / "topics.yaml" + manifest.write_text( + textwrap.dedent("""\ + topics: + - onex.evt.platform.manifest-event.v1 + """) + ) + + src = tmp_path / "src" + src.mkdir() + module = src / "module.py" + module.write_text('TOPIC = "onex.evt.platform.manifest-event.v1"\n') + + unregistered, registry = check_completeness( + src_root=src, manifest_roots=[manifests] + ) + assert len(unregistered) == 0 + assert "onex.evt.platform.manifest-event.v1" in registry + @pytest.mark.unit class TestMainCLI: @@ -264,6 +374,17 @@ def test_unregistered_returns_one(self, tmp_path: Path) -> None: result = main(["--src", str(tmp_path), "--quiet"]) assert result == 1 + def test_legacy_allowlist_returns_config_error(self, tmp_path: Path) -> None: + module = tmp_path / "module.py" + module.write_text('USE = "onex.evt.fake.unknown.v1"\n') + allowlist = tmp_path / "allowlist.yaml" + allowlist.write_text("allowlist:\n - onex.evt.fake.unknown.v1\n") + + result = main( + ["--src", str(tmp_path), "--allowlist", str(allowlist), "--quiet"] + ) + assert result == 2 + def test_show_registry(self, tmp_path: Path) -> None: topics_file = tmp_path / "topics.py" topics_file.write_text('T = "onex.evt.platform.test.v1"\n') diff --git a/tests/unit/services/observability/savings_estimation/test_consumer.py b/tests/unit/services/observability/savings_estimation/test_consumer.py index 4797e29c4e..f6704dc95b 100644 --- a/tests/unit/services/observability/savings_estimation/test_consumer.py +++ b/tests/unit/services/observability/savings_estimation/test_consumer.py @@ -12,17 +12,42 @@ from __future__ import annotations +import json +from datetime import UTC, datetime from unittest.mock import patch import pytest +from omnibase_infra.event_bus.event_bus_inmemory import EventBusInmemory +from omnibase_infra.event_bus.models.model_event_headers import ModelEventHeaders +from omnibase_infra.event_bus.models.model_event_message import ModelEventMessage from omnibase_infra.services.observability.savings_estimation.config import ( ConfigSavingsEstimation, ) from omnibase_infra.services.observability.savings_estimation.consumer import ( ServiceSavingsEstimator, + decode_event_message, ) + +def _build_event_message(topic: str, payload: dict[str, object]) -> ModelEventMessage: + """Build a typed ModelEventMessage exactly as the event bus delivers it. + + The Kafka and in-memory buses both encode the payload as JSON bytes in the + ``value`` field and deliver the typed model to the ``on_message`` callback. + """ + return ModelEventMessage( + topic=topic, + key=None, + value=json.dumps(payload).encode("utf-8"), + headers=ModelEventHeaders( + source="test-producer", + event_type=topic, + timestamp=datetime.now(UTC), + ), + ) + + # Base monotonic time for deterministic clock control in tests. _T0 = 1000.0 @@ -404,3 +429,124 @@ async def test_llm_only_session_produces_zero_savings_estimate( assert results[0]["session_id"] == "sess-llm-only" assert results[0]["direct_tokens_saved"] == 0 assert results[0]["direct_savings_usd"] == 0.0 + + +# --------------------------------------------------------------------------- +# OMN-13149: typed ModelEventMessage ingestion (regression) +# +# The Kafka and in-memory event buses deliver a typed ModelEventMessage to the +# consumer's on_message callback — never a raw dict or str. The original +# service_kernel wiring passed the typed message straight to ingest_event, whose +# correlation path calls payload.get("session_id"). ModelEventMessage has no +# .get(), so the live consumer raised: +# AttributeError: 'ModelEventMessage' object has no attribute 'get' +# These tests pin the typed-message decode path through the REAL consumer. +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_ingest_event_rejects_typed_message_documents_bug( + service: ServiceSavingsEstimator, +) -> None: + """Passing a typed ModelEventMessage to ingest_event reproduces the bug. + + This is the exact failure OMN-13149 fixes: the typed model has no ``.get()``, + so the dict-style correlation path raises AttributeError. decode_event_message + must be used to obtain the payload first (see tests below). + """ + message = _build_event_message( + "onex.evt.omniintelligence.llm-call-completed.v1", + { + "session_id": "sess-typed-bug", + "model_id": "claude-opus-4-6", + "prompt_tokens": 100, + "completion_tokens": 50, + }, + ) + with pytest.raises(AttributeError, match="get"): + # Why: ModelEventMessage is not a Mapping; the correlation path calls + # payload.get(). This is the pre-fix crash being pinned. + service.ingest_event(message.topic, message) # type: ignore[arg-type] + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_decode_event_message_correlates_typed_message( + service: ServiceSavingsEstimator, +) -> None: + """decode_event_message reads the typed value field; ingest_event correlates.""" + message = _build_event_message( + "onex.evt.omniintelligence.llm-call-completed.v1", + { + "session_id": "sess-typed-ok", + "model_id": "qwen3-coder-30b-a3b", + "prompt_tokens": 1000, + "completion_tokens": 500, + }, + ) + + topic, payload = decode_event_message(message) + service.ingest_event(topic, payload) + + assert service.active_session_count == 1 + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_decode_event_message_rejects_non_object_payload() -> None: + """A non-object JSON payload fails fast — no silent drop, no dict coercion.""" + message = ModelEventMessage( + topic="onex.evt.omniintelligence.llm-call-completed.v1", + key=None, + value=b"[1, 2, 3]", + headers=ModelEventHeaders( + source="test-producer", + event_type="onex.evt.omniintelligence.llm-call-completed.v1", + timestamp=datetime.now(UTC), + ), + ) + with pytest.raises(TypeError, match="must be a JSON object"): + decode_event_message(message) + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_real_bus_delivers_typed_message_to_consumer( + service: ServiceSavingsEstimator, +) -> None: + """End-to-end: the real in-memory bus delivers a typed ModelEventMessage. + + This pins the REAL consumer path the kernel wires (event bus subscribe -> + on_message(ModelEventMessage) -> decode_event_message -> ingest_event), not + handler isolation. The callback mirrors service_kernel._savings_on_message. + Before the fix this raised AttributeError inside the subscriber callback; + after the fix the session is correlated. + """ + bus = EventBusInmemory(environment="test", group="savings-test") + await bus.start() + try: + topic = "onex.evt.omniintelligence.llm-call-completed.v1" + + async def _savings_on_message(message: ModelEventMessage) -> None: + _topic, _payload = decode_event_message(message) + service.ingest_event(_topic, _payload) + + await bus.subscribe( + topic, + on_message=_savings_on_message, + group_id=f"savings-estimator.{topic}", + ) + + payload = { + "session_id": "sess-real-bus", + "model_id": "claude-opus-4-6", + "prompt_tokens": 1200, + "completion_tokens": 300, + } + await bus.publish(topic, None, json.dumps(payload).encode("utf-8")) + + assert service.active_session_count == 1 + assert service.is_finalized("sess-real-bus") is False + finally: + await bus.close() diff --git a/uv.lock b/uv.lock index 7d408ed7c7..cc9c6bcad8 100644 --- a/uv.lock +++ b/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ [manifest] overrides = [ - { name = "omnibase-core", git = "https://github.com/OmniNode-ai/omnibase_core.git?rev=c7d6b5ec010692b6724de1d49a0ee5a599d7ee06" }, + { name = "omnibase-core", git = "https://github.com/OmniNode-ai/omnibase_core.git?rev=ae8793bdad96c12a0b47e2f4d1d0f179618553b8" }, { name = "omnibase-spi", git = "https://github.com/OmniNode-ai/omnibase_spi.git?branch=main" }, ] @@ -2075,8 +2075,8 @@ wheels = [ [[package]] name = "omnibase-core" -version = "0.44.0" -source = { git = "https://github.com/OmniNode-ai/omnibase_core.git?rev=c7d6b5ec010692b6724de1d49a0ee5a599d7ee06#c7d6b5ec010692b6724de1d49a0ee5a599d7ee06" } +version = "0.44.2" +source = { git = "https://github.com/OmniNode-ai/omnibase_core.git?rev=ae8793bdad96c12a0b47e2f4d1d0f179618553b8#ae8793bdad96c12a0b47e2f4d1d0f179618553b8" } dependencies = [ { name = "blake3" }, { name = "click" }, @@ -2189,7 +2189,7 @@ requires-dist = [ { name = "jsonschema", specifier = ">=4.20.0,<5.0.0" }, { name = "mcp", specifier = ">=1.25.0,<2.0.0" }, { name = "neo4j", specifier = ">=5.15.0,<7.0.0" }, - { name = "omnibase-core", git = "https://github.com/OmniNode-ai/omnibase_core.git?rev=c7d6b5ec010692b6724de1d49a0ee5a599d7ee06" }, + { name = "omnibase-core", git = "https://github.com/OmniNode-ai/omnibase_core.git?rev=ae8793bdad96c12a0b47e2f4d1d0f179618553b8" }, { name = "omnibase-spi", git = "https://github.com/OmniNode-ai/omnibase_spi.git?branch=main" }, { name = "opentelemetry-api", specifier = ">=1.27.0,<2.0.0" }, { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.27.0,<2.0.0" }, diff --git a/workspace/sibling-pin-comparison.json b/workspace/sibling-pin-comparison.json new file mode 100644 index 0000000000..2787a7ac9f --- /dev/null +++ b/workspace/sibling-pin-comparison.json @@ -0,0 +1,7 @@ +{ + "lock_source": "placeholder", + "allow_drift": false, + "drift_count": 0, + "comparisons": [], + "_note": "Placeholder committed so the Dockerfile COPY always resolves. In workspace mode, stage_workspace.sh overwrites this with the real expected-vs-actual sibling-pin comparison (OMN-12977). Release mode ignores it." +}