diff --git a/.github/actions/validate-boundaries/action.yml b/.github/actions/validate-boundaries/action.yml new file mode 100644 index 0000000000..e8b850afd4 --- /dev/null +++ b/.github/actions/validate-boundaries/action.yml @@ -0,0 +1,272 @@ +--- +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +# Composite action: validate-boundaries (OMN-20074) +# +# Runs cross-repo boundary validation checks (Kafka parity, migration conflicts) +# in the calling repository's CI. Moved from the change-control repository's +# composite action of the same name (OMN-5770), with the same inputs, so a +# caller changes only the `uses:` line and keeps its job and context names. +# +# The checks run omnibase_core's handlers from this action's own checkout: +# omnibase_core.handlers.handler_boundary_parity +# omnibase_core.handlers.handler_migration_conflicts +# against the boundary manifest and migration inventory packaged beside them in +# src/omnibase_core/contracts/boundaries/. No validator source is checked out. +# +# Callable via: +# uses: OmniNode-ai/omnibase_core/.github/actions/validate-boundaries@ + +name: "ONEX Boundary Validation" +description: "Run cross-repo boundary checks (Kafka parity, migration conflicts) to detect seam regressions + before merge. Clones peer repos, runs selected checks, and aggregates results." +inputs: + checks: + description: > + Comma-separated list of checks to run. Available: boundary-parity, migration-conflicts + required: false + default: "boundary-parity,migration-conflicts" + warn-only: + description: > + If "true", exit 0 even when violations are found (CI warning mode). + required: false + default: "true" + repos: + description: > + Comma-separated list of peer repos to clone. Defaults to all ONEX repos with boundary seams. + required: false + default: "omniclaude,omnidash,omniintelligence,omnibase_infra,omnibase_core,omnimemory,omnimarket" + +runs: + using: "composite" + steps: + - name: Checkout calling repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: Detect boundary validation scope + id: scope + shell: bash + run: | + BOUNDARY_PARITY_REQUESTED=false + MIGRATION_CONFLICTS_REQUESTED=false + UNKNOWN_CHECK_REQUESTED=false + IFS=',' read -ra CHECKS <<< "${{ inputs.checks }}" + for check in "${CHECKS[@]}"; do + check=$(echo "$check" | xargs) + case "$check" in + boundary-parity) + BOUNDARY_PARITY_REQUESTED=true + ;; + migration-conflicts) + MIGRATION_CONFLICTS_REQUESTED=true + ;; + *) + if [[ -n "$check" ]]; then + UNKNOWN_CHECK_REQUESTED=true + fi + ;; + esac + done + + MIGRATION_CONFLICTS_SHOULD_RUN="${MIGRATION_CONFLICTS_REQUESTED}" + if [[ "${MIGRATION_CONFLICTS_REQUESTED}" == "true" && "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then + BASE_SHA="$(jq -r '.pull_request.base.sha // empty' "${GITHUB_EVENT_PATH}")" + if [[ -z "${BASE_SHA}" ]]; then + echo "::warning::Could not resolve pull request base SHA; running migration-conflicts conservatively." + elif ! git fetch --no-tags --depth=1 origin "${BASE_SHA}"; then + echo "::warning::Could not fetch pull request base SHA; running migration-conflicts conservatively." + else + git diff --name-only "${BASE_SHA}"...HEAD > changed.txt + if grep -Eq '^docker/migrations/.+\.sql$' changed.txt; then + echo "SQL migration files changed; migration-conflicts will run." + else + echo "No SQL migration files changed; migration-conflicts will be skipped." + MIGRATION_CONFLICTS_SHOULD_RUN=false + fi + fi + fi + + SHOULD_RUN_BOUNDARY_CHECKS=false + if [[ "${BOUNDARY_PARITY_REQUESTED}" == "true" || "${MIGRATION_CONFLICTS_SHOULD_RUN}" == "true" || "${UNKNOWN_CHECK_REQUESTED}" == "true" ]]; then + SHOULD_RUN_BOUNDARY_CHECKS=true + fi + + echo "migration_conflicts_should_run=${MIGRATION_CONFLICTS_SHOULD_RUN}" >> "$GITHUB_OUTPUT" + echo "should_run_boundary_checks=${SHOULD_RUN_BOUNDARY_CHECKS}" >> "$GITHUB_OUTPUT" + + - name: Set up Python + if: steps.scope.outputs.should_run_boundary_checks == 'true' + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Install uv + if: steps.scope.outputs.should_run_boundary_checks == 'true' + uses: astral-sh/setup-uv@c6081965dd0577ae48820cb8c288e12509571f0c # v7 + with: + version: "latest" + + - name: Install omnibase_core boundary handlers + if: steps.scope.outputs.should_run_boundary_checks == 'true' + id: core + shell: bash + env: + UV_HTTP_TIMEOUT: "600" + UV_SYNC_ATTEMPTS: "5" + UV_SYNC_RETRY_DELAY_SECONDS: "10" + UV_CONCURRENT_DOWNLOADS: "2" + run: | + # This action's own checkout of omnibase_core, at the ref the caller pinned. + CORE_ROOT="$(cd "${GITHUB_ACTION_PATH}/../../.." && pwd)" + echo "root=${CORE_ROOT}" >> "$GITHUB_OUTPUT" + git config --global http.version HTTP/1.1 + attempt=1 + until uv sync --project "${CORE_ROOT}" --frozen --no-dev; do + status=$? + if [ "$attempt" -ge "$UV_SYNC_ATTEMPTS" ]; then + echo "::error::uv sync failed after ${attempt} attempt(s)" + exit "$status" + fi + + sleep_seconds=$((UV_SYNC_RETRY_DELAY_SECONDS * attempt)) + echo "::warning::uv sync attempt ${attempt}/${UV_SYNC_ATTEMPTS} failed with exit ${status}; retrying in ${sleep_seconds}s" + sleep "$sleep_seconds" + attempt=$((attempt + 1)) + done + + - name: Clone peer repos + if: steps.scope.outputs.should_run_boundary_checks == 'true' + id: clone + shell: bash + run: | + rm -rf /tmp/omni_repos + mkdir -p /tmp/omni_repos + DEGRADED="" + CALLING_REPO="${GITHUB_REPOSITORY##*/}" + IFS=',' read -ra REPOS <<< "${{ inputs.repos }}" + # OMN-13701: older callers pass an explicit repo list without omnimarket, + # but the canonical intent-classified consumer now lives there. + OMNIMARKET_PRESENT=false + for repo in "${REPOS[@]}"; do + repo=$(echo "$repo" | xargs) + if [[ "$repo" == "omnimarket" ]]; then + OMNIMARKET_PRESENT=true + break + fi + done + if [[ "$OMNIMARKET_PRESENT" != "true" ]]; then + REPOS+=("omnimarket") + echo "Adding required boundary repo omnimarket" + fi + for repo in "${REPOS[@]}"; do + repo=$(echo "$repo" | xargs) # trim whitespace + if [[ "$repo" == "$CALLING_REPO" ]]; then + echo "Skipping clone of calling repo ${repo}; checked-out workspace will be symlinked." + continue + fi + if [ -n "${OMNI_REPO_CLONE_TOKEN:-}" ]; then + auth_header="AUTHORIZATION: basic $(printf 'x-access-token:%s' "${OMNI_REPO_CLONE_TOKEN}" | base64 | tr -d '\n')" + clone_args=(-c "http.https://github.com/.extraheader=${auth_header}") + else + clone_args=() + fi + cloned=false + for attempt in 1 2 3; do + rm -rf "/tmp/omni_repos/${repo}" + if git "${clone_args[@]}" clone --depth=1 "https://github.com/OmniNode-ai/${repo}.git" "/tmp/omni_repos/${repo}" 2>/dev/null; then + echo "Cloned ${repo} on attempt ${attempt}" + cloned=true + break + fi + if [ "${attempt}" -lt 3 ]; then + sleep_seconds=$((attempt * 10)) + echo "::warning::Boundary verification clone attempt ${attempt}/3 failed for ${repo}; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + done + if [[ "${cloned}" != "true" ]]; then + echo "::warning::Boundary verification DEGRADED: could not clone ${repo}. Seam parity NOT confirmed." + DEGRADED="${DEGRADED}${repo}," + fi + done + echo "degraded=${DEGRADED}" >> "$GITHUB_OUTPUT" + + - name: Symlink calling repo into workspace + if: steps.scope.outputs.should_run_boundary_checks == 'true' + shell: bash + run: | + # Extract repo name from GITHUB_REPOSITORY (e.g., OmniNode-ai/omnibase_infra -> omnibase_infra) + REPO_NAME="${GITHUB_REPOSITORY##*/}" + # Remove any cloned version (main) and replace with the PR's checkout + rm -rf "/tmp/omni_repos/${REPO_NAME}" + ln -s "$GITHUB_WORKSPACE" "/tmp/omni_repos/${REPO_NAME}" + echo "Symlinked ${REPO_NAME} -> $GITHUB_WORKSPACE" + + - name: Run boundary checks + if: steps.scope.outputs.should_run_boundary_checks == 'true' + shell: bash + env: + CORE_ROOT: ${{ steps.core.outputs.root }} + run: | + FAILURE_COUNT=0 + + # Check for degraded clone state + DEGRADED="${{ steps.clone.outputs.degraded }}" + if [[ -n "$DEGRADED" ]]; then + echo "::warning::Some peer repos could not be cloned: ${DEGRADED}" + echo "::warning::Boundary verification is DEGRADED — seam parity NOT fully confirmed." + FAILURE_COUNT=$((FAILURE_COUNT + 1)) + fi + + IFS=',' read -ra CHECKS <<< "${{ inputs.checks }}" + for check in "${CHECKS[@]}"; do + check=$(echo "$check" | xargs) + case "$check" in + boundary-parity) + echo "::group::Running boundary-parity check" + if uv run --project "${CORE_ROOT}" --frozen --no-dev --no-sync python -m omnibase_core.handlers.handler_boundary_parity --repos-root /tmp/omni_repos; then + echo "::endgroup::" + echo "boundary-parity: PASSED" + else + echo "::endgroup::" + echo "::warning::boundary-parity: FAILED — see above for details" + FAILURE_COUNT=$((FAILURE_COUNT + 1)) + fi + ;; + migration-conflicts) + if [[ "${{ steps.scope.outputs.migration_conflicts_should_run }}" != "true" ]]; then + echo "migration-conflicts: SKIPPED — no SQL migration files changed" + continue + fi + echo "::group::Running migration-conflicts check" + if uv run --project "${CORE_ROOT}" --frozen --no-dev --no-sync python -m omnibase_core.handlers.handler_migration_conflicts --repos-root /tmp/omni_repos --check-columns --warn-only; then + echo "::endgroup::" + echo "migration-conflicts: PASSED" + else + echo "::endgroup::" + echo "::warning::migration-conflicts: FAILED — see above for details" + FAILURE_COUNT=$((FAILURE_COUNT + 1)) + fi + ;; + *) + echo "::warning::Unknown check: ${check}" + ;; + esac + done + + echo "" + echo "==============================" + echo "Boundary check summary: ${FAILURE_COUNT} failure(s)" + echo "==============================" + + if [[ "${{ inputs.warn-only }}" == "true" ]]; then + echo "warn-only mode: exiting 0 regardless of failures" + exit 0 + fi + + if [[ "$FAILURE_COUNT" -gt 0 ]]; then + exit "$FAILURE_COUNT" + fi diff --git a/contracts/OMN-20074.yaml b/contracts/OMN-20074.yaml index d110ecf6f6..cfae3127f4 100644 --- a/contracts/OMN-20074.yaml +++ b/contracts/OMN-20074.yaml @@ -228,6 +228,43 @@ requirements: / occ-preflight / eligibility, repo-evidence / dod-verify is an L4 context for pull_request runs only whose absence holds and whose failure fails CI Summary, and compare-with-occ stays "true". -- falsifier: uv run pytest tests/ci/test_s6_part1_cutover_omn20074.py -q -p no:randomly' + - id: req-omn20074-boundary-parity-handler + statement: The Kafka boundary parity decisions of the onex_change_control check-boundary-parity script + must be reproduced by an omnibase_core handler over the boundary manifest carried byte for byte, + so the boundary-validation callers can leave onex_change_control without a change in verdict. + acceptance: + - id: AC24 + statement: 'AC24: HandlerBoundaryParity reproduces check-boundary-parity of onex_change_control + rev a89a6f30fa on fixture repositories: a producer file that no longer references its topic + exits 1 naming the topic and the producer file, a clean pair exits 0, a pending entry inside + its 14-day grace is skipped and one past it fails, and the packaged kafka_boundaries.yaml parses + to the data of the source manifest -- falsifier: uv run pytest tests/unit/handlers/test_handler_boundary_parity.py + -q -p no:randomly fails at the merge base' + - id: req-omn20074-migration-conflicts-handler + statement: The migration conflict and column reference decisions of the onex_change_control check-migration-conflicts + script must be reproduced by an omnibase_core handler over the migration inventory carried byte + for byte. + acceptance: + - id: AC25 + statement: 'AC25: HandlerMigrationConflicts reproduces check-migration-conflicts of onex_change_control + rev a89a6f30fa on fixture repositories: one table created with different columns in one uninventoried + boundary is a NAME_CONFLICT naming the table and both files and exits 1 (0 under --warn-only), + equal columns are an EXACT_DUPLICATE, two inventoried databases never conflict, excluded paths + are not scanned, an undeclared column in a Python SQL literal is a MISSING_COLUMN, and the packaged + migration_inventory.yaml parses to the data of the source inventory -- falsifier: uv run pytest + tests/unit/handlers/test_handler_migration_conflicts.py -q -p no:randomly fails at the merge + base' + - id: req-omn20074-validate-boundaries-action-home + statement: omnibase_core must own the validate-boundaries composite action so the four boundary-validation + callers can repoint their uses line and keep their job and context names without onex_change_control. + acceptance: + - id: AC26 + statement: 'AC26: .github/actions/validate-boundaries/action.yml in omnibase_core is a composite + action with the inputs checks, warn-only and repos at the moved defaults, runs the two core + handlers from its own checkout with the moved flags (migration-conflicts with --check-columns + --warn-only), pins every action by full sha, has one checkout (the caller) and references onex_change_control + nowhere -- falsifier: uv run pytest tests/ci/test_validate_boundaries_action_omn20074.py -q + -p no:randomly fails at the merge base' evidence_requirements: - kind: tests description: The workflow test asserts that verify has no job-level condition and that enforcing caller @@ -327,6 +364,20 @@ evidence_requirements: description: The S6 part 1 tests prove the preflight-free skip-token pin, the removed nested preflight context, and CI Summary's pull_request-only expectation of repo-evidence / dod-verify. command: uv run pytest tests/ci/test_s6_part1_cutover_omn20074.py -q -p no:randomly + - kind: tests + description: Fixture repositories against the decisions of the pinned check-boundary-parity script, + with a parity break naming topic and file, a clean pass, pending grace and expiry, and the parsed + manifest hash. + command: uv run pytest tests/unit/handlers/test_handler_boundary_parity.py -q -p no:randomly + - kind: tests + description: Fixture repositories against the decisions of the pinned check-migration-conflicts script, + with a name conflict, an exact duplicate, a clean pass, inventory separation, excluded paths, a + missing column and the inventory hash. + command: uv run pytest tests/unit/handlers/test_handler_migration_conflicts.py -q -p no:randomly + - kind: tests + description: The action tests prove the composite action's inputs and defaults, the core handler invocations, + the sha pins and the absence of any onex_change_control reference, with a positive control. + command: uv run pytest tests/ci/test_validate_boundaries_action_omn20074.py -q -p no:randomly emergency_bypass: enabled: false justification: '' @@ -559,3 +610,33 @@ dod_evidence: cwd: ${OMNI_HOME}/omnibase_core binds_ac: - AC22 + - id: dod-omn-20074-ac24 + description: The handler tests reproduce the pinned check-boundary-parity verdicts on fixture repositories + and the parsed manifest hash. + source: manual + checks: + - check_type: test_passes + check_value: uv run pytest tests/unit/handlers/test_handler_boundary_parity.py -q -p no:randomly + cwd: ${OMNI_HOME}/omnibase_core + binds_ac: + - AC24 + - id: dod-omn-20074-ac25 + description: The handler tests reproduce the pinned check-migration-conflicts verdicts on fixture + repositories and the inventory hash. + source: manual + checks: + - check_type: test_passes + check_value: uv run pytest tests/unit/handlers/test_handler_migration_conflicts.py -q -p no:randomly + cwd: ${OMNI_HOME}/omnibase_core + binds_ac: + - AC25 + - id: dod-omn-20074-ac26 + description: The action tests prove the validate-boundaries composite action's inputs, handler invocations, + sha pins and the absence of any onex_change_control reference. + source: manual + checks: + - check_type: test_passes + check_value: uv run pytest tests/ci/test_validate_boundaries_action_omn20074.py -q -p no:randomly + cwd: ${OMNI_HOME}/omnibase_core + binds_ac: + - AC26 diff --git a/src/omnibase_core/contracts/boundaries/kafka_boundaries.yaml b/src/omnibase_core/contracts/boundaries/kafka_boundaries.yaml new file mode 100644 index 0000000000..9d78fe9276 --- /dev/null +++ b/src/omnibase_core/contracts/boundaries/kafka_boundaries.yaml @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +--- +# kafka_boundaries.yaml - Cross-repo Kafka boundary manifest +# +# Maps Kafka topics that cross repository boundaries through active Kafka +# producers and consumers. Do not list projection-only dashboard topics here: +# omnidash now reads projection state and no longer owns a Kafka consumer +# manifest for the legacy event-observability topics. +# +# OMN-5640: Layer 1 - Static Kafka Boundary Parity +# +# Rules: +# - Only list topics where producer_repo != consumer_repo +# - Only list active Kafka consumers, not dashboard projection readers +# - producer_file / consumer_file are relative to the repo root +# - topic_pattern is a regex that must match somewhere in the file contents +# - event_schema is the Pydantic model name (informational, not validated yet) + +version: "1" + +boundaries: + + # ========================================================================= + # omniclaude -> omniintelligence (command topics consumed by intelligence) + # ========================================================================= + - topic_name: onex.cmd.omniintelligence.claude-hook-event.v1 + producer_repo: omniclaude + consumer_repo: omniintelligence + producer_file: "src/omniclaude/hooks/topics.py" + consumer_file: "src/omniintelligence/nodes/node_claude_hook_event_effect/contract.yaml" + topic_pattern: "claude-hook-event\\.v1" + event_schema: "ModelClaudeCodeHookEvent" + + - topic_name: onex.cmd.omniintelligence.tool-content.v1 + producer_repo: omniclaude + consumer_repo: omniintelligence + producer_file: "src/omniclaude/hooks/topics.py" + consumer_file: "src/omniintelligence/nodes/node_claude_hook_event_effect/contract.yaml" + topic_pattern: "tool-content\\.v1" + event_schema: "ModelToolContentPayload" + + - topic_name: onex.cmd.omniintelligence.session-outcome.v1 + producer_repo: omniclaude + consumer_repo: omniintelligence + producer_file: "src/omniclaude/hooks/topics.py" + consumer_file: "src/omniintelligence/nodes/node_pattern_feedback_effect/contract.yaml" + topic_pattern: "session-outcome\\.v1" + event_schema: "ModelSessionOutcomePayload" + + - topic_name: onex.cmd.omniintelligence.compliance-evaluate.v1 + producer_repo: omniclaude + consumer_repo: omniintelligence + producer_file: "src/omniclaude/hooks/topics.py" + consumer_file: "src/omniintelligence/nodes/node_compliance_evaluate_effect/contract.yaml" + topic_pattern: "compliance-evaluate\\.v1" + event_schema: "ModelComplianceEvaluatePayload" + + # ========================================================================= + # omniintelligence -> omnimarket (intent classified topic) + # Consumer migrated from omnimemory to omnimarket (OMN-13701) + # ========================================================================= + - topic_name: onex.evt.omniintelligence.intent-classified.v1 + producer_repo: omniintelligence + consumer_repo: omnimarket + producer_file: "src/omniintelligence/nodes/node_claude_hook_event_effect/contract.yaml" + consumer_file: "src/omnimarket/nodes/node_intent_event_consumer_effect/contract.yaml" + topic_pattern: "intent-classified\\.v1" + event_schema: "ModelIntentClassifiedEvent" + + # ========================================================================= + # onex_change_control -> omniclaude (contract drift detection) + # ========================================================================= + - topic_name: onex.evt.onex-change-control.contract-drift-detected.v1 + producer_repo: onex_change_control + consumer_repo: omniclaude + producer_file: "src/onex_change_control/nodes/node_contract_drift_effect/contract.yaml" + consumer_file: "src/omniclaude/hooks/topics.py" + topic_pattern: "contract-drift-detected\\.v1" + event_schema: "ModelContractDriftOutput" + status: active diff --git a/src/omnibase_core/contracts/boundaries/migration_inventory.yaml b/src/omnibase_core/contracts/boundaries/migration_inventory.yaml new file mode 100644 index 0000000000..a0dcd7c0e2 --- /dev/null +++ b/src/omnibase_core/contracts/boundaries/migration_inventory.yaml @@ -0,0 +1,635 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +--- +version: "1" +description: > + Cross-repo migration inventory. Maps every SQL migration file to its target database, runner mechanism, + and tables created/modified. Validated by check_migration_inventory.py in CI. #magic___^_^___line +databases: + omnibase_infra: + connection_env: OMNIBASE_INFRA_DB_URL + migration_sets: + - source_repo: omnibase_infra + directory: docker/migrations/forward + runner: scripts/run-forward-migrations.sh + tracking_table: public.schema_migrations + migrations: + - file: "001_registration_projection.sql" + tables: [registration_projections] + - file: "002_updated_at_audit_index.sql" + tables: [registration_projections] + - file: "003_capability_fields.sql" + tables: [registration_projections] + - file: "004_capability_fields_concurrent.sql" + tables: [registration_projections] + - file: "005_create_contracts_topics.sql" + tables: [contracts, topics] + - file: "020_create_agent_actions_table.sql" + tables: [agent_actions] + - file: "021_create_agent_routing_decisions_table.sql" + tables: [agent_routing_decisions] + - file: "022_create_agent_transformation_events_table.sql" + tables: [agent_transformation_events] + - file: "023_create_router_performance_metrics_table.sql" + tables: [router_performance_metrics] + - file: "024_create_agent_detection_failures_table.sql" + tables: [agent_detection_failures] + - file: "025_create_agent_execution_logs_table.sql" + tables: [agent_execution_logs] + - file: "026_injection_effectiveness_tables.sql" + tables: [injection_effectiveness, latency_breakdowns, pattern_hit_rates] + - file: "027_create_agent_status_events_table.sql" + tables: [agent_status_events] + - file: "028_absorb_omniclaude_columns.sql" + tables: [agent_actions, agent_detection_failures, agent_execution_logs, agent_routing_decisions, + agent_transformation_events] + - file: "029_create_db_metadata.sql" + tables: [db_metadata] + - file: "030_add_schema_fingerprint.sql" + tables: [db_metadata] + - file: "031_create_llm_call_metrics_and_cost_aggregates.sql" + tables: [llm_call_metrics, llm_cost_aggregates] + - file: "032_create_gmail_intent_evaluations.sql" + tables: [gmail_intent_evaluations] + - file: "033_create_db_error_tickets.sql" + tables: [db_error_tickets] + - file: "034_add_routing_method_latency_to_routing_decisions.sql" + tables: [agent_routing_decisions] + - file: "035_create_skill_executions.sql" + tables: [skill_executions] + - file: "036_create_schema_migrations.sql" + tables: [schema_migrations] + - file: "037_migration_sentinel.sql" + tables: [db_metadata] + - file: "038_placeholder.sql" + tables: [] + - file: "039_create_delta_bundles.sql" + tables: [delta_bundles] + - file: "040_create_delta_metrics_by_model.sql" + tables: [delta_metrics_by_model] + - file: "041_create_agent_trace_tables.sql" + tables: [change_frames, failure_signatures, fix_transitions, frame_pr_association, pr_envelopes] + - file: "042_create_keycloak_db.sql" + tables: [] + - file: "043_create_merge_gate_decisions.sql" + tables: [merge_gate_decisions] + - file: "044_create_event_ledger.sql" + tables: [event_ledger] + - file: "045_create_validation_event_ledger.sql" + tables: [validation_event_ledger] + - file: "046_create_decision_store.sql" + tables: [decision_conflicts, decision_store] + - file: "047_create_manifest_injection_lifecycle_table.sql" + tables: [manifest_injection_lifecycle] + - file: "048_create_skill_executions.sql" + tables: [skill_executions] + - file: "049_add_topic_columns_to_node_registrations.sql" + tables: [registration_projections] + - file: "050_create_baselines_tables.sql" + tables: [baselines_breakdown, baselines_comparisons, baselines_trend] + - file: "051_backfill_schema_migration_checksums.sql" + tables: [schema_migrations] + - file: "052_create_role_omniweb.sql" + tables: [] + - file: "053_create_context_audit_events.sql" + tables: [context_audit_events] + - file: "054_create_consumer_health_triage.sql" + tables: [consumer_health_triage, consumer_restart_state] + - file: "055_create_runtime_error_triage.sql" + tables: [runtime_error_triage] + - file: "056_create_consumer_health_events.sql" + tables: [consumer_health_events] + - file: "057_create_agent_learnings.sql" + tables: [agent_learnings] + - file: "058_create_agent_identities.sql" + tables: [agent_identities, agent_session_snapshots] + - file: "059_create_user_persona_snapshots.sql" + tables: [user_persona_snapshots] + - file: "060_create_routing_outcomes.sql" + tables: [capability_scores, routing_outcomes] + - file: "061_create_build_loop_cycles.sql" + tables: [build_loop_cycles] + - file: "062_context_enrichment_injection_recorded_tables.sql" + tables: [context_enrichment_events, injection_recorded_events] + - file: "063_create_session_outcomes.sql" + tables: [session_outcomes] + - file: "065_create_llm_routing_decisions.sql" + tables: [llm_routing_decisions] + - file: "066_add_quality_score_to_routing_outcomes.sql" + tables: [routing_outcomes] + - file: "067_create_baselines_table.sql" + tables: [baselines] + - file: "068_create_landing_content_projection.sql" + tables: [landing_content_projection] + - file: "069_create_remote_task_state.sql" + tables: [remote_task_state] + - file: "070_build_loop_runs.sql" + tables: [build_loop_runs] + - file: "071_add_llm_call_metrics_idempotency_unique.sql" + tables: [llm_call_metrics] + - file: "072_add_llm_call_metrics_attribution.sql" + tables: [llm_call_metrics] + - file: "073_add_llm_call_metrics_gpu_fields.sql" + tables: [llm_call_metrics, llm_cost_aggregates] + - file: "074_create_savings_estimates.sql" + tables: [savings_estimates] + - file: "075_create_waste_findings.sql" + tables: [waste_findings] + - file: "076_add_savings_estimate_provenance.sql" + tables: [savings_estimates] + - file: "077_migrate_usage_source_vocab.sql" + tables: [llm_call_metrics] + - file: "078_add_data_provenance_to_projection_tables.sql" + tables: [contracts, registration_projections, topics] + - file: "079_create_runtime_manifests.sql" + tables: [runtime_manifests] + - file: "080_create_infra_routing_decisions.sql" + tables: [infra_routing_decisions] + - file: "081_create_evidence_dashboard_projection_tables.sql" + tables: [evidence_correlation_trace_projection, evidence_dashboard_projection, evidence_readiness_aggregate_projection] + - file: "082_swarm_runs.sql" + tables: [swarm_runs] + - file: "083_create_log_entries.sql" + tables: [log_entries] + - file: "084_add_runtime_error_triage_open_suppressed_unique.sql" + tables: [runtime_error_triage] + - file: "085_add_runner_completed_at_to_db_metadata.sql" + tables: [db_metadata] + - file: "086_create_dlq_replay_history.sql" + tables: [dlq_replay_history] + - file: "087_drop_stale_delegation_events_decoy.sql" + tables: [delegation_events] + - file: "088_create_overseer_tick_ledger.sql" + tables: [overseer_tick_ledger] + - file: "089_create_verification_receipt_ledger.sql" + tables: [verification_receipt_ledger] + - file: "090_create_delegation_workflow_state.sql" + tables: [delegation_workflow_state] + - file: "091_pr_state.sql" + tables: [pr_state] + - file: "092_pr_state_add_is_draft.sql" + tables: [pr_state] + - file: "093_add_delegation_workflow_state_outbox_columns.sql" + tables: [delegation_workflow_state] + - file: "094_create_app_dashboard_role.sql" + tables: [] + - file: "095_add_attach_readiness_to_runtime_manifests.sql" + tables: [runtime_manifests] + - file: "096_grant_role_omnidash_omnidash_analytics.sql" + tables: [] + - file: "097_grant_app_dashboard_connect_omnidash_analytics.sql" + tables: [] + - file: "098_create_omninode_internal_schema.sql" + tables: [] + - file: "099_create_omninode_internal_live_events.sql" + tables: [omninode_internal.live_events] + - file: "100_create_gateway_link_health.sql" + tables: [omninode_internal.gateway_link_health, omninode_internal.gateway_link_health_status] + - file: "101_converge_savings_estimates_to_node_declared_shape.sql" + tables: [savings_estimates] + - file: "102_create_session_phase_state.sql" + tables: [session_phase_state] + - file: "103_create_tenant_projection_writer_role.sql" + tables: [] + # 104 is not a gap in this inventory: no 104_*.sql exists in + # omnibase_infra/docker/migrations/forward. The number was skipped. + - file: "105_create_ledger_chain.sql" + tables: [ledger_chain] + - file: "106_add_delegation_workflow_state_traffic_class.sql" + tables: [delegation_workflow_state] + + omniintelligence: + connection_env: OMNIINTELLIGENCE_DB_URL + migration_sets: + - source_repo: omnibase_infra + directory: docker/migrations/intelligence + runner: scripts/run-intelligence-migrations.sh + tracking_table: schema_migrations + migrations: + - file: "000_extensions.sql" + tables: [] + - file: "001_create_fsm_state_table.sql" + tables: [fsm_state] + - file: "002_create_fsm_state_history.sql" + tables: [fsm_state_history] + - file: "003_create_workflow_executions.sql" + tables: [workflow_executions] + - file: "004_create_domain_taxonomy.sql" + tables: [domain_taxonomy] + - file: "005_create_learned_patterns.sql" + tables: [learned_patterns] + - file: "006_create_pattern_disable_events.sql" + tables: [pattern_disable_events] + - file: "007_create_pattern_injections.sql" + tables: [pattern_injections] + - file: "008_create_disabled_patterns_current_view.sql" + tables: [] + - file: "009_add_signature_hash.sql" + tables: [learned_patterns] + - file: "010_create_pattern_lifecycle_audit.sql" + tables: [pattern_lifecycle_transitions] + - file: "011_add_evidence_tier_to_learned_patterns.sql" + tables: [learned_patterns] + - file: "012_create_pattern_measured_attributions.sql" + tables: [pattern_measured_attributions] + - file: "013_add_run_id_to_pattern_injections.sql" + tables: [pattern_injections] + - file: "014_create_routing_feedback_scores.sql" + tables: [routing_feedback_scores] + - file: "015_create_db_metadata.sql" + tables: [db_metadata] + - file: "016_create_review_pairing_tables.sql" + tables: [finding_fix_pairs, pattern_candidates, pattern_lifecycle, review_findings, review_fixes] + - file: "017_create_objective_evaluations.sql" + tables: [objective_evaluations] + - file: "018_migrate_routing_feedback_scores_outcome_raw.sql" + tables: [routing_feedback_scores] + - file: "019_agent_actions_and_workflow_steps.sql" + tables: [agent_actions, workflow_steps] + - file: "020_plan_reviewer_accuracy_and_runs.sql" + tables: [plan_reviewer_model_accuracy, plan_reviewer_strategy_runs] + - file: "021_create_llm_routing_decisions.sql" + tables: [llm_routing_decisions] + - file: "022_add_project_scope_to_learned_patterns.sql" + tables: [learned_patterns] + - file: "023_create_debug_intelligence_tables.sql" + tables: [ci_failure_events, debug_fix_records, debug_trigger_records, failure_streaks] + - file: "023_create_dispatch_eval_results.sql" + tables: [dispatch_eval_results] + - file: "024_create_llm_delegation_projection_tables.sql" + tables: [llm_delegation_call_log, llm_delegation_daily_projection] + - file: "025_fix_llm_delegation_call_log_date_index.sql" + tables: [llm_delegation_call_log] + - file: "026_create_code_entities.sql" + tables: [code_entities, code_relationships] + - source_repo: omniintelligence + directory: deployment/database/migrations + runner: k8s migration job + tracking_table: schema_migrations + note: "Canonical source; omnibase_infra/docker/migrations/intelligence/ is a copy" + migrations: + - file: "000_extensions.sql" + tables: [] + - file: "001_create_fsm_state_table.sql" + tables: [fsm_state] + - file: "002_create_fsm_state_history.sql" + tables: [fsm_state_history] + - file: "003_create_workflow_executions.sql" + tables: [workflow_executions] + - file: "004_create_domain_taxonomy.sql" + tables: [domain_taxonomy] + - file: "005_create_learned_patterns.sql" + tables: [learned_patterns] + - file: "006_create_pattern_disable_events.sql" + tables: [pattern_disable_events] + - file: "007_create_pattern_injections.sql" + tables: [pattern_injections] + - file: "008_create_disabled_patterns_current_view.sql" + tables: [] + - file: "009_add_signature_hash.sql" + tables: [learned_patterns] + - file: "010_create_pattern_lifecycle_audit.sql" + tables: [pattern_lifecycle_transitions] + - file: "011_add_evidence_tier_to_learned_patterns.sql" + tables: [learned_patterns] + - file: "012_create_pattern_measured_attributions.sql" + tables: [pattern_measured_attributions] + - file: "013_add_run_id_to_pattern_injections.sql" + tables: [pattern_injections] + - file: "014_create_routing_feedback_scores.sql" + tables: [routing_feedback_scores] + - file: "015_create_db_metadata.sql" + tables: [db_metadata] + - file: "016_create_review_pairing_tables.sql" + tables: [finding_fix_pairs, pattern_candidates, pattern_lifecycle, review_findings, review_fixes] + - file: "017_create_objective_evaluations.sql" + tables: [objective_evaluations] + - file: "018_migrate_routing_feedback_scores_outcome_raw.sql" + tables: [routing_feedback_scores] + - file: "019_agent_actions_and_workflow_steps.sql" + tables: [agent_actions, workflow_steps] + - file: "020_plan_reviewer_accuracy_and_runs.sql" + tables: [plan_reviewer_model_accuracy, plan_reviewer_strategy_runs] + - file: "021_create_llm_routing_decisions.sql" + tables: [llm_routing_decisions] + - file: "022_add_project_scope_to_learned_patterns.sql" + tables: [learned_patterns] + - file: "023_create_debug_intelligence_tables.sql" + tables: [ci_failure_events, debug_fix_records, debug_trigger_records, failure_streaks] + - file: "024_add_agent_name_to_objective_evaluations.sql" + tables: [objective_evaluations] + - file: "025_review_calibration_runs.sql" + tables: [calibration_event_outbox, review_calibration_model_scores, review_calibration_runs] + - file: "028_agent_actions_retention.sql" + tables: [agent_actions] + + omnidash_analytics: + connection_env: OMNIDASH_ANALYTICS_DB_URL + migration_sets: + - source_repo: omnidash-archived + directory: migrations + runner: "drizzle-kit (npm run db:migrate)" + tracking_table: drizzle internal + note: "Read-model schema frozen in omnidash-archived; active omnidash (Vite/React) has no DB migrations." + migrations: + - file: "0000_flat_silver_centurion.sql" + tables: [users] + - file: "0001_omnidash_analytics_read_model.sql" + tables: [agent_actions, agent_manifest_injections, agent_routing_decisions, agent_transformation_events, + document_access_log, document_metadata, injection_effectiveness, latency_breakdowns, learned_patterns, + node_service_registry, onex_compliance_stamps, pattern_hit_rates, pattern_learning_artifacts, + pattern_lineage_edges, pattern_lineage_nodes, pattern_quality_metrics, projection_watermarks, + task_completion_metrics, validation_runs, validation_violations] + - file: "0002_pattern_enforcement_events.sql" + tables: [pattern_enforcement_events] + - file: "0003_llm_cost_aggregates.sql" + tables: [llm_cost_aggregates] + - file: "0004_baselines_roi.sql" + tables: [baselines_breakdown, baselines_comparisons, baselines_snapshots, baselines_trend] + - file: "0005_baselines_trend_unique.sql" + tables: [] + - file: "0006_baselines_breakdown_unique.sql" + tables: [] + - file: "0007_delegation_events.sql" + tables: [delegation_events, delegation_shadow_comparisons] + - file: "0008_drop_duplicate_delegation_indexes.sql" + tables: [] + - file: "0009_wave2_pipeline_tables.sql" + tables: [debug_escalation_counts, epic_run_events, epic_run_lease, gate_decisions, pipeline_budget_state, + pr_watch_state] + - file: "0010_drop_learned_patterns.sql" + tables: [] + - file: "0011_llm_routing_decisions_correlation_id_uuid.sql" + tables: [llm_routing_decisions] + - file: "0012_routing_config.sql" + tables: [routing_config] + - file: "0013_routing_decisions_tokens.sql" + tables: [llm_routing_decisions] + - file: "0014_pattern_pipeline_tables.sql" + tables: [pattern_injections, pattern_lifecycle_transitions, pattern_measured_attributions] + - file: "0015_plan_review_runs.sql" + tables: [plan_review_runs] + - file: "0016_rename_duplicate_prefix_migrations.sql" + tables: [] + - file: "0017_model_efficiency_rollups.sql" + tables: [model_efficiency_rollups] + - file: "0018_agent_routing_decisions_nullable_required_fields.sql" + tables: [agent_routing_decisions] + - file: "0019_agent_routing_decisions_session_id_text.sql" + tables: [agent_routing_decisions] + - file: "0020_correlation_trace_spans.sql" + tables: [correlation_trace_spans] + - file: "0021_session_outcomes.sql" + tables: [session_outcomes] + - file: "0022_phase_metrics_events.sql" + tables: [phase_metrics_events] + - file: "0023_dod_verification_tables.sql" + tables: [dod_guard_events, dod_verify_runs] + - file: "0024_ci_debug_events.sql" + tables: [ci_debug_escalation_events] + - file: "0025_circuit_breaker_events.sql" + tables: [circuit_breaker_events] + - file: "0026_compliance_evaluations.sql" + tables: [compliance_evaluations] + - file: "0027_dlq_messages.sql" + tables: [dlq_messages] + - file: "0028_intent_drift_events.sql" + tables: [intent_drift_events] + - file: "0029_llm_health_snapshots.sql" + tables: [llm_health_snapshots] + - file: "0030_omnimemory_tables.sql" + tables: [memory_documents, memory_retrievals] + - file: "0031_routing_feedback_events.sql" + tables: [routing_feedback_events] + - file: "0032_skill_invocations.sql" + tables: [skill_invocations] + - file: "0033_rename_duplicate_prefix_migrations_v2.sql" + tables: [] + - file: "0034_add_routing_shadow_decisions.sql" + tables: [routing_shadow_decisions] + - file: "0035_consumer_health_events.sql" + tables: [consumer_health_events] + - file: "0036_runtime_error_events.sql" + tables: [runtime_error_events] + - file: "0037_intent_signals.sql" + tables: [intent_signals] + - file: "0038_pattern_learning_unique_pattern_id.sql" + tables: [pattern_learning_artifacts] + - file: "0039_runtime_error_triage_state.sql" + tables: [runtime_error_triage_state] + - file: "0040_add_evidence_tier_to_patlearn.sql" + tables: [pattern_learning_artifacts] + - file: "0041_hostile_reviewer_runs.sql" + tables: [hostile_reviewer_runs] + - file: "0042_purge_seed_pattern_enforcement.sql" + tables: [] + - file: "0043_purge_noise_patterns.sql" + tables: [] + - file: "0044_agent_status_events.sql" + tables: [agent_status_events] + - file: "0045_subsystem_health_results.sql" + tables: [subsystem_health_results] + - file: "0046_purge_bloat_and_stale_data.sql" + tables: [] + - file: "0047_team_events.sql" + tables: [team_events] + - file: "0048_contract_drift_events.sql" + tables: [contract_drift_events] + - file: "0049_eval_reports.sql" + tables: [eval_reports] + - file: "0050_hook_health_events.sql" + tables: [hook_health_events] + - file: "0051_golden_chain_sweep_results.sql" + tables: [golden_chain_sweep_results] + - file: "0052_infra_routing_decisions.sql" + tables: [infra_routing_decisions] + - file: "0053_build_loop_orchestrator_events.sql" + tables: [build_loop_orchestrator_events] + - file: "0054_infra_routing_tier.sql" + tables: [infra_routing_decisions] + - file: "0055_bloom_eval_results.sql" + tables: [intelligence_bloom_eval_results] + - file: "0056_session_post_mortems.sql" + tables: [session_post_mortems] + - file: "0057_latency_breakdowns_prompt_id_nullable.sql" + tables: [latency_breakdowns] + - file: "0058_session_outcomes_correlation_id.sql" + tables: [session_outcomes] + - file: "0059_rename_duplicate_prefix_migrations_v3.sql" + tables: [] + - file: "0060_purge_sentinel_pattern_enforcement.sql" + tables: [] + - file: "0061_purge_orphaned_migration_records.sql" + tables: [] + - file: "0062_purge_noise_patterns.sql" + tables: [] + - file: "0063_llm_routing_decisions.sql" + tables: [llm_routing_decisions] + - file: "0064_llm_routing_decisions_nullable_fuzzy_agent.sql" + tables: [llm_routing_decisions] + - file: "0065_objective_evaluation_tables.sql" + tables: [objective_anti_gaming_alerts, objective_evaluations, objective_gate_failures, policy_state] + - file: "0066_create_rl_episodes.sql" + tables: [rl_episodes] + - file: "0067_savings_estimates.sql" + tables: [savings_estimates] + - file: "0068_skill_invocations_status_dedup.sql" + tables: [skill_invocations] + - file: "0069_widen_pattern_name.sql" + tables: [pattern_learning_artifacts] + - file: "0070_review_calibration_runs.sql" + tables: [review_calibration_runs_rm] + - file: "0071_github_webhook_deliveries.sql" + tables: [github_webhook_deliveries] + - file: "0072_sweep_results.sql" + tables: [sweep_results] + - file: "0073_validation_lifecycle_candidates.sql" + tables: [validation_candidates] + - file: "0074_purge_a_suffix_orphan_migration_records.sql" + tables: [] + - file: "0075_context_enrichment_events.sql" + tables: [context_enrichment_events] + + omnimemory: + connection_env: OMNIMEMORY_DB_URL + migration_sets: + - source_repo: omnimemory + directory: deployment/database/migrations + runner: k8s migration job + migrations: + - file: "001_create_subscription_tables.sql" + tables: [subscriptions] + - file: "002_create_memories_table.sql" + tables: [memories] + - file: "003_create_crawl_state_table.sql" + tables: [omnimemory_crawl_state] + - file: "004_create_navigation_sessions_table.sql" + tables: [navigation_sessions] + - file: "005_backfill_null_checksums.sql" + tables: [schema_migrations] + + omninode_cloud: + connection_env: OMNINODE_CLOUD_DB_URL + migration_sets: + - source_repo: omninode_infra + directory: db/migrations + runner: "k8s migration job (Dockerfile.migrate)" + tracking_table: schema_migrations + migrations: + - file: "00000000_migrations_tracking.sql" + tables: [schema_migrations] + - file: "021_workflow_results.sql" + tables: [schema_migrations, workflow_results] + - file: "20251207_tenants_uuid_pk.sql" + tables: [migrations_log, tenants] + - file: "20251209_m4_usage_schema.sql" + tables: [migrations_log, plan_tiers, tenant_plans, usage_events] + - file: "20251209b_m4_usage_privileges.sql" + tables: [migrations_log, usage_privileges] + - file: "20251210_m4_usage_privileges_seed.sql" + tables: [migrations_log] + - file: "20251211_m4_api_keys.sql" + tables: [migrations_log, tenant_api_keys] + - file: "20251212_m4_plan_metrics.sql" + tables: [migrations_log, plan_entitlements] + - file: "20251215_m4_stripe_billing.sql" + tables: [migrations_log, plan_tiers, stripe_webhook_events, tenant_billing] + - file: "20251219_m4_api_keys_key_hash_unique.sql" + tables: [migrations_log, tenant_api_keys] + - file: "20251219_m4_performance_indexes.sql" + tables: [migrations_log] + - file: "20251222_m4_usage_events_idempotency.sql" + tables: [migrations_log, usage_events] + - file: "20260130_create_app_users.sql" + tables: [migrations_log] + - file: "20260226_s2_alpha_provisioning_schema.sql" + tables: [migrations_log, tenants] + - file: "20260226_s2_bootstrap_tokens.sql" + tables: [bootstrap_tokens, migrations_log] + - file: "20260226_s4_tenant_hardening.sql" + tables: [migrations_log, tenants] + - file: "20260228_owner_email_tenants.sql" + tables: [migrations_log, tenants] + - file: "20260427_backfill_tenant_billing_plan_code.sql" + tables: [migrations_log, tenant_billing] + - file: "20260427_bootstrap_tokens.sql" + tables: [bootstrap_tokens, migrations_log] + - file: "20260427_provisioning_state_check_extend.sql" + tables: [migrations_log, tenants] + - file: "20260427_tenant_billing_plan_columns.sql" + tables: [migrations_log, tenant_billing] + - file: "20260427_tenant_plans_lifecycle_columns.sql" + tables: [migrations_log, tenant_plans] + - file: "20260429_plan_entitlements.sql" + tables: [migrations_log, plan_entitlements] + - file: "20260512_migrations_log_direction_cleanup.sql" + tables: [migrations_log] + - file: "20260703_tenant_credentials.sql" + tables: [migrations_log, tenant_credentials, tenants] + - file: "20260721_gateway_workflows.sql" + tables: [gateway_workflows, gateway_rate_limit_events, migrations_log] + - file: "20260722_tenants_stripe_customer_id.sql" + tables: [tenants, migrations_log] + - file: "20260725_backfill_tenants_provisioning_state.sql" + tables: [tenants, migrations_log] + - file: "20260825_backfill_gateway_workflows_completed_at.sql" + tables: [gateway_workflows, migrations_log] + - file: "20260725_gateway_workflows_terminal.sql" + tables: [gateway_workflows, migrations_log] + - file: "20260725_tenant_suspended_plan.sql" + tables: [plan_tiers, plan_entitlements, migrations_log] + - file: "20260726_compute_minutes_metering.sql" + tables: [gateway_workflows, plan_entitlements, usage_events, migrations_log] + - file: "20260727_storage_bytes_metering.sql" + tables: [usage_events, migrations_log] + - file: "20260812_tenants_acl_state_column.sql" + tables: [migrations_log, tenants] + - file: "20260812_tenants_created_at_column.sql" + tables: [migrations_log, tenants] + - file: "20260813_gateway_sessions.sql" + tables: [gateway_session_events, gateway_sessions, migrations_log] + - file: "20260813_tenants_quota_state_column.sql" + tables: [migrations_log, tenants] + - file: "20260814_tenants_acl_state_three_state_reconciliation.sql" + tables: [migrations_log, tenants] + - file: "20260815_gateway_sessions_terminal_states.sql" + tables: [gateway_session_events, gateway_sessions, migrations_log] + - file: "20260816_tenant_event_outbox.sql" + tables: [tenant_event_outbox] + - file: "20260825_gateway_workflows_result_content.sql" + tables: [gateway_workflows] + - file: "20260906_gateway_workflows_failure_attribution.sql" + tables: [gateway_workflows, migrations_log] + - file: "20260906_tenant_api_keys_unique_active_name.sql" + tables: [migrations_log, tenant_api_keys] + - file: "20260910_gateway_workflows_terminal_provenance.sql" + tables: [gateway_workflows, migrations_log] + - file: "20260911_gateway_workflows_terminal_credential_source.sql" + tables: [gateway_workflows, migrations_log] + - file: "20260913_gateway_workflows_terminal_rule_evaluations.sql" + tables: [gateway_workflows, migrations_log] + + omniclaude: + connection_env: OMNICLAUDE_DB_URL + migration_sets: + - source_repo: omniclaude + directory: sql/migrations + runner: k8s migration job + migrations: + - file: "001_create_claude_session_tables.sql" + tables: [claude_session_event_idempotency, claude_session_prompts, claude_session_snapshots, + claude_session_tools] + - file: "001_create_claude_session_tables_down.sql" + tables: [] + - file: "002_create_hook_events_table.sql" + tables: [hook_events] + + omniweb: + connection_env: null + migration_sets: + - source_repo: omniweb + directory: scripts/migrations + runner: k8s ConfigMap + migrations: + - file: "001_waitlist_signups.sql" + tables: [waitlist_signups] diff --git a/src/omnibase_core/handlers/handler_boundary_parity.py b/src/omnibase_core/handlers/handler_boundary_parity.py new file mode 100644 index 0000000000..a309c7fa86 --- /dev/null +++ b/src/omnibase_core/handlers/handler_boundary_parity.py @@ -0,0 +1,361 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Cross-repo Kafka boundary parity from onex_change_control (OMN-20074). + +The handler is pure over declared boundaries, source snapshots and an explicit +clock instant. The CLI reads sources through NodeSourceFileGatherEffect. +The source's --check-schemas is not ported: the composite action never passed it. +""" + +from __future__ import annotations + +import argparse +import importlib.resources +import json +import re +import sys +from collections.abc import Mapping, Sequence +from datetime import UTC, datetime +from pathlib import Path +from typing import Final, cast + +from omnibase_core.enums.enum_core_error_code import EnumCoreErrorCode +from omnibase_core.errors.model_onex_error import ModelOnexError +from omnibase_core.models.nodes.boundary_parity.model_boundary_parity_input import ( + ModelBoundaryParityInput, +) +from omnibase_core.models.nodes.boundary_parity.model_boundary_parity_report import ( + ModelBoundaryParityReport, +) +from omnibase_core.models.nodes.boundary_parity.model_boundary_parity_result import ( + ModelBoundaryParityResult, +) +from omnibase_core.models.nodes.boundary_parity.model_kafka_boundary_entry import ( + ModelKafkaBoundaryEntry, +) +from omnibase_core.models.nodes.no_utcnow_check.model_source_file import ModelSourceFile +from omnibase_core.models.nodes.source_file_gather.model_source_file_gather_input import ( + ModelSourceFileGatherInput, +) +from omnibase_core.nodes.node_source_file_gather_effect.handler import ( + NodeSourceFileGatherEffect, +) +from omnibase_core.utils.util_safe_yaml_loader import load_yaml_mapping_no_duplicates + +PENDING_GRACE_PERIOD_DAYS: Final[int] = 14 +_TOPIC_MIN_SEGMENTS: Final[int] = 4 +MANIFEST_RESOURCE: Final[str] = "kafka_boundaries.yaml" + + +def load_manifest_yaml() -> str: + """Read the byte-for-byte OCC manifest packaged with the contracts.""" + return ( + importlib.resources.files("omnibase_core.contracts") + / "boundaries" + / MANIFEST_RESOURCE + ).read_text(encoding="utf-8") + + +def load_boundary_manifest(manifest_yaml: str) -> list[ModelKafkaBoundaryEntry]: + """Parse the manifest with the source's defaults and no duplicate keys.""" + data = load_yaml_mapping_no_duplicates(manifest_yaml, source=MANIFEST_RESOURCE) + if "boundaries" not in data: + raise ModelOnexError( + message="Invalid manifest: missing 'boundaries' key", + error_code=EnumCoreErrorCode.CONTRACT_VALIDATION_ERROR, + ) + items = cast("list[dict[str, object]]", data["boundaries"]) + entries: list[ModelKafkaBoundaryEntry] = [] + for item in items: + entries.append( + ModelKafkaBoundaryEntry.model_validate( + { + "topic_name": item["topic_name"], + "producer_repo": item["producer_repo"], + "consumer_repo": item["consumer_repo"], + "producer_file": item["producer_file"], + "consumer_file": item["consumer_file"], + "topic_pattern": item["topic_pattern"], + "event_schema": item.get("event_schema", ""), + "status": item.get("status", "active"), + "pending_since": item.get("pending_since", ""), + "pending_reason": item.get("pending_reason", ""), + } + ) + ) + return entries + + +def check_file_for_topic( + source: str | None, + topic_pattern: str, + topic_name: str, +) -> tuple[bool, bool]: + """Check if a file exists and contains a reference to the topic. + + Returns: + (file_exists, topic_found) + """ + if source is None: + return False, False + + content = source + + # First try the regex pattern from the manifest + if re.search(topic_pattern, content): + return True, True + + # Fall back to literal topic name search + if topic_name in content: + return True, True + + # Try just the event-name segment (e.g. "agent-actions" from + # "onex.evt.omniclaude.agent-actions.v1") + parts = topic_name.split(".") + if len(parts) >= _TOPIC_MIN_SEGMENTS: + event_name = parts[3] + if event_name in content: + return True, True + + return True, False + + +def check_boundary( + entry: ModelKafkaBoundaryEntry, + files: Mapping[str, str], +) -> ModelBoundaryParityResult: + """Check a single boundary for parity.""" + producer_path = f"{entry.producer_repo}/{entry.producer_file}" + consumer_path = f"{entry.consumer_repo}/{entry.consumer_file}" + + producer_exists, producer_found = check_file_for_topic( + files.get(producer_path), entry.topic_pattern, entry.topic_name + ) + consumer_exists, consumer_found = check_file_for_topic( + files.get(consumer_path), entry.topic_pattern, entry.topic_name + ) + + error_parts: list[str] = [] + if not producer_exists: + error_parts.append( + f"producer file missing: {entry.producer_repo}/{entry.producer_file}" + ) + elif not producer_found: + error_parts.append( + f"topic not found in producer: {entry.producer_repo}/{entry.producer_file}" + ) + if not consumer_exists: + error_parts.append( + f"consumer file missing: {entry.consumer_repo}/{entry.consumer_file}" + ) + elif not consumer_found: + error_parts.append( + f"topic not found in consumer: {entry.consumer_repo}/{entry.consumer_file}" + ) + + return ModelBoundaryParityResult( + boundary=entry, + producer_ok=producer_found, + consumer_ok=consumer_found, + producer_file_exists=producer_exists, + consumer_file_exists=consumer_exists, + error="; ".join(error_parts), + ) + + +class HandlerBoundaryParity: + """Decide parity without accessing the filesystem or clock.""" + + def handle(self, request: ModelBoundaryParityInput) -> ModelBoundaryParityReport: + """Return checked boundaries and pending entries still within grace.""" + files = {file.path: file.source for file in request.files} + results: list[ModelBoundaryParityResult] = [] + pending_in_grace: list[ModelKafkaBoundaryEntry] = [] + for entry in request.boundaries: + if entry.status == "pending": + pending_date = datetime.strptime( + entry.pending_since, "%Y-%m-%d" + ).replace(tzinfo=UTC) + elapsed_days = (request.now - pending_date).days + if elapsed_days <= PENDING_GRACE_PERIOD_DAYS: + pending_in_grace.append(entry) + continue + results.append( + ModelBoundaryParityResult( + boundary=entry, + producer_ok=False, + consumer_ok=False, + producer_file_exists=True, + consumer_file_exists=False, + error=( + f"EXPIRED PENDING: {entry.topic_name} has been pending for " + f"{elapsed_days} days (since {entry.pending_since}, " + f"grace period is {PENDING_GRACE_PERIOD_DAYS} days). " + f"Reason: {entry.pending_reason or 'none'}" + ), + ) + ) + continue + results.append(check_boundary(entry, files)) + return ModelBoundaryParityReport( + results=results, pending_in_grace=pending_in_grace + ) + + +def format_report(report: ModelBoundaryParityReport) -> str: + """Format the parity report for human consumption.""" + lines: list[str] = [] + lines.append("=" * 72) + lines.append("Kafka Boundary Parity Report") + lines.append("=" * 72) + lines.append("") + + ok_count = sum(1 for r in report.results if r.producer_ok and r.consumer_ok) + fail_count = report.mismatch_count + total = len(report.results) + + lines.append(f"Total boundaries: {total}") + lines.append(f" OK: {ok_count}") + lines.append(f" MISMATCH: {fail_count}") + lines.append("") + + if fail_count > 0: + lines.append("-" * 72) + lines.append("MISMATCHES:") + lines.append("-" * 72) + for result in report.results: + if not result.producer_ok or not result.consumer_ok: + lines.append("") + lines.append(f" Topic: {result.boundary.topic_name}") + lines.append( + f" Producer: {result.boundary.producer_repo} -> " + f"Consumer: {result.boundary.consumer_repo}" + ) + lines.append(f" Error: {result.error}") + lines.append("") + + if ok_count > 0 and fail_count == 0: + lines.append("All boundaries are in parity.") + elif ok_count > 0: + lines.append("-" * 72) + lines.append("OK boundaries:") + lines.append("-" * 72) + for result in report.results: + if result.producer_ok and result.consumer_ok: + lines.append( + f" [OK] {result.boundary.topic_name} " + f"({result.boundary.producer_repo} -> " + f"{result.boundary.consumer_repo})" + ) + + lines.append("") + return "\n".join(lines) + + +def main(argv: Sequence[str] | None = None) -> int: + """Gather declared files and run the parity CLI.""" + parser = argparse.ArgumentParser( + description="Check cross-repo Kafka boundary parity" + ) + parser.add_argument( + "--manifest", + type=str, + default=None, + help="Path to kafka_boundaries.yaml manifest (default: bundled in package)", + ) + parser.add_argument( + "--repos-root", + type=str, + required=True, + help="Root directory containing bare repo clones", + ) + parser.add_argument( + "--json", + action="store_true", + dest="json_output", + help="Output results as JSON instead of human-readable text", + ) + args = parser.parse_args(argv) + manifest_path = Path(args.manifest) if args.manifest else None + repos_root = Path(args.repos_root) + if manifest_path is not None and not manifest_path.is_file(): + sys.stderr.write(f"ERROR: Manifest not found: {manifest_path}\n") + return 1 + if not repos_root.is_dir(): + sys.stderr.write(f"ERROR: Repos root not found: {repos_root}\n") + return 1 + manifest_yaml = ( + manifest_path.read_text(encoding="utf-8") + if manifest_path + else load_manifest_yaml() + ) + entries = load_boundary_manifest(manifest_yaml) + paths = { + f"{repo}/{file}": str((repos_root / repo / file).absolute()) + for entry in entries + for repo, file in ( + (entry.producer_repo, entry.producer_file), + (entry.consumer_repo, entry.consumer_file), + ) + } + gathered = ( + NodeSourceFileGatherEffect().handle( + ModelSourceFileGatherInput( + root=str(repos_root), + explicit_paths=list(paths.values()), + include_patterns=["*"], + decode_errors="replace", + ) + ) + if paths + else None + ) + if gathered is not None: + for skipped in gathered.skipped: + if skipped.reason != "not a file": + sys.stderr.write(f"ERROR: {skipped.path}: {skipped.reason}\n") + return 1 + texts = {file.path: file.source for file in gathered.files} if gathered else {} + report = HandlerBoundaryParity().handle( + ModelBoundaryParityInput( + boundaries=entries, + files=[ + ModelSourceFile(path=key, source=texts[path]) + for key, path in paths.items() + if path in texts + ], + now=datetime.now(tz=UTC), + ) + ) + for entry in report.pending_in_grace: + sys.stdout.write( + f" PENDING (grace): {entry.topic_name} " + f"({entry.producer_repo} -> {entry.consumer_repo}) " + f"— {entry.pending_reason or 'no reason given'}\n" + ) + if args.json_output: + output: dict[str, object] = { + "total": len(report.results), + "ok": sum(1 for r in report.results if r.producer_ok and r.consumer_ok), + "mismatches": report.mismatch_count, + "boundaries": [ + { + "topic": r.boundary.topic_name, + "producer_repo": r.boundary.producer_repo, + "consumer_repo": r.boundary.consumer_repo, + "producer_ok": r.producer_ok, + "consumer_ok": r.consumer_ok, + "error": r.error, + } + for r in report.results + ], + } + sys.stdout.write(json.dumps(output, indent=2) + "\n") + else: + sys.stdout.write(format_report(report) + "\n") + return 1 if report.has_mismatches else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/omnibase_core/handlers/handler_migration_conflicts.py b/src/omnibase_core/handlers/handler_migration_conflicts.py new file mode 100644 index 0000000000..b28bf1d9ed --- /dev/null +++ b/src/omnibase_core/handlers/handler_migration_conflicts.py @@ -0,0 +1,756 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Migration conflicts and column references from onex_change_control (OMN-20074). + +The handler ports the source's decisions over explicit SQL and Python snapshots. +Discovery and reads belong to main and NodeSourceFileGatherEffect. +The source's --suppressions-file is not ported: the composite action never passed +it, and this repository does not introduce a new suppression mechanism. +Output retains the source's text without its colorama colour codes. +""" + +from __future__ import annotations + +import argparse +import importlib.resources +import re +import sys +from collections.abc import Iterator, Sequence +from pathlib import Path +from typing import Final + +from omnibase_core.enums.enum_migration_conflict_type import EnumMigrationConflictType +from omnibase_core.models.nodes.migration_conflicts.model_column_violation import ( + ModelColumnViolation, +) +from omnibase_core.models.nodes.migration_conflicts.model_migration_conflict import ( + ModelMigrationConflict, +) +from omnibase_core.models.nodes.migration_conflicts.model_migration_conflict_input import ( + ModelMigrationConflictInput, +) +from omnibase_core.models.nodes.migration_conflicts.model_migration_conflict_report import ( + ModelMigrationConflictReport, +) +from omnibase_core.models.nodes.migration_conflicts.model_migration_table_definition import ( + ModelMigrationTableDefinition, +) +from omnibase_core.models.nodes.migration_conflicts.model_repo_source_file import ( + ModelRepoSourceFile, +) +from omnibase_core.models.nodes.source_file_gather.model_source_file_gather_input import ( + ModelSourceFileGatherInput, +) +from omnibase_core.nodes.node_source_file_gather_effect.handler import ( + NodeSourceFileGatherEffect, +) +from omnibase_core.utils.util_safe_yaml_loader import load_yaml_mapping_no_duplicates + +INVENTORY_RESOURCE: Final[str] = "migration_inventory.yaml" + +UNINVENTORIED_BOUNDARY: Final[str] = "__uninventoried__" + +CREATE_TABLE_RE: Final[re.Pattern[str]] = re.compile( + r"CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?(\w+)\s*\((.*?)\);", + re.IGNORECASE | re.DOTALL, +) + +COLUMN_RE: Final[re.Pattern[str]] = re.compile( + r"^\s*(\w+)\s+([\w\[\]()]+)", + re.MULTILINE, +) + +CONSTRAINT_KEYWORDS: Final[frozenset[str]] = frozenset( + { + "PRIMARY", + "UNIQUE", + "CHECK", + "FOREIGN", + "CONSTRAINT", + "INDEX", + "CREATE", + "REFERENCES", + } +) + +MIGRATION_SCAN_EXCLUDED_PATH_PARTS: Final[frozenset[str]] = frozenset( + { + ".cache", + ".claude", + ".mypy_cache", + ".nox", + ".pytest_cache", + ".ruff_cache", + ".tox", + ".turbo", + ".venv", + "__pycache__", + "build", + "dist", + "htmlcov", + "node_modules", + "omni_worktrees", + "site-packages", + "venv", + } +) + +_MIN_TEST_FIXTURE_PATH_PARTS: Final[int] = 3 + +_INSERT_RE: Final[re.Pattern[str]] = re.compile( + r"INSERT\s+INTO\s+(\w+)\s*\(([^)]+)\)", + re.IGNORECASE, +) + +_SELECT_RE: Final[re.Pattern[str]] = re.compile( + r"SELECT\s+(.*?)\s+FROM\s+(\w+)", + re.IGNORECASE | re.DOTALL, +) + +_UPDATE_RE: Final[re.Pattern[str]] = re.compile( + r"UPDATE\s+(\w+)\s+SET\s+(.*?)(?:\s+WHERE|\s+RETURNING|\s*;|\s*$)", + re.IGNORECASE | re.DOTALL, +) + +_ON_CONFLICT_RE: Final[re.Pattern[str]] = re.compile( + r"ON\s+CONFLICT\s*\(([^)]+)\)", + re.IGNORECASE, +) + +_RETURNING_RE: Final[re.Pattern[str]] = re.compile( + r"RETURNING\s+(.*?)(?:\s*;|\s*$|\s*\))", + re.IGNORECASE | re.DOTALL, +) + +_ALTER_ADD_RE: Final[re.Pattern[str]] = re.compile( + r"ALTER\s+TABLE\s+(?:IF\s+EXISTS\s+)?(\w+)\s+ADD\s+(?:COLUMN\s+)?(\w+)\s+", + re.IGNORECASE, +) + +_ALTER_DROP_RE: Final[re.Pattern[str]] = re.compile( + r"ALTER\s+TABLE\s+(?:IF\s+EXISTS\s+)?(\w+)\s+DROP\s+(?:COLUMN\s+)?(?:IF\s+EXISTS\s+)?(\w+)", + re.IGNORECASE, +) + +_SQL_KEYWORDS: Final[frozenset[str]] = frozenset( + { + "*", + "null", + "true", + "false", + "default", + "now", + "current_timestamp", + "count", + "sum", + "avg", + "min", + "max", + "coalesce", + "case", + "when", + "then", + "else", + "end", + "as", + "distinct", + "all", + } +) + + +def load_inventory_yaml() -> str: + """Read the packaged byte-for-byte OCC migration inventory.""" + return ( + importlib.resources.files("omnibase_core.contracts") + / "boundaries" + / INVENTORY_RESOURCE + ).read_text(encoding="utf-8") + + +def _dict_items(value: object) -> Iterator[tuple[object, object]]: + """Return dict items for untyped YAML values.""" + if isinstance(value, dict): + yield from value.items() + + +def _list_items(value: object) -> Iterator[object]: + """Return list items for untyped YAML values.""" + if isinstance(value, list): + yield from value + + +def iter_inventory_files( + data: object, +) -> Iterator[tuple[str, str, str, str]]: + """Yield database, source repo, directory, and SQL filename from inventory.""" + databases = data.get("databases", {}) if isinstance(data, dict) else {} + for db_name, db_config in _dict_items(databases): + if not isinstance(db_config, dict): + continue + for migration_set in _list_items(db_config.get("migration_sets", [])): + if not isinstance(migration_set, dict): + continue + source_repo = migration_set.get("source_repo") + directory = migration_set.get("directory") + if not isinstance(source_repo, str) or not isinstance(directory, str): + continue + + for entry in _list_items(migration_set.get("migrations", [])): + if not isinstance(entry, dict): + continue + filename = entry.get("file") + if not isinstance(filename, str) or not filename.endswith(".sql"): + continue + yield str(db_name), source_repo, directory, filename + + +def extract_tables_from_sql( + source: str, path: str, repo_name: str +) -> list[ModelMigrationTableDefinition]: + """Extract CREATE TABLE definitions from a SQL migration file.""" + content = source + + tables = [] + for match in CREATE_TABLE_RE.finditer(content): + table_name = match.group(1).lower() + body = match.group(2) + + columns: set[str] = set() + for col_match in COLUMN_RE.finditer(body): + col_name = col_match.group(1).upper() + if col_name not in CONSTRAINT_KEYWORDS: + columns.add(col_match.group(1).lower()) + + if columns: + tables.append( + ModelMigrationTableDefinition( + table_name=table_name, + columns=frozenset(columns), + file_path=path, + repo_name=repo_name, + ) + ) + + return tables + + +def _collect_repo_schemas( + migration_files: Sequence[ModelRepoSourceFile], +) -> dict[str, dict[str, set[str]]]: + """Collect per-repo column schemas from migration files.""" + repo_schemas: dict[str, dict[str, set[str]]] = {} + + for sql_file in migration_files: + repo_name = sql_file.repo_name + + if repo_name not in repo_schemas: + repo_schemas[repo_name] = {} + + for table_def in extract_tables_from_sql( + sql_file.source, sql_file.path, repo_name + ): + tname = table_def.table_name + if tname not in repo_schemas[repo_name]: + repo_schemas[repo_name][tname] = set(table_def.columns) + else: + repo_schemas[repo_name][tname] |= set(table_def.columns) + + _apply_alter_statements(sql_file.source, repo_name, repo_schemas) + + return repo_schemas + + +def _apply_alter_statements( + content: str, + repo_name: str, + repo_schemas: dict[str, dict[str, set[str]]], +) -> None: + """Apply ALTER TABLE ADD/DROP COLUMN to repo schemas.""" + for m in _ALTER_ADD_RE.finditer(content): + tname = m.group(1).lower() + col = m.group(2).lower() + if tname not in repo_schemas[repo_name]: + repo_schemas[repo_name][tname] = set() + repo_schemas[repo_name][tname].add(col) + + for m in _ALTER_DROP_RE.finditer(content): + tname = m.group(1).lower() + col = m.group(2).lower() + if tname in repo_schemas[repo_name]: + repo_schemas[repo_name][tname].discard(col) + + +def _merge_schemas( + repo_schemas: dict[str, dict[str, set[str]]], +) -> tuple[dict[str, set[str]], set[str]]: + """Merge per-repo schemas, detecting ambiguous tables.""" + table_columns: dict[str, set[str]] = {} + ambiguous_tables: set[str] = set() + + table_to_repos: dict[str, dict[str, set[str]]] = {} + for repo_name, schemas in repo_schemas.items(): + for tname, cols in schemas.items(): + if tname not in table_to_repos: + table_to_repos[tname] = {} + table_to_repos[tname][repo_name] = cols + + for tname, repo_cols in table_to_repos.items(): + if len(repo_cols) > 1: + col_sets = list(repo_cols.values()) + if not all(c == col_sets[0] for c in col_sets): + ambiguous_tables.add(tname) + continue + merged: set[str] = set() + for cols in repo_cols.values(): + merged |= cols + table_columns[tname] = merged + + return table_columns, ambiguous_tables + + +def _extract_column_names(raw: str) -> list[str]: + """Extract column names from a comma-separated SQL fragment. + + Filters out expressions, function calls, *, and keywords. + """ + columns: list[str] = [] + for raw_token in raw.split(","): + token = raw_token.strip() + # Skip empty, expressions with parens, casts, qualified refs (foo.col) + if not token or "(" in token or "::" in token or "." in token: + continue + # Take first word only (handles "col AS alias") + word = token.split()[0].strip().lower() + # Skip keywords and non-identifiers + if word in _SQL_KEYWORDS or not re.match(r"^[a-z_]\w*$", word): + continue + columns.append(word) + return columns + + +def _extract_update_targets(raw: str) -> list[str]: + """Extract SET target column names from UPDATE ... SET fragment.""" + columns: list[str] = [] + for assignment in raw.split(","): + parts = assignment.split("=", 1) + if len(parts) >= 2: + col = parts[0].strip().lower() + if re.match(r"^[a-z_]\w*$", col): + columns.append(col) + return columns + + +def _add_column_ref( + refs: dict[str, set[str]], + known_tables: set[str], + table: str, + cols: list[str], +) -> None: + """Add column references for a table if it's in the known set.""" + table = table.lower() + if table in known_tables: + if table not in refs: + refs[table] = set() + refs[table].update(cols) + + +def _find_preceding_table( + content: str, + pos: int, +) -> str | None: + """Find the table name from the nearest preceding INSERT or UPDATE.""" + prefix = content[:pos] + insert_match = list(_INSERT_RE.finditer(prefix)) + update_match = list(_UPDATE_RE.finditer(prefix)) + if insert_match: + return insert_match[-1].group(1) + if update_match: + return update_match[-1].group(1) + return None + + +def _scan_python_for_column_refs( + content: str, + known_tables: set[str], +) -> dict[str, set[str]]: + """Scan a Python file for SQL string literals referencing known tables. + + Returns dict[table_name, set[referenced_column_names]]. + """ + refs: dict[str, set[str]] = {} + + for m in _INSERT_RE.finditer(content): + _add_column_ref( + refs, known_tables, m.group(1), _extract_column_names(m.group(2)) + ) + + for m in _SELECT_RE.finditer(content): + _add_column_ref( + refs, known_tables, m.group(2), _extract_column_names(m.group(1)) + ) + + for m in _UPDATE_RE.finditer(content): + _add_column_ref( + refs, known_tables, m.group(1), _extract_update_targets(m.group(2)) + ) + + for m in _ON_CONFLICT_RE.finditer(content): + insert_match = list(_INSERT_RE.finditer(content[: m.start()])) + if insert_match: + _add_column_ref( + refs, + known_tables, + insert_match[-1].group(1), + _extract_column_names(m.group(1)), + ) + + for m in _RETURNING_RE.finditer(content): + table = _find_preceding_table(content, m.start()) + if table: + _add_column_ref( + refs, known_tables, table, _extract_column_names(m.group(1)) + ) + + return refs + + +class HandlerMigrationConflicts: + """Detect conflicts and column drift without accessing the filesystem.""" + + def handle( + self, request: ModelMigrationConflictInput + ) -> ModelMigrationConflictReport: + """Return findings in the source's table, boundary and file order.""" + table_defs: dict[str, list[tuple[ModelMigrationTableDefinition, str]]] = {} + for file in request.sql_files: + for definition in extract_tables_from_sql( + file.source, file.path, file.repo_name + ): + table_defs.setdefault(definition.table_name, []).append( + (definition, file.resolved_path) + ) + conflicts: list[ModelMigrationConflict] = [] + for table_name, defs in table_defs.items(): + grouped: dict[str, list[ModelMigrationTableDefinition]] = {} + for definition, resolved_path in defs: + boundary = request.inventory_boundaries.get(resolved_path) + if boundary is None: + grouped = { + UNINVENTORIED_BOUNDARY: [definition for definition, _ in defs] + } + break + grouped.setdefault(boundary, []).append(definition) + for boundary_defs in grouped.values(): + if len(boundary_defs) <= 1: + continue + column_sets = [d.columns for d in boundary_defs] + conflict_type = ( + EnumMigrationConflictType.EXACT_DUPLICATE + if all(c == column_sets[0] for c in column_sets) + else EnumMigrationConflictType.NAME_CONFLICT + ) + conflicts.append( + ModelMigrationConflict( + conflict_type=conflict_type, + table_name=table_name, + definitions=boundary_defs, + ) + ) + if not request.check_columns: + return ModelMigrationConflictReport(conflicts=conflicts) + table_columns, ambiguous_tables = _merge_schemas( + _collect_repo_schemas(request.sql_files) + ) + known_tables = set(table_columns.keys()) + violations: list[ModelColumnViolation] = [] + for file in request.python_files: + refs = _scan_python_for_column_refs(file.source, known_tables) + for table, columns in refs.items(): + if table in ambiguous_tables: + continue + canonical = table_columns.get(table, set()) + for col in columns: + if col not in canonical: + violations.append( + ModelColumnViolation( + table=table, + column=col, + python_file=file.path, + repo=file.repo_name, + ) + ) + return ModelMigrationConflictReport( + conflicts=conflicts, + columns_checked=True, + column_violations=violations, + ambiguous_tables=sorted(ambiguous_tables), + ) + + +def format_conflicts(conflicts: list[ModelMigrationConflict]) -> str: + """Format conflicts for human-readable output.""" + if not conflicts: + return "No migration conflicts found." + + lines = [ + f"Found {len(conflicts)} migration conflict(s):", + "", + ] + + for conflict in conflicts: + if conflict.conflict_type == EnumMigrationConflictType.NAME_CONFLICT: + label = "NAME_CONFLICT" + else: + label = "EXACT_DUPLICATE" + + lines.append(f" {label}: table `{conflict.table_name}`") + + for defn in conflict.definitions: + lines.append( + f" - {defn.repo_name}: {Path(defn.file_path).name} " + f"({len(defn.columns)} columns)" + ) + + if conflict.conflict_type == EnumMigrationConflictType.NAME_CONFLICT: + # Show column diff + all_columns: set[str] = set() + for defn in conflict.definitions: + all_columns |= defn.columns + for defn in conflict.definitions: + missing = all_columns - defn.columns + if missing: + lines.append( + f" missing in {Path(defn.file_path).name}: " + f"{', '.join(sorted(missing))}" + ) + + lines.append("") + + return "\n".join(lines) + + +def format_column_violations(result: ModelMigrationConflictReport) -> str: + """Format column-reference violations for human-readable output.""" + lines: list[str] = [] + + if result.ambiguous_tables: + lines.append( + f"SCHEMA_AMBIGUOUS: {len(result.ambiguous_tables)} table(s) " + f"defined with conflicting schemas across repos — " + f"column validation skipped:" + ) + for t in sorted(result.ambiguous_tables): + lines.append(f" - {t}") + lines.append("") + + if not result.column_violations: + if not result.ambiguous_tables: + lines.append("No column-reference violations found.") + return "\n".join(lines) + + lines.append( + f"Found {len(result.column_violations)} column-reference violation(s):" + ) + lines.append("") + + for v in result.column_violations: + lines.append(f" MISSING_COLUMN: `{v.column}` in table `{v.table}`") + lines.append(f" file: {v.repo}/{Path(v.python_file).name}") + lines.append("") + + return "\n".join(lines) + + +def _is_excluded_migration_path(relative_parts: tuple[str, ...]) -> bool: + parts = tuple(part.lower() for part in relative_parts) + if any(part in MIGRATION_SCAN_EXCLUDED_PATH_PARTS for part in parts): + return True + return ( + len(parts) >= _MIN_TEST_FIXTURE_PATH_PARTS + and parts[0] in {"test", "tests"} + and "fixtures" in parts[1:] + ) + + +def find_migration_files( + repos_root: Path, repos: list[str] | None = None +) -> list[Path]: + """Find all SQL migration files under the given repos root.""" + if repos: + dirs = [repos_root / r for r in repos] + else: + dirs = [ + d for d in repos_root.iterdir() if d.is_dir() and not d.name.startswith(".") + ] + + migration_files = [] + for repo_dir in dirs: + if not repo_dir.is_dir(): + continue + for sql_file in repo_dir.rglob("**/migrations/**/*.sql"): + try: + relative_parts = sql_file.relative_to(repo_dir).parts + except ValueError: + relative_parts = sql_file.parts + if _is_excluded_migration_path(relative_parts): + continue + # Skip rollback migrations + if "rollback" in str(sql_file).lower(): + continue + migration_files.append(sql_file) + + return sorted(migration_files) + + +def _build_parser() -> argparse.ArgumentParser: + """Build the CLI argument parser.""" + parser = argparse.ArgumentParser( + description="Check for migration conflicts across repositories." + ) + parser.add_argument( + "--repos-root", + type=Path, + required=True, + help="Root directory containing repository directories.", + ) + parser.add_argument( + "--repos", + nargs="*", + default=None, + help="Specific repos to check (default: all under repos-root).", + ) + parser.add_argument( + "--warn-only", + action="store_true", + help="Exit 0 even if conflicts found (CI warning mode).", + ) + parser.add_argument( + "--check-columns", + action="store_true", + help="Also check that Python SQL references match migration DDL columns.", + ) + parser.add_argument( + "--migration-inventory", + type=Path, + default=None, + help="YAML inventory mapping migration files to logical database boundaries.", + ) + parser.add_argument( + "--warn-columns", + action="store_true", + help="Treat column-reference violations as warnings (exit 0).", + ) + return parser + + +def _gather_repo_files( + repos_root: Path, paths: list[Path], pattern: str +) -> list[ModelRepoSourceFile]: + """Read explicit paths through the effect, retaining discovery order.""" + if not paths: + return [] + absolute_paths = [str(path.absolute()) for path in paths] + gathered = NodeSourceFileGatherEffect().handle( + ModelSourceFileGatherInput( + root=str(repos_root), + explicit_paths=absolute_paths, + include_patterns=[pattern], + decode_errors="replace", + ) + ) + texts = {file.path: file.source for file in gathered.files} + files: list[ModelRepoSourceFile] = [] + for path, absolute_path in zip(paths, absolute_paths, strict=True): + if absolute_path not in texts: + continue + try: + repo_name = path.relative_to(repos_root).parts[0] + except ValueError: + repo_name = path.parent.name + files.append( + ModelRepoSourceFile( + repo_name=repo_name, + path=str(path), + resolved_path=str(path.resolve()), + source=texts[absolute_path], + ) + ) + return files + + +def main(argv: Sequence[str] | None = None) -> int: + """Discover sources and run the migration conflict CLI.""" + args = _build_parser().parse_args(argv) + repos_root: Path = args.repos_root + if not repos_root.is_dir(): + sys.stderr.write(f"Error: repos-root '{repos_root}' is not a directory.\n") + return 1 + inventory_path: Path | None = args.migration_inventory + if inventory_path is None: + inventory_yaml = load_inventory_yaml() + elif inventory_path.is_file(): + inventory_yaml = inventory_path.read_text(encoding="utf-8") + else: + # The source treats an absent inventory as no inventory at all. + inventory_yaml = "" + data = ( + load_yaml_mapping_no_duplicates( + inventory_yaml, source=str(inventory_path or INVENTORY_RESOURCE) + ) + if inventory_yaml.strip() + else {} + ) + boundaries = { + str((repos_root / source_repo / directory / filename).resolve()): db_name + for db_name, source_repo, directory, filename in iter_inventory_files(data) + } + sql_files = _gather_repo_files( + repos_root, find_migration_files(repos_root, args.repos), "*.sql" + ) + python_files: list[ModelRepoSourceFile] = [] + if args.check_columns: + dirs = ( + [repos_root / r for r in args.repos] + if args.repos + else [ + d + for d in repos_root.iterdir() + if d.is_dir() and not d.name.startswith(".") + ] + ) + for repo_dir in dirs: + src_dir = repo_dir / "src" + if not src_dir.is_dir(): + continue + try: + repo_name = repo_dir.relative_to(repos_root).parts[0] + except ValueError: + repo_name = repo_dir.name + python_files.extend( + file.model_copy(update={"repo_name": repo_name}) + for file in _gather_repo_files( + repos_root, list(src_dir.rglob("*.py")), "*.py" + ) + ) + report = HandlerMigrationConflicts().handle( + ModelMigrationConflictInput( + sql_files=sql_files, + inventory_boundaries=boundaries, + check_columns=args.check_columns, + python_files=python_files, + ) + ) + sys.stdout.write(format_conflicts(report.conflicts) + "\n") + if report.columns_checked: + sys.stdout.write(format_column_violations(report) + "\n") + if args.warn_only: + return 0 + if report.conflicts or (report.column_violations and not args.warn_columns): + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/omnibase_core/models/nodes/boundary_parity/__init__.py b/src/omnibase_core/models/nodes/boundary_parity/__init__.py new file mode 100644 index 0000000000..b8a73530de --- /dev/null +++ b/src/omnibase_core/models/nodes/boundary_parity/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Typed inputs and outputs for the cross-repo Kafka boundary parity check.""" diff --git a/src/omnibase_core/models/nodes/boundary_parity/model_boundary_parity_input.py b/src/omnibase_core/models/nodes/boundary_parity/model_boundary_parity_input.py new file mode 100644 index 0000000000..b1b74c5b14 --- /dev/null +++ b/src/omnibase_core/models/nodes/boundary_parity/model_boundary_parity_input.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Explicit snapshot supplied to the pure boundary parity handler.""" + +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_core.models.nodes.boundary_parity.model_kafka_boundary_entry import ( + ModelKafkaBoundaryEntry, +) +from omnibase_core.models.nodes.no_utcnow_check.model_source_file import ModelSourceFile + +__all__ = ["ModelBoundaryParityInput"] + + +class ModelBoundaryParityInput(BaseModel): + """What the parity handler decides over, without filesystem or clock access. + + files holds the producer and consumer files that exist, each keyed by + /; a declared file absent from + files is reported missing. now is the timezone-aware instant the + pending grace period is measured against. + """ + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + boundaries: list[ModelKafkaBoundaryEntry] = Field(default_factory=list) + files: list[ModelSourceFile] = Field(default_factory=list) + now: datetime diff --git a/src/omnibase_core/models/nodes/boundary_parity/model_boundary_parity_report.py b/src/omnibase_core/models/nodes/boundary_parity/model_boundary_parity_report.py new file mode 100644 index 0000000000..000e0eabf5 --- /dev/null +++ b/src/omnibase_core/models/nodes/boundary_parity/model_boundary_parity_report.py @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Aggregated result of the boundary parity handler.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_core.models.nodes.boundary_parity.model_boundary_parity_result import ( + ModelBoundaryParityResult, +) +from omnibase_core.models.nodes.boundary_parity.model_kafka_boundary_entry import ( + ModelKafkaBoundaryEntry, +) + +__all__ = ["ModelBoundaryParityReport"] + + +class ModelBoundaryParityReport(BaseModel): + """One result per checked boundary, plus the pending entries still in grace.""" + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + results: list[ModelBoundaryParityResult] = Field(default_factory=list) + pending_in_grace: list[ModelKafkaBoundaryEntry] = Field(default_factory=list) + + @property + def has_mismatches(self) -> bool: + """True when any boundary's producer or consumer misses its topic.""" + return any(not r.producer_ok or not r.consumer_ok for r in self.results) + + @property + def mismatch_count(self) -> int: + """Number of boundaries whose producer or consumer misses its topic.""" + return sum(1 for r in self.results if not r.producer_ok or not r.consumer_ok) diff --git a/src/omnibase_core/models/nodes/boundary_parity/model_boundary_parity_result.py b/src/omnibase_core/models/nodes/boundary_parity/model_boundary_parity_result.py new file mode 100644 index 0000000000..ae95911aaf --- /dev/null +++ b/src/omnibase_core/models/nodes/boundary_parity/model_boundary_parity_result.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""The parity verdict for one Kafka boundary.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +from omnibase_core.models.nodes.boundary_parity.model_kafka_boundary_entry import ( + ModelKafkaBoundaryEntry, +) + +__all__ = ["ModelBoundaryParityResult"] + + +class ModelBoundaryParityResult(BaseModel): + """Whether the producer and consumer files of one boundary reference its topic.""" + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + boundary: ModelKafkaBoundaryEntry + producer_ok: bool + consumer_ok: bool + producer_file_exists: bool + consumer_file_exists: bool + error: str = "" diff --git a/src/omnibase_core/models/nodes/boundary_parity/model_kafka_boundary_entry.py b/src/omnibase_core/models/nodes/boundary_parity/model_kafka_boundary_entry.py new file mode 100644 index 0000000000..a6b434a805 --- /dev/null +++ b/src/omnibase_core/models/nodes/boundary_parity/model_kafka_boundary_entry.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""One Kafka boundary declaration of the boundary manifest.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +__all__ = ["ModelKafkaBoundaryEntry"] + + +class ModelKafkaBoundaryEntry(BaseModel): + """A topic that crosses from a producer repository to a consumer repository. + + producer_file and consumer_file are relative to their repository + root. topic_pattern is a regular expression that must match somewhere in + each file. status is active or pending; a pending entry is + skipped while pending_since (YYYY-MM-DD) is within the grace period. + """ + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + topic_name: str + producer_repo: str + consumer_repo: str + producer_file: str + consumer_file: str + topic_pattern: str + event_schema: str = "" + status: str = "active" + pending_since: str = "" + pending_reason: str = "" diff --git a/src/omnibase_core/models/nodes/migration_conflicts/__init__.py b/src/omnibase_core/models/nodes/migration_conflicts/__init__.py new file mode 100644 index 0000000000..f7deb47c31 --- /dev/null +++ b/src/omnibase_core/models/nodes/migration_conflicts/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Typed inputs and outputs for the cross-repo migration conflict check.""" diff --git a/src/omnibase_core/models/nodes/migration_conflicts/model_column_violation.py b/src/omnibase_core/models/nodes/migration_conflicts/model_column_violation.py new file mode 100644 index 0000000000..ce2f87622c --- /dev/null +++ b/src/omnibase_core/models/nodes/migration_conflicts/model_column_violation.py @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""A column referenced from Python SQL that no migration declares.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +__all__ = ["ModelColumnViolation"] + + +class ModelColumnViolation(BaseModel): + """The table, the missing column and the Python file that referenced it.""" + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + table: str + column: str + python_file: str + repo: str diff --git a/src/omnibase_core/models/nodes/migration_conflicts/model_migration_conflict.py b/src/omnibase_core/models/nodes/migration_conflicts/model_migration_conflict.py new file mode 100644 index 0000000000..48cf3453ef --- /dev/null +++ b/src/omnibase_core/models/nodes/migration_conflicts/model_migration_conflict.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""One table defined more than once inside a logical database boundary.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_core.enums.enum_migration_conflict_type import ( + EnumMigrationConflictType, +) +from omnibase_core.models.nodes.migration_conflicts.model_migration_table_definition import ( + ModelMigrationTableDefinition, +) + +__all__ = ["ModelMigrationConflict"] + + +class ModelMigrationConflict(BaseModel): + """A NAME_CONFLICT (columns differ) or EXACT_DUPLICATE (columns equal).""" + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + conflict_type: EnumMigrationConflictType + table_name: str + definitions: list[ModelMigrationTableDefinition] = Field(default_factory=list) diff --git a/src/omnibase_core/models/nodes/migration_conflicts/model_migration_conflict_input.py b/src/omnibase_core/models/nodes/migration_conflicts/model_migration_conflict_input.py new file mode 100644 index 0000000000..f942604c28 --- /dev/null +++ b/src/omnibase_core/models/nodes/migration_conflicts/model_migration_conflict_input.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Explicit snapshot supplied to the pure migration conflict handler.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_core.models.nodes.migration_conflicts.model_repo_source_file import ( + ModelRepoSourceFile, +) + +__all__ = ["ModelMigrationConflictInput"] + + +class ModelMigrationConflictInput(BaseModel): + """What the migration conflict handler decides over, without filesystem access. + + sql_files are the migration files in discovery order. + inventory_boundaries maps a migration file's resolved path to the logical + database the migration inventory assigns it. python_files are the + src/ Python sources scanned for column references when + check_columns is set, in discovery order. + """ + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + sql_files: list[ModelRepoSourceFile] = Field(default_factory=list) + inventory_boundaries: dict[str, str] = Field(default_factory=dict) + check_columns: bool = False + python_files: list[ModelRepoSourceFile] = Field(default_factory=list) diff --git a/src/omnibase_core/models/nodes/migration_conflicts/model_migration_conflict_report.py b/src/omnibase_core/models/nodes/migration_conflicts/model_migration_conflict_report.py new file mode 100644 index 0000000000..ab58afdde3 --- /dev/null +++ b/src/omnibase_core/models/nodes/migration_conflicts/model_migration_conflict_report.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Aggregated result of the migration conflict handler.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_core.models.nodes.migration_conflicts.model_column_violation import ( + ModelColumnViolation, +) +from omnibase_core.models.nodes.migration_conflicts.model_migration_conflict import ( + ModelMigrationConflict, +) + +__all__ = ["ModelMigrationConflictReport"] + + +class ModelMigrationConflictReport(BaseModel): + """Conflicts per table and, when columns were checked, the column findings. + + columns_checked is False when the request did not ask for the column + check; column_violations and ambiguous_tables are then empty. + """ + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + conflicts: list[ModelMigrationConflict] = Field(default_factory=list) + columns_checked: bool = False + column_violations: list[ModelColumnViolation] = Field(default_factory=list) + ambiguous_tables: list[str] = Field(default_factory=list) diff --git a/src/omnibase_core/models/nodes/migration_conflicts/model_migration_table_definition.py b/src/omnibase_core/models/nodes/migration_conflicts/model_migration_table_definition.py new file mode 100644 index 0000000000..b4d57fba1b --- /dev/null +++ b/src/omnibase_core/models/nodes/migration_conflicts/model_migration_table_definition.py @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""A table definition extracted from one migration file.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +__all__ = ["ModelMigrationTableDefinition"] + + +class ModelMigrationTableDefinition(BaseModel): + """The lower-cased table name and column names one CREATE TABLE declares.""" + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + table_name: str + columns: frozenset[str] + file_path: str + repo_name: str diff --git a/src/omnibase_core/models/nodes/migration_conflicts/model_repo_source_file.py b/src/omnibase_core/models/nodes/migration_conflicts/model_repo_source_file.py new file mode 100644 index 0000000000..58cca2015a --- /dev/null +++ b/src/omnibase_core/models/nodes/migration_conflicts/model_repo_source_file.py @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""A source file read from one repository under the repos root.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +__all__ = ["ModelRepoSourceFile"] + + +class ModelRepoSourceFile(BaseModel): + """A (repository, path, source) triple. + + path is the path as discovered under the repos root. resolved_path + is that path with symbolic links resolved, the key the migration inventory + is matched on; it is empty for files the inventory is never matched against. + """ + + model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True) + + repo_name: str + path: str + resolved_path: str = "" + source: str diff --git a/tests/ci/test_validate_boundaries_action_omn20074.py b/tests/ci/test_validate_boundaries_action_omn20074.py new file mode 100644 index 0000000000..449ee15bc2 --- /dev/null +++ b/tests/ci/test_validate_boundaries_action_omn20074.py @@ -0,0 +1,117 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""OMN-20074: the validate-boundaries composite action moved into omnibase_core. + +A caller that repoints its ``uses:`` line to this action keeps its own job, so +its context name does not change; the inputs must be the moved action's +(``checks``, ``warn-only``, ``repos`` with the same defaults). The checks run +omnibase_core's handlers from the action's own checkout and never check out +or name the change-control repository. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any + +import pytest +import yaml + +pytestmark = pytest.mark.unit + +REPO_ROOT = Path(__file__).resolve().parents[2] +ACTION_PATH = REPO_ROOT / ".github" / "actions" / "validate-boundaries" / "action.yml" + +_FORBIDDEN_REFERENCE = "onex_change_control" +_FULL_SHA_USES = re.compile(r"^[\w.-]+/[\w./-]+@[0-9a-f]{40}$") +# Defaults of the moved action, verbatim. +_INPUT_DEFAULTS = { + "checks": "boundary-parity,migration-conflicts", + "warn-only": "true", + "repos": "omniclaude,omnidash,omniintelligence,omnibase_infra,omnibase_core,omnimemory,omnimarket", +} +_PARITY_CALL = ( + "python -m omnibase_core.handlers.handler_boundary_parity" + " --repos-root /tmp/omni_repos;" +) +_MIGRATION_CALL = ( + "python -m omnibase_core.handlers.handler_migration_conflicts" + " --repos-root /tmp/omni_repos --check-columns --warn-only;" +) + + +def _load() -> dict[str, Any]: + data = yaml.safe_load(ACTION_PATH.read_text(encoding="utf-8")) + assert isinstance(data, dict) + return data + + +def _steps() -> list[dict[str, Any]]: + steps = _load()["runs"]["steps"] + assert isinstance(steps, list) + return steps + + +def _step(name: str) -> dict[str, Any]: + matches = [s for s in _steps() if s.get("name") == name] + assert len(matches) == 1, name + return matches[0] + + +def test_is_a_composite_action_with_the_moved_inputs() -> None: + data = _load() + assert data["runs"]["using"] == "composite" + inputs = data["inputs"] + assert set(inputs) == set(_INPUT_DEFAULTS) + for name, default in _INPUT_DEFAULTS.items(): + assert inputs[name]["default"] == default, name + assert inputs[name]["required"] is False, name + + +def test_runs_the_core_handlers_with_the_moved_flags() -> None: + script = _step("Run boundary checks")["run"] + assert _PARITY_CALL in script + assert _MIGRATION_CALL in script + assert 'exit "$FAILURE_COUNT"' in script + assert '"${{ inputs.warn-only }}" == "true"' in script + + +def test_handlers_run_from_the_actions_own_checkout() -> None: + install = _step("Install omnibase_core boundary handlers")["run"] + assert 'CORE_ROOT="$(cd "${GITHUB_ACTION_PATH}/../../.." && pwd)"' in install + assert 'uv sync --project "${CORE_ROOT}" --frozen --no-dev' in install + run = _step("Run boundary checks") + assert run["env"]["CORE_ROOT"] == "${{ steps.core.outputs.root }}" + + +def test_never_references_the_change_control_repository() -> None: + text = ACTION_PATH.read_text(encoding="utf-8") + assert _FORBIDDEN_REFERENCE not in text + # Positive control: the scan finds the name when it is present. + assert ( + _FORBIDDEN_REFERENCE in text + "\n# repository: OmniNode-ai/onex_change_control" + ) + checkouts = [ + s for s in _steps() if str(s.get("uses", "")).startswith("actions/checkout@") + ] + assert len(checkouts) == 1 + assert "repository" not in checkouts[0].get("with", {}) + + +def test_every_action_is_pinned_by_full_sha() -> None: + uses = [s["uses"] for s in _steps() if "uses" in s] + assert uses + for ref in uses: + assert _FULL_SHA_USES.match(ref), ref + + +def test_handler_modules_expose_main() -> None: + from omnibase_core.handlers import ( + handler_boundary_parity, + handler_migration_conflicts, + ) + + assert callable(handler_boundary_parity.main) + assert callable(handler_migration_conflicts.main) diff --git a/tests/fixtures/occ_boundaries/manifest.yaml b/tests/fixtures/occ_boundaries/manifest.yaml new file mode 100644 index 0000000000..74dc02190d --- /dev/null +++ b/tests/fixtures/occ_boundaries/manifest.yaml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +--- +# +# Hashes of the onex_change_control boundary data the packaged copies carry +# (OMN-20074), taken over parsed content: sha256 of the JSON dump with sorted +# keys. The packaged copies add the SPDX header and write the six topic_name +# values of kafka_boundaries.yaml unquoted, the form of +# contracts/topic_registry.yaml; their parsed data is the source's. +source_repo: onex_change_control +source_rev: a89a6f30fa +kafka_boundaries_parsed_sha256: db2e9d19e20eac0ef2f17de2167f6923e958a3d9d168860ac598705637cb61aa +migration_inventory_parsed_sha256: 0bd4be2d6b2d653c6d0a599b7aeb5498ea62c14f9848b90f783d98e299ab3141 diff --git a/tests/unit/handlers/test_handler_boundary_parity.py b/tests/unit/handlers/test_handler_boundary_parity.py new file mode 100644 index 0000000000..7dba317084 --- /dev/null +++ b/tests/unit/handlers/test_handler_boundary_parity.py @@ -0,0 +1,206 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Decisions of the OCC ``check-boundary-parity`` script, ported (OMN-20074). + +The handler must reach the verdict of onex_change_control rev a89a6f30fa +(``src/onex_change_control/scripts/check_boundary_parity.py``) over fixture +repositories: a producer that no longer references its topic fails and names +the topic and the file, a clean pair passes, a pending entry inside its grace +period is skipped and one past it fails. +""" + +from __future__ import annotations + +import hashlib +import json +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest +import yaml + +from omnibase_core.handlers.handler_boundary_parity import ( + HandlerBoundaryParity, + load_boundary_manifest, + load_manifest_yaml, + main, +) +from omnibase_core.models.nodes.boundary_parity.model_boundary_parity_input import ( + ModelBoundaryParityInput, +) +from omnibase_core.models.nodes.no_utcnow_check.model_source_file import ModelSourceFile + +pytestmark = pytest.mark.unit + +FIXTURES = Path(__file__).resolve().parents[2] / "fixtures" / "occ_boundaries" +TOPIC = "onex.cmd.omniintelligence.claude-hook-event.v1" +PRODUCER_FILE = "src/omniclaude/hooks/topics.py" +CONSUMER_FILE = "src/omniintelligence/nodes/node_claude_hook_event_effect/contract.yaml" + +MANIFEST = f"""--- +version: "1" +boundaries: + - topic_name: "{TOPIC}" + producer_repo: omniclaude + consumer_repo: omniintelligence + producer_file: "{PRODUCER_FILE}" + consumer_file: "{CONSUMER_FILE}" + topic_pattern: "claude-hook-event\\\\.v1" + event_schema: "ModelClaudeCodeHookEvent" +""" + + +def _write(root: Path, rel: str, text: str) -> None: + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def _fixture(tmp_path: Path, *, producer: str | None, consumer: str) -> Path: + repos = tmp_path / "repos" + if producer is not None: + _write(repos / "omniclaude", PRODUCER_FILE, producer) + _write(repos / "omniintelligence", CONSUMER_FILE, consumer) + _write(tmp_path, "manifest.yaml", MANIFEST) + return repos + + +def _run( + tmp_path: Path, repos: Path, capsys: pytest.CaptureFixture[str] +) -> tuple[int, str]: + code = main( + ["--repos-root", str(repos), "--manifest", str(tmp_path / "manifest.yaml")] + ) + return code, capsys.readouterr().out + + +CLEAN_PRODUCER = ( + 'CLAUDE_HOOK_EVENT = "onex.cmd.omniintelligence.claude-hook-event.v1"\n' +) +CLEAN_CONSUMER = ( + "subscribe_topics:\n - onex.cmd.omniintelligence.claude-hook-event.v1\n" +) + + +def test_clean_fixture_passes( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + repos = _fixture(tmp_path, producer=CLEAN_PRODUCER, consumer=CLEAN_CONSUMER) + code, out = _run(tmp_path, repos, capsys) + assert code == 0 + assert "Total boundaries: 1" in out + assert " OK: 1" in out + assert " MISMATCH: 0" in out + assert "All boundaries are in parity." in out + + +def test_producer_pattern_missing_fails_naming_topic_and_file( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + repos = _fixture( + tmp_path, producer="UNRELATED = 'nothing here'\n", consumer=CLEAN_CONSUMER + ) + code, out = _run(tmp_path, repos, capsys) + assert code == 1 + assert " MISMATCH: 1" in out + assert f" Topic: {TOPIC}" in out + assert " Producer: omniclaude -> Consumer: omniintelligence" in out + assert f" Error: topic not found in producer: omniclaude/{PRODUCER_FILE}" in out + + +def test_event_name_segment_alone_satisfies_the_check( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + # The source's third fallback: segment 4 of the topic name ("claude-hook-event"). + repos = _fixture( + tmp_path, producer="NAME = 'claude-hook-event'\n", consumer=CLEAN_CONSUMER + ) + code, _ = _run(tmp_path, repos, capsys) + assert code == 0 + + +def test_missing_producer_file_fails( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + repos = _fixture(tmp_path, producer=None, consumer=CLEAN_CONSUMER) + code, out = _run(tmp_path, repos, capsys) + assert code == 1 + assert f" Error: producer file missing: omniclaude/{PRODUCER_FILE}" in out + + +def test_missing_repos_root_is_an_error( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + _write(tmp_path, "manifest.yaml", MANIFEST) + code = main( + [ + "--repos-root", + str(tmp_path / "absent"), + "--manifest", + str(tmp_path / "manifest.yaml"), + ] + ) + assert code == 1 + assert "ERROR: Repos root not found" in capsys.readouterr().err + + +def _pending_request(since: str, now: datetime) -> ModelBoundaryParityInput: + entry = load_boundary_manifest(MANIFEST)[0].model_copy( + update={ + "status": "pending", + "pending_since": since, + "pending_reason": "consumer lands next", + } + ) + return ModelBoundaryParityInput(boundaries=[entry], files=[], now=now) + + +def test_pending_inside_grace_is_skipped() -> None: + now = datetime(2026, 10, 10, 12, tzinfo=UTC) + since = (now - timedelta(days=14)).strftime("%Y-%m-%d") + report = HandlerBoundaryParity().handle(_pending_request(since, now)) + assert report.results == [] + assert [e.topic_name for e in report.pending_in_grace] == [TOPIC] + assert not report.has_mismatches + + +def test_pending_past_grace_fails() -> None: + now = datetime(2026, 10, 10, 12, tzinfo=UTC) + since = (now - timedelta(days=15)).strftime("%Y-%m-%d") + report = HandlerBoundaryParity().handle(_pending_request(since, now)) + assert report.mismatch_count == 1 + assert report.results[0].error == ( + f"EXPIRED PENDING: {TOPIC} has been pending for 15 days (since {since}, " + "grace period is 14 days). Reason: consumer lands next" + ) + + +def test_handler_reports_per_side_verdicts() -> None: + entries = load_boundary_manifest(MANIFEST) + report = HandlerBoundaryParity().handle( + ModelBoundaryParityInput( + boundaries=entries, + files=[ + ModelSourceFile(path=f"omniclaude/{PRODUCER_FILE}", source="x = 1\n"), + ModelSourceFile( + path=f"omniintelligence/{CONSUMER_FILE}", source=CLEAN_CONSUMER + ), + ], + now=datetime(2026, 10, 10, tzinfo=UTC), + ) + ) + (result,) = report.results + assert result.producer_file_exists + assert not result.producer_ok + assert result.consumer_ok + + +def test_packaged_manifest_is_the_occ_manifest() -> None: + expected = yaml.safe_load((FIXTURES / "manifest.yaml").read_text(encoding="utf-8")) + text = load_manifest_yaml() + parsed = json.dumps(yaml.safe_load(text), sort_keys=True).encode("utf-8") + assert ( + hashlib.sha256(parsed).hexdigest() == expected["kafka_boundaries_parsed_sha256"] + ) + assert len(load_boundary_manifest(text)) == 6 diff --git a/tests/unit/handlers/test_handler_migration_conflicts.py b/tests/unit/handlers/test_handler_migration_conflicts.py new file mode 100644 index 0000000000..0bf79cc268 --- /dev/null +++ b/tests/unit/handlers/test_handler_migration_conflicts.py @@ -0,0 +1,191 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Decisions of the OCC ``check-migration-conflicts`` script, ported (OMN-20074). + +The handler must reach the verdict of onex_change_control rev a89a6f30fa +(``src/onex_change_control/scripts/check_migration_conflicts.py``) over fixture +repositories: one table created with different columns by two migrations in the +same (uninventoried) boundary is a NAME_CONFLICT, the same columns an +EXACT_DUPLICATE, two logical databases of the inventory never conflict, and a +Python SQL literal naming an undeclared column is a column violation. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest +import yaml + +from omnibase_core.enums.enum_migration_conflict_type import ( + EnumMigrationConflictType, +) +from omnibase_core.handlers.handler_migration_conflicts import ( + load_inventory_yaml, + main, +) + +pytestmark = pytest.mark.unit + +FIXTURES = Path(__file__).resolve().parents[2] / "fixtures" / "occ_boundaries" + +FOO_TWO_COLUMNS = "CREATE TABLE foo (\n id UUID PRIMARY KEY,\n name TEXT\n);\n" +FOO_THREE_COLUMNS = ( + "CREATE TABLE IF NOT EXISTS foo (\n" + " id UUID PRIMARY KEY,\n" + " name TEXT,\n" + " created_at TIMESTAMPTZ\n" + ");\n" +) +BAR = "CREATE TABLE bar (\n id UUID PRIMARY KEY\n);\n" +EMPTY_INVENTORY = '---\nversion: "1"\ndatabases: {}\n' + + +def _write(root: Path, rel: str, text: str) -> None: + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def _run( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + *extra: str, + inventory: str = EMPTY_INVENTORY, +) -> tuple[int, str]: + _write(tmp_path, "inventory.yaml", inventory) + code = main( + [ + "--repos-root", + str(tmp_path / "repos"), + "--migration-inventory", + str(tmp_path / "inventory.yaml"), + *extra, + ] + ) + return code, capsys.readouterr().out + + +def test_name_conflict_fails_naming_table_and_files( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + repos = tmp_path / "repos" + _write(repos / "repo_a", "docker/migrations/001_foo.sql", FOO_TWO_COLUMNS) + _write(repos / "repo_b", "sql/migrations/007_foo.sql", FOO_THREE_COLUMNS) + code, out = _run(tmp_path, capsys) + assert code == 1 + assert "Found 1 migration conflict(s):" in out + assert " NAME_CONFLICT: table `foo`" in out + assert " - repo_a: 001_foo.sql (2 columns)" in out + assert " - repo_b: 007_foo.sql (3 columns)" in out + assert " missing in 001_foo.sql: created_at" in out + + +def test_warn_only_reports_but_exits_zero( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + repos = tmp_path / "repos" + _write(repos / "repo_a", "docker/migrations/001_foo.sql", FOO_TWO_COLUMNS) + _write(repos / "repo_b", "sql/migrations/007_foo.sql", FOO_THREE_COLUMNS) + code, out = _run(tmp_path, capsys, "--warn-only") + assert code == 0 + assert " NAME_CONFLICT: table `foo`" in out + + +def test_exact_duplicate(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + repos = tmp_path / "repos" + _write(repos / "repo_a", "docker/migrations/001_foo.sql", FOO_TWO_COLUMNS) + _write(repos / "repo_b", "sql/migrations/002_foo.sql", FOO_TWO_COLUMNS) + code, out = _run(tmp_path, capsys) + assert code == 1 + assert " EXACT_DUPLICATE: table `foo`" in out + + +def test_clean_fixture_passes( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + repos = tmp_path / "repos" + _write(repos / "repo_a", "docker/migrations/001_foo.sql", FOO_TWO_COLUMNS) + _write(repos / "repo_b", "sql/migrations/001_bar.sql", BAR) + code, out = _run(tmp_path, capsys) + assert code == 0 + assert "No migration conflicts found." in out + + +def test_inventory_separates_logical_databases( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + repos = tmp_path / "repos" + _write(repos / "repo_a", "docker/migrations/001_foo.sql", FOO_TWO_COLUMNS) + _write(repos / "repo_b", "sql/migrations/007_foo.sql", FOO_THREE_COLUMNS) + inventory = """--- +version: "1" +databases: + db_a: + migration_sets: + - source_repo: repo_a + directory: docker/migrations + migrations: + - file: "001_foo.sql" + db_b: + migration_sets: + - source_repo: repo_b + directory: sql/migrations + migrations: + - file: "007_foo.sql" +""" + code, out = _run(tmp_path, capsys, inventory=inventory) + assert code == 0 + assert "No migration conflicts found." in out + + +def test_excluded_paths_are_not_scanned( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + repos = tmp_path / "repos" + _write(repos / "repo_a", "docker/migrations/001_foo.sql", FOO_TWO_COLUMNS) + _write( + repos / "repo_a", + "tests/fixtures/x/migrations/001_foo.sql", + FOO_THREE_COLUMNS, + ) + _write(repos / "repo_b", "migrations/rollback/001_foo.sql", FOO_THREE_COLUMNS) + _write(repos / "repo_b", ".venv/migrations/001_foo.sql", FOO_THREE_COLUMNS) + code, out = _run(tmp_path, capsys) + assert code == 0 + assert "No migration conflicts found." in out + + +def test_column_reference_to_undeclared_column( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + repos = tmp_path / "repos" + _write(repos / "repo_a", "docker/migrations/001_foo.sql", FOO_TWO_COLUMNS) + _write( + repos / "repo_a", + "src/repo_a/writer.py", + 'SQL = "INSERT INTO foo (id, name, nope) VALUES ($1, $2, $3)"\n', + ) + code, out = _run(tmp_path, capsys, "--check-columns") + assert code == 1 + assert "No migration conflicts found." in out + assert "Found 1 column-reference violation(s):" in out + assert " MISSING_COLUMN: `nope` in table `foo`" in out + assert " file: repo_a/writer.py" in out + code, _ = _run(tmp_path, capsys, "--check-columns", "--warn-columns") + assert code == 0 + + +def test_conflict_types_are_the_core_enum() -> None: + assert EnumMigrationConflictType.NAME_CONFLICT.value == "name_conflict" + assert EnumMigrationConflictType.EXACT_DUPLICATE.value == "exact_duplicate" + + +def test_packaged_inventory_is_the_occ_inventory() -> None: + expected = yaml.safe_load((FIXTURES / "manifest.yaml").read_text(encoding="utf-8")) + parsed = json.dumps(yaml.safe_load(load_inventory_yaml()), sort_keys=True) + digest = hashlib.sha256(parsed.encode("utf-8")).hexdigest() + assert digest == expected["migration_inventory_parsed_sha256"]