diff --git a/contracts/OMN-20415.yaml b/contracts/OMN-20415.yaml new file mode 100644 index 0000000000..2e5d35c6f1 --- /dev/null +++ b/contracts/OMN-20415.yaml @@ -0,0 +1,71 @@ +--- +schema_version: "1.0.0" +ticket_id: "OMN-20415" +title: "The landing decision node groups red PRs by shared cause and dispatches one bounded cause worker per cluster" +summary: "Red PRs in one repo failing one check with one normalized failure signature form a cluster, and the decision node dispatches one bounded cause worker per cause. A spent cause parks with one operator escalation, and its members fall back to per-PR dispatch so red PRs keep getting fixed." +is_seam_ticket: false +interface_change: false +interfaces_touched: [] +requirements: + - id: "req-shared-cause-grouping" + statement: "The pure landing decision node groups red PRs by shared cause and dispatches one bounded cause worker per cause, with the fixer HOLD, the attempt budget, the park and the fallback of parked members to per-PR dispatch." + acceptance: + - id: "AC1" + statement: "The replay of controller tick 722 dispatches exactly one cause worker for omnimarket with at least 16 members, no per-PR worker for any member, and covers omnimarket#3321 by a cause." + - id: "AC2" + statement: "The same facts in any input order give a byte-identical decision." + - id: "AC3" + statement: "Scenarios S21 to S28 from the T1 model each have a test that fails on the pinned node and passes after the change." + - id: "AC4" + statement: "A cause is dispatched at most twice per park episode and then parked with one escalate_operator action, its members fall back to per-PR dispatch while it is parked, and a fixer HOLD on its repo yields no dispatch and a revoke for every live lease there." + - id: "AC5" + statement: "An escalated PR alone becomes a singleton cause on the next tick." +evidence_requirements: + - kind: "tests" + description: "Execute the shared-cause tests and the unchanged S1 to S20 scenario file against the decision node. S1 to S20 pass at the merge base by design, so they are evidence here and carry no must-fail bound check." + command: "uv run pytest tests/unit/nodes/node_pr_landing_decision_compute -q" +emergency_bypass: + enabled: false + justification: "" + follow_up_ticket_id: "" +dod_evidence: + - id: "dod-replay-tick722" + description: "The tick 722 replay forms one cause and dispatches no per-PR member worker." + source: "manual" + checks: + - check_type: "test_passes" + check_value: "uv run pytest tests/unit/nodes/node_pr_landing_decision_compute/test_landing_decision_causes.py -q -k replay_tick722" + cwd: "${OMNI_HOME}/omnimarket" + binds_ac: ["AC1"] + - id: "dod-cause-permutation" + description: "Input order does not change the decision." + source: "manual" + checks: + - check_type: "test_passes" + check_value: "uv run pytest tests/unit/nodes/node_pr_landing_decision_compute/test_landing_decision_causes.py -q -k cause_permutation" + cwd: "${OMNI_HOME}/omnimarket" + binds_ac: ["AC2"] + - id: "dod-cause-scenarios" + description: "Scenarios S21 to S28 hold." + source: "manual" + checks: + - check_type: "test_passes" + check_value: "uv run pytest tests/unit/nodes/node_pr_landing_decision_compute/test_landing_decision_causes.py -q -k cause_scenario" + cwd: "${OMNI_HOME}/omnimarket" + binds_ac: ["AC3"] + - id: "dod-budget-hold-fallback" + description: "The attempt budget, the fixer HOLD and the fallback of parked members hold." + source: "manual" + checks: + - check_type: "test_passes" + check_value: "uv run pytest tests/unit/nodes/node_pr_landing_decision_compute/test_landing_decision_causes.py -q -k 'budget or fixer_hold or fallback'" + cwd: "${OMNI_HOME}/omnimarket" + binds_ac: ["AC4"] + - id: "dod-escalated-singleton" + description: "An escalated PR becomes a singleton cause." + source: "manual" + checks: + - check_type: "test_passes" + check_value: "uv run pytest tests/unit/nodes/node_pr_landing_decision_compute/test_landing_decision_causes.py -q -k escalated_singleton" + cwd: "${OMNI_HOME}/omnimarket" + binds_ac: ["AC5"] diff --git a/pyproject.toml b/pyproject.toml index f72cef9f81..d35f1813bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "omnimarket" -version = "0.4.290" +version = "0.4.291" description = "OmniMarket - Portable ONEX workflow package registry" authors = [ {name = "OmniNode.ai", email = "contact@omninode.ai"} diff --git a/src/omnimarket/nodes/node_pr_landing_decision_compute/contract.yaml b/src/omnimarket/nodes/node_pr_landing_decision_compute/contract.yaml index 2cb7d7fcab..30fb1421e1 100644 --- a/src/omnimarket/nodes/node_pr_landing_decision_compute/contract.yaml +++ b/src/omnimarket/nodes/node_pr_landing_decision_compute/contract.yaml @@ -15,6 +15,17 @@ # written ahead of delivery). The fixed worker briefs are typed models in the # output. No clock (observed_at is the snapshot's time), no I/O, no model. # +# 0.2.0 adds shared red causes: red PRs in one repo failing one check with one +# normalized failure signature form a cluster, and one bounded cause worker is +# dispatched per cause (cause caps inside the pool, a two-attempt budget per +# park episode, a 12-hour park with one operator escalation, a fixer HOLD that +# revokes and dispatches nothing), with per-PR dispatch suppressed for the +# members of a leased, fixing or owned cause; a cause parked with no owner +# releases its members to per-PR dispatch (the fallback). +# An exit with no result after the spawn window spends an attempt (LC-F2), and the +# park's operator MSG is deduped against the ledger row, not the state (LC-F3); +# both from the T1 model, omnibase_internal#144. +# # The landing controller calls it in process, so it has no event_bus; it is # addressable through runtime_dispatch like the other directly-invoked compute # nodes. Experimental until the controller runs it (observe-only first). @@ -25,11 +36,11 @@ lifecycle: experimental node_type: "COMPUTE_GENERIC" contract_version: major: 0 - minor: 1 + minor: 2 patch: 0 node_version: major: 0 - minor: 1 + minor: 2 patch: 0 description: > Pure definition-B decision of one landing-controller tick. Reads one @@ -37,9 +48,10 @@ description: > worker result files, producer acknowledgements and failures, and the controller's last state) and returns the tick's actions (merge pinned to a head, rerun, update-branch, eligibility rerun, companion rebuild or close, - dispatch of a fixed worker brief, kill, discard, observe-only), the outcomes - recorded, DEGRADED reasons, violations and the next state to write ahead of - acting. Deterministic in any input order. + dispatch of a fixed worker brief or of one cause brief per shared red cause, + kill, discard, observe-only, operator escalation of a parked cause), the + outcomes recorded, DEGRADED reasons, violations and the next state to write + ahead of acting. Deterministic in any input order. input_model: name: "ModelLandingFacts" module: "omnimarket.nodes.node_pr_landing_decision_compute.models.model_landing_facts" diff --git a/src/omnimarket/nodes/node_pr_landing_decision_compute/handlers/handler_pr_landing_decision.py b/src/omnimarket/nodes/node_pr_landing_decision_compute/handlers/handler_pr_landing_decision.py index a9d4b65289..c489619fac 100644 --- a/src/omnimarket/nodes/node_pr_landing_decision_compute/handlers/handler_pr_landing_decision.py +++ b/src/omnimarket/nodes/node_pr_landing_decision_compute/handlers/handler_pr_landing_decision.py @@ -26,14 +26,29 @@ delivery was recorded (S19, S20). 6. Companions, before any other selection (R2 rules 1 to 5), then uncovered members (``CoverRequest``). -7. Product PRs: chain roots only (R3), update-branch once per head, a head +7. Shared causes, before any per-PR dispatch: a park ends at its time or on + a RELEASE row; a verified fix PR that merged gets one rerun per member head, + and the members' next grade ends the attempt. Red PRs in one repo sharing a + (check, failure signature) form a cluster (3 members, 2 under the landing + floor), largest first, absorbing clusters mostly inside an earlier cause; a + parked PR no cluster covers is a cause of one. Causes dispatch in floor, + size and key order under the cause caps, inside the pool, with a two-attempt + budget per park episode; a spent cause parks and escalates once. +8. Product PRs: chain roots only (R3), update-branch once per head, a head held at BLOCKED only by stale cancelled check copies refreshed once per head and at most ``max_stale_refreshes`` times per PR (then DEGRADED, no worker), blocked-outcome suppression by fingerprint (R8), one rerun per head (R5), the escalation ladder and parking (R1), the token (R6), pinned merge under a live lease (``PinMerge``), and dispatch in priority order (R4) under the - worker pool, the load pause and the available engines (D5). + worker pool, the per-repo cap, the load pause and the available engines + (D5). No per-PR worker is dispatched for a member of a cause that holds a + live lease or an open fix PR, or a park an owner's CLAIM covers; a park + alone releases its members to per-PR dispatch (the fallback). + +A fixer HOLD (a repo, or all) dispatches no worker in its scope and revokes +every live lease there through the kill sequence; merges, reruns and branch +updates go on. Actions for an observe-only repo are returned in ``observed_actions`` and never performed; no worker is dispatched in a draining repo. @@ -42,11 +57,13 @@ from __future__ import annotations import hashlib +import re from dataclasses import dataclass, field from datetime import datetime, timedelta from omnimarket.nodes.node_pr_landing_decision_compute.models.enum_landing import ( BLOCKED_OUTCOMES, + CAUSE_EXCLUDED_SUSPENSIONS, RERUNNABLE_RED_CLASSES, EnumLandingActionKind, EnumLandingBriefClass, @@ -65,10 +82,12 @@ EnumLandingRedClass, EnumLandingRefUpdateKind, EnumLandingResultKind, + EnumLandingSuspension, ) from omnimarket.nodes.node_pr_landing_decision_compute.models.model_landing_decision import ( BRIEF_INSTRUCTIONS, ModelLandingAction, + ModelLandingCauseBrief, ModelLandingCompanionVerdictRow, ModelLandingDecision, ModelLandingDegraded, @@ -77,7 +96,9 @@ ModelLandingWorkerBrief, ) from omnimarket.nodes.node_pr_landing_decision_compute.models.model_landing_facts import ( + FIXER_HOLD_ALL, ModelLandingBlockerRef, + ModelLandingCauseEscalation, ModelLandingCompanionFacts, ModelLandingFacts, ModelLandingPrFacts, @@ -86,6 +107,8 @@ ModelLandingWorkerResult, ) from omnimarket.nodes.node_pr_landing_decision_compute.models.model_landing_state import ( + ModelLandingCausePair, + ModelLandingCauseRecord, ModelLandingControllerState, ModelLandingEligibilityRerun, ModelLandingLease, @@ -102,13 +125,77 @@ EnumLandingResultKind.BLOCKED: EnumLandingOutcomeReason.BARE_BLOCKED, EnumLandingResultKind.UNPARSEABLE: EnumLandingOutcomeReason.UNPARSEABLE, } +_CAUSE_RESULT_KINDS: frozenset[EnumLandingResultKind] = frozenset( + { + EnumLandingResultKind.CAUSE_FIX_SUBMITTED, + EnumLandingResultKind.CAUSE_NOT_SHARED, + } +) + + +CAUSE_PREFIX = "cause:" +UNREAD = "unread" +SIGNATURE_TEXT_LIMIT = 300 + +_GENERIC_EXIT = re.compile(r"^Process completed with exit code \d+\.?$") +_TIMESTAMP = re.compile( + r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?" +) +_SHA = re.compile(r"\b(?=[0-9a-f]*\d)[0-9a-f]{7,64}\b") +_PR_NUMBER = re.compile(r"#\d+") +_BRACKETED = re.compile(r"\[[^\[\]]*\]") +_DIGITS = re.compile(r"\d+") +_SPACE = re.compile(r"\s+") + + +def is_cause(subject: str) -> bool: + """Whether a subject is a cause key rather than a PR.""" + return subject.startswith(CAUSE_PREFIX) def repo_of(subject: str) -> str: - """The ``owner/name`` of a ``repo#pr`` subject (a repo subject is itself).""" + """The ``owner/name`` of a ``repo#pr`` subject or a cause key (a repo is itself).""" + if is_cause(subject): + return subject[len(CAUSE_PREFIX) :].rsplit(":", 1)[0] return subject.split("#", 1)[0] +def cause_key(repo: str, signature: str) -> str: + """``cause:/:``.""" + return f"{CAUSE_PREFIX}{repo}:{signature}" + + +def normalize_annotation(text: str) -> str: + """A failure annotation with everything that varies per PR or per run replaced. + + The generic "Process completed with exit code N" line is skipped; + timestamps, shas, ``#n`` references, bracketed id lists and digit runs are + replaced; whitespace is collapsed; the result is cut to 300 characters. + """ + lines = (line.strip() for line in text.splitlines()) + out = " ".join(line for line in lines if line and not _GENERIC_EXIT.match(line)) + out = _TIMESTAMP.sub("", out) + out = _SHA.sub("", out) + out = _PR_NUMBER.sub("#", out) + out = _BRACKETED.sub("[]", out) + out = _DIGITS.sub("", out) + return _SPACE.sub(" ", out).strip()[:SIGNATURE_TEXT_LIMIT] + + +def normalize_signature(check: str, text: str | None) -> str: + """The failure signature of one red check: 12 hex of sha256(check, normalized text). + + A check with no annotation, or one that normalizes to nothing, is + ``unread`` and never clusters. + """ + if text is None: + return UNREAD + normalized = normalize_annotation(text) + if not normalized: + return UNREAD + return hashlib.sha256(f"{check}\n{normalized}".encode()).hexdigest()[:12] + + def rebuild_key(target_repo: str, members: tuple[ModelLandingMemberRef, ...]) -> str: """The idempotency key K: target repo plus the sorted membership fingerprint (R2).""" lines = sorted(f"{m.pr}@{m.head_sha}" for m in members) @@ -165,6 +252,10 @@ class _Tick: token: str | None next_lease_id: int close_requested: set[str] + causes: dict[str, ModelLandingCauseRecord] = field(default_factory=dict) + hold: set[str] = field(default_factory=set) + formed: set[str] = field(default_factory=set) + suppressed: set[str] = field(default_factory=set) elig_reruns: list[ModelLandingEligibilityRerun] = field(default_factory=list) actions: list[ModelLandingAction] = field(default_factory=list) observed: list[ModelLandingAction] = field(default_factory=list) @@ -197,6 +288,10 @@ def fingerprint(self, pr: str) -> str: facts = self.prs.get(pr) return blocker_fingerprint(facts.blocker_refs) if facts is not None else "" + def held(self, repo: str) -> bool: + """Under a fixer HOLD: no dispatch, and every live lease is revoked.""" + return FIXER_HOLD_ALL in self.hold or repo in self.hold + # ---------------------------------------------------------------- escalation def _escalate(t: _Tick, pr: str) -> None: @@ -309,6 +404,9 @@ def _read_result(t: _Tick, result: ModelLandingWorkerResult) -> None: ): t.emit(discard) return + if is_cause(lease.pr): + _read_cause_result(t, lease, result) + return pr = lease.pr subj = t.subject(pr) kind = result.kind @@ -336,6 +434,8 @@ def _read_result(t: _Tick, result: ModelLandingWorkerResult) -> None: outcome = EnumLandingOutcome.EXTERNAL_BLOCKER else: reason = EnumLandingOutcomeReason.BARE_BLOCKED + elif kind in _CAUSE_RESULT_KINDS: # a cause result from a per-PR worker + reason = EnumLandingOutcomeReason.UNVERIFIED_CLAIM else: # fix_submitted reason = _verify_fix(lease, result, subj) if reason is EnumLandingOutcomeReason.NONE: @@ -390,6 +490,9 @@ def _observe_lease(t: _Tick, lease: ModelLandingLease) -> None: alive = probe is None or probe.group_alive or probe.tagged_alive if not alive: del t.leases[pr] + if is_cause(pr): + _cause_lease_ended(t, lease) + return if lease.result_recorded_at is None: if lease.revoked: why = EnumLandingOutcomeReason.REVOKED @@ -402,7 +505,7 @@ def _observe_lease(t: _Tick, lease: ModelLandingLease) -> None: _escalate(t, pr) return if ( - repo_of(pr) in t.draining + (repo_of(pr) in t.draining or t.held(repo_of(pr))) and lease.result_recorded_at is None and not lease.revoked ): @@ -558,12 +661,15 @@ def _try_dispatch( ) -> bool: """Dispatch one worker, when the pool, the load and the engines allow (D5, P7).""" policy = t.facts.policy - if repo_of(subject) in t.draining or subject in t.leases: + repo = repo_of(subject) + if repo in t.draining or t.held(repo) or subject in t.leases: return False if t.facts.load1 > policy.load_pause_threshold: return False if len(t.leases) >= policy.max_workers: return False + if sum(1 for s in t.leases if repo_of(s) == repo) >= policy.max_workers_per_repo: + return False rec = t.record_of(subject) available = set(t.facts.available_engines) engine: EnumLandingEngine | None = next( @@ -779,6 +885,597 @@ def _cover(t: _Tick) -> None: _request(t, unc.target_repo, members, None) +# ------------------------------------------------------------------- causes +def _cause_lease_ended(t: _Tick, lease: ModelLandingLease) -> None: + """A cause lease confirmed terminated with no result. + + The deadline passing with the process alive spends an attempt, and so does + an exit with no result at or after the spawn window (LC-F2: counted + nowhere, a cause whose worker keeps dying was redispatched without bound). + Only an exit inside the window (a usage limit, a spawn failure) counts + toward the spawn-failure budget instead; a revoke spends nothing. + """ + if lease.result_recorded_at is not None: + return + policy = t.facts.policy + if lease.revoked: + why = EnumLandingOutcomeReason.REVOKED + elif t.now >= lease.deadline_at: + why = EnumLandingOutcomeReason.DEADLINE + else: + why = EnumLandingOutcomeReason.EXITED + t.recorded.append( + ModelLandingRecordedOutcome( + pr=lease.pr, + lease_id=lease.lease_id, + outcome=EnumLandingOutcome.TIMED_OUT, + reason=why, + ) + ) + rec = t.causes.get(lease.pr) + if rec is None: + return + update: dict[str, object] = {"outcome": EnumLandingOutcome.TIMED_OUT, "reason": why} + window = timedelta(seconds=policy.cause_spawn_failure_seconds) + if why is EnumLandingOutcomeReason.EXITED and t.now - lease.dispatched_at < window: + update["spawn_failures"] = rec.spawn_failures + 1 + elif why is not EnumLandingOutcomeReason.REVOKED: + update["attempts"] = rec.attempts + 1 + t.causes[lease.pr] = rec.model_copy(update=update) + + +def _read_cause_result( + t: _Tick, lease: ModelLandingLease, result: ModelLandingWorkerResult +) -> None: + """A cause worker's one result: recorded, verified, and one attempt spent.""" + key = lease.pr + kind = result.kind + outcome = EnumLandingOutcome.INVALID + reason = EnumLandingOutcomeReason.NONE + if kind is EnumLandingResultKind.CAUSE_FIX_SUBMITTED: + fix = t.prs.get(result.fix_ref) if result.fix_ref else None + verified = ( + fix is not None + and result.fix_head is not None + and ( + fix.state is EnumLandingPrState.MERGED + or ( + fix.state is EnumLandingPrState.OPEN + and fix.head_sha == result.fix_head + ) + ) + ) + if verified: + outcome = EnumLandingOutcome.CAUSE_FIX_SUBMITTED + else: + reason = EnumLandingOutcomeReason.UNVERIFIED_CLAIM + elif kind is EnumLandingResultKind.CAUSE_NOT_SHARED: + outcome = EnumLandingOutcome.CAUSE_NOT_SHARED + elif kind is EnumLandingResultKind.EXTERNAL_BLOCKER: + if result.blocker_kind is not None and result.blocker_ref: + outcome = EnumLandingOutcome.EXTERNAL_BLOCKER + else: + reason = EnumLandingOutcomeReason.BARE_BLOCKED + elif kind in _INVALID_REASON: + reason = _INVALID_REASON[kind] + else: # a per-PR result from a cause worker + reason = EnumLandingOutcomeReason.UNVERIFIED_CLAIM + t.leases[key] = lease.model_copy(update={"result_recorded_at": t.now}) + t.recorded.append( + ModelLandingRecordedOutcome( + pr=key, lease_id=lease.lease_id, outcome=outcome, reason=reason + ) + ) + if outcome is EnumLandingOutcome.INVALID: + t.violations.append( + ModelLandingViolation(pr=key, lease_id=lease.lease_id, reason=reason) + ) + rec = t.causes.get(key) + if rec is None: + return + update: dict[str, object] = { + "attempts": rec.attempts + 1, + "outcome": outcome, + "reason": reason, + } + if outcome is EnumLandingOutcome.CAUSE_FIX_SUBMITTED: + update.update(fix_ref=result.fix_ref, fix_head=result.fix_head, rerun_at=None) + elif outcome is EnumLandingOutcome.CAUSE_NOT_SHARED: + # back to the per-PR path at rung 0; the key stays blocked for this set + current = tuple( + ModelLandingMemberRef(pr=m.pr, head_sha=t.prs[m.pr].head_sha) + for m in rec.members + if m.pr in t.prs + ) + if current: + update["members"] = current + for m in current: + t.put_record( + m.pr, + ladder_index=0, + parked_head=None, + parked_fingerprint="", + awaiting_head=None, + ) + t.causes[key] = rec.model_copy(update=update) + + +def _parked(t: _Tick, rec: ModelLandingCauseRecord) -> bool: + return rec.parked_until is not None and t.now < rec.parked_until + + +def _leased_or_fixing(t: _Tick, rec: ModelLandingCauseRecord) -> bool: + leased = ( + rec.key in t.leases and rec.outcome is not EnumLandingOutcome.CAUSE_NOT_SHARED + ) + fixing = rec.outcome is EnumLandingOutcome.CAUSE_FIX_SUBMITTED + return leased or fixing + + +def _active(t: _Tick, rec: ModelLandingCauseRecord) -> bool: + """A live cause: a live lease, a fix PR in flight, or a park.""" + return _leased_or_fixing(t, rec) or _parked(t, rec) + + +def _holds_members(t: _Tick, rec: ModelLandingCauseRecord) -> bool: + """A cause holding its members off per-PR dispatch. + + A live lease, a fix PR in flight, or a park that an owner's CLAIM still + covers. A park alone does not: its members take the per-PR path. + """ + return _leased_or_fixing(t, rec) or (_parked(t, rec) and _owned(t, rec)) + + +def _blocks(t: _Tick, rec: ModelLandingCauseRecord) -> bool: + """cause_not_shared blocks its key until a member's head moves or it leaves.""" + if rec.outcome is not EnumLandingOutcome.CAUSE_NOT_SHARED: + return False + for m in rec.members: + p = t.prs.get(m.pr) + if p is None or p.state is not EnumLandingPrState.OPEN: + return False + if p.head_sha != m.head_sha: + return False + return True + + +def _owned(t: _Tick, rec: ModelLandingCauseRecord) -> bool: + """Owned by a live CLAIM naming the cause (by key, or by its repo and a check).""" + checks = {pair.check for pair in rec.pairs} + return any( + owner.cause == rec.key + or (owner.cause is None and owner.repo == rec.repo and owner.check in checks) + for owner in t.facts.cause_owners + ) + + +def _keep_cause(t: _Tick, rec: ModelLandingCauseRecord) -> bool: + return ( + rec.key in t.formed or rec.key in t.leases or _active(t, rec) or _blocks(t, rec) + ) + + +def _regraded(t: _Tick, rec: ModelLandingCauseRecord) -> bool: + """Every member rerun after the fix merged has graded again (or moved on).""" + assert rec.rerun_at is not None + for m in rec.members: + p = t.prs.get(m.pr) + if p is None or p.state is not EnumLandingPrState.OPEN: + continue + if ModelLandingMemberRef(pr=m.pr, head_sha=p.head_sha) not in rec.rerun_heads: + continue # its head moved: its own change, graded with the clusters + if p.ci is EnumLandingCi.PENDING: + return False + if not any( + run.head_sha == p.head_sha and run.created_at >= rec.rerun_at + for run in p.rerun_runs + ): + return False + return True + + +def _advance_causes(t: _Tick) -> None: + """Parks end; a merged fix reruns each member head once; a re-grade ends the attempt.""" + park = timedelta(hours=t.facts.policy.cause_park_hours) + for key in sorted(t.causes): + rec = t.causes[key] + if rec.parked_until is not None: + start = rec.parked_until - park + released = any( + r.cause == key and r.at >= start for r in t.facts.cause_releases + ) + if t.now >= rec.parked_until or released: + rec = rec.model_copy( + update={ + "parked_until": None, + "attempts": 0, + "spawn_failures": 0, + "outcome": None, + "reason": EnumLandingOutcomeReason.NONE, + } + ) + else: + t.degraded.append( + ModelLandingDegraded( + reason=EnumLandingDegradedReason.CAUSE_EXHAUSTED, subject=key + ) + ) + if rec.outcome is EnumLandingOutcome.CAUSE_FIX_SUBMITTED and rec.fix_ref: + fix = t.prs.get(rec.fix_ref) + ended = False + if fix is None or fix.state is EnumLandingPrState.OPEN: + pass # members wait on the fix PR (an unread fix PR fails closed) + elif fix.state is EnumLandingPrState.CLOSED: + ended = True # closed unmerged: the attempt failed + elif rec.rerun_at is None: + heads: list[ModelLandingMemberRef] = [] + for m in sorted(rec.members, key=lambda m: m.pr): + p = t.prs.get(m.pr) + if p is None or p.state is not EnumLandingPrState.OPEN: + continue + ref = ModelLandingMemberRef(pr=m.pr, head_sha=p.head_sha) + if ref in rec.rerun_heads: + continue + heads.append(ref) + t.emit( + ModelLandingAction( + kind=EnumLandingActionKind.RERUN, + subject=m.pr, + head_sha=p.head_sha, + ) + ) + rec = rec.model_copy( + update={ + "rerun_at": t.now, + "rerun_heads": (*rec.rerun_heads, *heads), + } + ) + else: + ended = _regraded(t, rec) + if ended: + rec = rec.model_copy( + update={ + "outcome": None, + "reason": EnumLandingOutcomeReason.NONE, + "rerun_at": None, + } + ) + t.causes[key] = rec + + +def _cause_candidate(t: _Tick, p: ModelLandingPrFacts) -> bool: + """A red PR that may join a cluster. + + A person's hold, a draft and do-not-land never join; a lane's CLAIM + (``owned``) does; a ``gate`` joins once it is older than the stall window. + A head whose one rerun is still unused reruns first. + """ + if ( + p.state is not EnumLandingPrState.OPEN + or p.ci is not EnumLandingCi.RED + or p.collaborator + ): + return False + if any(s in CAUSE_EXCLUDED_SUSPENSIONS for s in p.suspensions): + return False + if EnumLandingSuspension.GATE in p.suspensions: + stall = timedelta(minutes=t.facts.policy.wait_stall_minutes) + if p.gate_since is None or t.now - p.gate_since <= stall: + return False + return not ( + p.red_class in RERUNNABLE_RED_CLASSES + and p.head_sha not in t.record_of(p.pr).rerun_heads + ) + + +def _pairs_of(t: _Tick, p: ModelLandingPrFacts) -> list[tuple[str, str, str]]: + """The (check, signature, annotation) of each readable, clusterable red check.""" + excluded = set(t.facts.policy.cluster_excluded_checks) + out = [] + for check in sorted(set(p.red_checks)): + if check in excluded: + continue + text = p.red_annotations.get(check) + signature = normalize_signature(check, text) + if signature != UNREAD and text is not None: + out.append((check, signature, text)) + return out + + +@dataclass +class _Cause: + key: str + repo: str + pairs: list[ModelLandingCausePair] + members: set[str] + + +def _parked_pr(t: _Tick, p: ModelLandingPrFacts) -> bool: + rec = t.record_of(p.pr) + return ( + rec.parked_head is not None + and rec.parked_head == p.head_sha + and rec.parked_fingerprint == t.fingerprint(p.pr) + ) + + +def _form_causes(t: _Tick) -> None: + """Clusters, largest first with absorption, then escalated singletons; upserted by key.""" + policy = t.facts.policy + floor = set(t.facts.floor_breached_repos) + held_members = { + m.pr for rec in t.causes.values() if _active(t, rec) for m in rec.members + } + groups: dict[tuple[str, str, str], set[str]] = {} + examples: dict[tuple[str, str, str], str] = {} + for p in sorted(t.prs.values(), key=lambda p: p.pr): + if p.pr in held_members or not _cause_candidate(t, p): + continue + for check, signature, text in _pairs_of(t, p): + group = (repo_of(p.pr), check, signature) + groups.setdefault(group, set()).add(p.pr) + examples.setdefault(group, text) + + def need(repo: str) -> int: + if repo in floor: + return policy.cluster_min_members_floor + return policy.cluster_min_members + + clusters = sorted( + ((g, frozenset(m)) for g, m in groups.items() if len(m) >= need(g[0])), + key=lambda c: (-len(c[1]), c[0]), + ) + chosen: list[_Cause] = [] + taken: dict[str, str] = {} + for (repo, check, signature), members in clusters: + pair = ModelLandingCausePair( + check=check, + signature=signature, + example=examples[(repo, check, signature)], + members=len(members), + ) + host = next( + ( + c + for c in chosen + if c.repo == repo + and len(members & c.members) + >= policy.cluster_absorb_ratio * len(members) + ), + None, + ) + if host is not None: + host.pairs.append(pair) + for pr in sorted(members - taken.keys()): + host.members.add(pr) + taken[pr] = host.key + continue + free = set(members - taken.keys()) + if len(free) < need(repo): + continue + cause = _Cause(cause_key(repo, signature), repo, [pair], free) + chosen.append(cause) + taken.update(dict.fromkeys(free, cause.key)) + for p in sorted(t.prs.values(), key=lambda p: p.pr): + if ( + p.pr in taken + or p.pr in held_members + or p.state is not EnumLandingPrState.OPEN + or p.collaborator + or any(s in CAUSE_EXCLUDED_SUSPENSIONS for s in p.suspensions) + or not _parked_pr(t, p) + ): + continue + pairs = [ + ModelLandingCausePair(check=c, signature=s, example=x, members=1) + for c, s, x in _pairs_of(t, p) + ] + if not pairs: + continue # unread: no signature to key a cause by; it stays parked + cause = _Cause( + cause_key(repo_of(p.pr), pairs[0].signature), repo_of(p.pr), pairs, {p.pr} + ) + if any(c.key == cause.key for c in chosen): + continue + chosen.append(cause) + taken[p.pr] = cause.key + for cause in chosen: + _upsert_cause(t, cause) + + +def _upsert_cause(t: _Tick, cause: _Cause) -> None: + members = tuple( + ModelLandingMemberRef(pr=pr, head_sha=t.prs[pr].head_sha) + for pr in sorted(cause.members) + ) + rec = t.causes.get(cause.key) + if rec is not None and rec.outcome is EnumLandingOutcome.CAUSE_NOT_SHARED: + if _blocks(t, rec): + return # not shared for this member set: no cause, members stay per-PR + rec = None # a member moved: a fresh cause + if rec is None: + rec = ModelLandingCauseRecord( + key=cause.key, repo=cause.repo, pairs=tuple(cause.pairs), members=members + ) + elif _active(t, rec): # new PRs failing an active cause join it + known = {(p.check, p.signature) for p in rec.pairs} + pairs = ( + *rec.pairs, + *(p for p in cause.pairs if (p.check, p.signature) not in known), + ) + joined = {m.pr: m for m in rec.members} + for m in members: + joined.setdefault(m.pr, m) + rec = rec.model_copy( + update={ + "pairs": pairs, + "members": tuple(joined[pr] for pr in sorted(joined)), + } + ) + else: + rec = rec.model_copy(update={"pairs": tuple(cause.pairs), "members": members}) + t.causes[cause.key] = rec + t.formed.add(cause.key) + + +def _covering_escalation( + t: _Tick, rec: ModelLandingCauseRecord +) -> ModelLandingCauseEscalation | None: + """The ledger MSG that already escalates this cause's open park episode (LC-F3). + + A row covers the episode when it is younger than a park and no RELEASE row + naming the cause is dated at or after it. The newest such row wins, ties by + key, so the choice does not depend on input order. + """ + park = timedelta(hours=t.facts.policy.cause_park_hours) + rows = [ + r + for r in t.facts.cause_escalations + if r.cause == rec.key + and t.now < r.at + park + and not any( + rel.cause == rec.key and rel.at >= r.at for rel in t.facts.cause_releases + ) + ] + return max(rows, key=lambda r: (r.at, r.dedupe_key), default=None) + + +def _park(t: _Tick, rec: ModelLandingCauseRecord) -> None: + """A spent cause parks; one escalation per park episode, deduped on the ledger row. + + When the ledger already holds this episode's operator MSG (the controller + died between the append and the state write), the park adopts that row's + time and key and emits nothing: the state file only caches the answer. + """ + park = timedelta(hours=t.facts.policy.cause_park_hours) + row = _covering_escalation(t, rec) + if row is not None: + until = row.at + park + dedupe = row.dedupe_key + else: + until = t.now + park + dedupe = f"{rec.key}@{until.isoformat()}" + t.causes[rec.key] = rec.model_copy( + update={"parked_until": until, "escalated": dedupe} + ) + t.degraded.append( + ModelLandingDegraded( + reason=EnumLandingDegradedReason.CAUSE_EXHAUSTED, subject=rec.key + ) + ) + if row is None and rec.escalated != dedupe: + t.emit( + ModelLandingAction( + kind=EnumLandingActionKind.ESCALATE_OPERATOR, + subject=rec.key, + members=rec.members, + dedupe_key=dedupe, + ) + ) + + +def _try_dispatch_cause(t: _Tick, rec: ModelLandingCauseRecord) -> bool: + """One lease per cause key, under the cause caps and inside the pool (D5).""" + policy = t.facts.policy + key = rec.key + if key in t.leases or _parked(t, rec) or _active(t, rec) or _owned(t, rec): + return False + if rec.repo in t.draining or t.held(rec.repo): + return False + if ( + rec.attempts >= policy.cause_attempt_budget + or rec.spawn_failures >= policy.cause_spawn_failure_budget + ): + _park(t, rec) + return False + if t.facts.load1 > policy.load_pause_threshold: + return False + if len(t.leases) >= policy.max_workers: + return False + cause_leases = [s for s in t.leases if is_cause(s)] + if len(cause_leases) >= policy.max_cause_workers: + return False + if sum(1 for s in cause_leases if repo_of(s) == rec.repo) >= ( + policy.max_cause_workers_per_repo + ): + return False + if ( + sum(1 for s in t.leases if repo_of(s) == rec.repo) + >= policy.max_workers_per_repo + ): + return False + ladder = policy.cause_engine_ladder + available = set(t.facts.available_engines) + engine = next( + (e for e in ladder[min(rec.attempts, len(ladder) - 1) :] if e in available), + None, + ) + if engine is None: + return False + lease_id = t.next_lease_id + t.next_lease_id += 1 + deadline = t.now + timedelta(seconds=policy.cause_lease_seconds) + head = rec.members[0].head_sha + checks = tuple(sorted({pair.check for pair in rec.pairs})) + t.leases[key] = ModelLandingLease( + pr=key, + lease_id=lease_id, + brief_class=EnumLandingBriefClass.SHARED_CAUSE, + engine=engine, + dispatched_at=t.now, + deadline_at=deadline, + dispatch_head=head, + seen_heads=(head,), + last_seen_head=head, + dispatch_red_checks=checks, + ) + t.causes[key] = rec.model_copy( + update={"outcome": None, "reason": EnumLandingOutcomeReason.NONE} + ) + base = { + check + for repo_checks in t.facts.base_red_checks + if repo_checks.repo == rec.repo + for check in repo_checks.checks + } + t.emit( + ModelLandingAction( + kind=EnumLandingActionKind.DISPATCH_WORKER, + subject=key, + head_sha=head, + lease_id=lease_id, + members=rec.members, + cause_brief=ModelLandingCauseBrief( + cause=key, + repo=rec.repo, + pairs=rec.pairs, + members=rec.members, + base_failing_checks=tuple(c for c in checks if c in base), + attempt=rec.attempts + 1, + lease_id=lease_id, + engine=engine, + deadline_at=deadline, + instructions=BRIEF_INSTRUCTIONS[EnumLandingBriefClass.SHARED_CAUSE], + ), + ) + ) + return True + + +def _dispatch_causes(t: _Tick) -> None: + """Causes go before per-PR workers: floor-breached repos first, then size, then key.""" + floor = set(t.facts.floor_breached_repos) + order = sorted( + (t.causes[k] for k in t.formed), + key=lambda r: (r.repo not in floor, -len(r.members), r.key), + ) + for rec in order: + _try_dispatch_cause(t, t.causes[rec.key]) + for rec in t.causes.values(): + if _holds_members(t, rec) or (rec.key in t.formed and _owned(t, rec)): + t.suppressed.update(m.pr for m in rec.members) + + # ----------------------------------------------------------------------- PRs def _mergeable(p: ModelLandingPrFacts) -> bool: return p.ci is EnumLandingCi.GREEN and p.merge_state is EnumLandingMergeState.CLEAN @@ -933,6 +1630,8 @@ def _product_pr(t: _Tick, p: ModelLandingPrFacts) -> None: ) ) return + if pr in t.suppressed: # a member of a leased, fixing or owned parked cause + return if _brief_class(p, rec.ladder_index) is None: return if rec.awaiting_head is not None: @@ -985,6 +1684,8 @@ def decide_landing(facts: ModelLandingFacts) -> ModelLandingDecision: token=state.token_holder, next_lease_id=state.next_lease_id, close_requested=set(state.close_requested), + causes={c.key: c for c in state.causes}, + hold=set(facts.fixer_hold), ) holder = t.prs.get(t.token) if t.token is not None else None if t.token is not None and ( @@ -1005,6 +1706,9 @@ def decide_landing(facts: ModelLandingFacts) -> ModelLandingDecision: if comp.state is EnumLandingPrState.OPEN: _companion(t, comp) _cover(t) + _advance_causes(t) + _form_causes(t) + _dispatch_causes(t) for p in sorted(t.prs.values(), key=lambda p: _priority(t, p)): _product_pr(t, p) return _decision(t) @@ -1037,6 +1741,9 @@ def _decision(t: _Tick) -> ModelLandingDecision: ModelLandingEligibilityRerun(pr=pr, companion=c) for pr, c in reruns ), close_requested=tuple(sorted(t.close_next)), + causes=tuple( + t.causes[k] for k in sorted(t.causes) if _keep_cause(t, t.causes[k]) + ), ) return ModelLandingDecision( tick=t.facts.tick, @@ -1059,9 +1766,15 @@ def handle(self, request: ModelLandingFacts) -> ModelLandingDecision: __all__: list[str] = [ + "CAUSE_PREFIX", + "UNREAD", "HandlerPrLandingDecision", "blocker_fingerprint", + "cause_key", "decide_landing", + "is_cause", + "normalize_annotation", + "normalize_signature", "rebuild_key", "repo_of", ] diff --git a/src/omnimarket/nodes/node_pr_landing_decision_compute/models/enum_landing.py b/src/omnimarket/nodes/node_pr_landing_decision_compute/models/enum_landing.py index acd2af54a7..5580b32b82 100644 --- a/src/omnimarket/nodes/node_pr_landing_decision_compute/models/enum_landing.py +++ b/src/omnimarket/nodes/node_pr_landing_decision_compute/models/enum_landing.py @@ -83,19 +83,34 @@ class EnumLandingSuspension(StrEnum): GATE = "gate" DRAFT = "draft" DO_NOT_LAND = "do_not_land" + OWNED = "owned" + + +# A person's hold, a draft and do-not-land keep a PR out of every shared cause; +# ``owned`` (a lane's CLAIM on the PR) and a stalled ``gate`` do not. +CAUSE_EXCLUDED_SUSPENSIONS: frozenset[EnumLandingSuspension] = frozenset( + { + EnumLandingSuspension.HOLD, + EnumLandingSuspension.DRAFT, + EnumLandingSuspension.DO_NOT_LAND, + } +) class EnumLandingResultKind(StrEnum): """What a worker's one result file may say. - Only the first four are valid results (R1). The rest are recorded - ``invalid`` and escalate. + Only the first four are valid results of a per-PR worker (R1). A cause + worker's valid results are ``cause_fix_submitted``, ``cause_not_shared`` + and ``external_blocker``. The rest are recorded ``invalid`` and escalate. """ MERGED = "merged" ARMED = "armed" FIX_SUBMITTED = "fix_submitted" EXTERNAL_BLOCKER = "external_blocker" + CAUSE_FIX_SUBMITTED = "cause_fix_submitted" + CAUSE_NOT_SHARED = "cause_not_shared" WAITING_CI = "waiting_ci" WAITING_ORDER = "waiting_order" REPORT_ONLY = "report_only" @@ -108,7 +123,8 @@ class EnumLandingOutcome(StrEnum): The first six are the outcomes of a dispatch (P3: every dispatch reaches exactly one). ``blocked_on`` is recorded by the controller for a PR it did - not dispatch because a parent is open (R3). + not dispatch because a parent is open (R3). The two ``cause_`` outcomes + are the verified results of a cause worker. """ MERGED = "merged" @@ -118,6 +134,8 @@ class EnumLandingOutcome(StrEnum): INVALID = "invalid" TIMED_OUT = "timed_out" BLOCKED_ON = "blocked_on" + CAUSE_FIX_SUBMITTED = "cause_fix_submitted" + CAUSE_NOT_SHARED = "cause_not_shared" BLOCKED_OUTCOMES: frozenset[EnumLandingOutcome] = frozenset( @@ -196,13 +214,15 @@ class EnumLandingBriefClass(StrEnum): COMPANION_RED = "companion_red" COMPANION_ORPHAN = "companion_orphan" RUNTIME = "runtime" + SHARED_CAUSE = "shared_cause" class EnumLandingActionKind(StrEnum): """The controller's actions (the handover mapping's left column). ``kill_worker`` and ``discard_result`` act on the controller's own - workers; ``observe_only`` switches a drained repo to observe-only. + workers; ``observe_only`` switches a drained repo to observe-only; + ``escalate_operator`` asks the operator once about a parked cause. """ MERGE = "merge" @@ -215,6 +235,7 @@ class EnumLandingActionKind(StrEnum): KILL_WORKER = "kill_worker" DISCARD_RESULT = "discard_result" OBSERVE_ONLY = "observe_only" + ESCALATE_OPERATOR = "escalate_operator" class EnumLandingRebuildStatus(StrEnum): @@ -269,10 +290,12 @@ class EnumLandingDegradedReason(StrEnum): ESCALATION_EXHAUSTED = "escalation_exhausted" REBUILD_EXHAUSTED = "rebuild_exhausted" STALE_REFRESH_EXHAUSTED = "stale_refresh_exhausted" + CAUSE_EXHAUSTED = "cause_exhausted" __all__: list[str] = [ "BLOCKED_OUTCOMES", + "CAUSE_EXCLUDED_SUSPENSIONS", "CLAUDE_ENGINES", "RERUNNABLE_RED_CLASSES", "EnumLandingActionKind", diff --git a/src/omnimarket/nodes/node_pr_landing_decision_compute/models/model_landing_decision.py b/src/omnimarket/nodes/node_pr_landing_decision_compute/models/model_landing_decision.py index 4916dd0c1e..1f34060176 100644 --- a/src/omnimarket/nodes/node_pr_landing_decision_compute/models/model_landing_decision.py +++ b/src/omnimarket/nodes/node_pr_landing_decision_compute/models/model_landing_decision.py @@ -10,7 +10,9 @@ A worker brief is fixed per class: the text a worker receives is chosen by the class and never composed by a model. The brief names the four outcomes a -result may carry and the push rule its evidence is verified against. +result may carry and the push rule its evidence is verified against. A cause +brief is the same idea for one shared cause: fixed text, the cause's (check, +signature, example) pairs and its members, and the three results it may carry. """ from __future__ import annotations @@ -31,10 +33,13 @@ EnumLandingResultKind, ) from omnimarket.nodes.node_pr_landing_decision_compute.models.model_landing_state import ( + CAUSE_KEY_PATTERN, KEY_PATTERN, PR_KEY_PATTERN, REPO_PATTERN, SHA_PATTERN, + SUBJECT_PATTERN, + ModelLandingCausePair, ModelLandingControllerState, ModelLandingMemberRef, ) @@ -65,6 +70,28 @@ "write the ledger and do not choose other work." ) +CAUSE_RESULT_KINDS: Final[tuple[EnumLandingResultKind, ...]] = ( + EnumLandingResultKind.CAUSE_FIX_SUBMITTED, + EnumLandingResultKind.CAUSE_NOT_SHARED, + EnumLandingResultKind.EXTERNAL_BLOCKER, +) + +CAUSE_PUSH_RULE: Final[str] = ( + "Fix once at the source, in one pull request in the source repository. Never " + "push to a member PR's branch; list the members that need their own change " + "in the fix PR's body instead." +) + +CAUSE_RESULT_RULE: Final[str] = ( + "Write exactly one result file, with exactly one of: cause_fix_submitted" + "{fix_ref, fix_head} naming the one fix PR and its head; cause_not_shared" + "{reason} when the members only share an annotation by coincidence; " + "external_blocker{kind, ref} with kind one of operator_decision, " + "production_gate, hold_row, collaborator, upstream_open. Any other result is " + "invalid and spends the attempt. Do not write the ledger and do not choose " + "other work." +) + BRIEF_INSTRUCTIONS: Final[dict[EnumLandingBriefClass, str]] = { EnumLandingBriefClass.REAL_RED: ( "A check on this head is red for a product reason, or stayed red after its " @@ -102,6 +129,16 @@ "This runtime PR is red. Fix it and prove the fix on the lab runtime before " "pushing; name the lab readback in the result." ), + EnumLandingBriefClass.SHARED_CAUSE: ( + "These PRs in one repository fail the same check with the same failure " + "signature. In a detached worktree of the source repository at its base: " + "read one member's failing job log and the base head's run of the same " + "check; classify the source (the base head is red; the gate itself produces " + "the message for PRs that did nothing wrong; a sibling pin; change-control " + "companion contention, minimal unblock only; or not shared); fix it once at " + "the source; draft code through the code delegation step and text through " + "the landing text step." + ), } @@ -134,6 +171,43 @@ def _fixed_text(self) -> ModelLandingWorkerBrief: return self +class ModelLandingCauseBrief(BaseModel): + """The fixed brief one headless cause worker is dispatched with.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + brief_class: EnumLandingBriefClass = EnumLandingBriefClass.SHARED_CAUSE + cause: str = Field(..., pattern=CAUSE_KEY_PATTERN) + repo: str = Field(..., pattern=REPO_PATTERN) + pairs: tuple[ModelLandingCausePair, ...] = () + members: tuple[ModelLandingMemberRef, ...] = Field(..., min_length=1) + base_failing_checks: tuple[str, ...] = Field( + default=(), description="The cause's checks that also fail on the base head." + ) + attempt: int = Field(..., ge=1, description="This attempt's number in the episode.") + lease_id: int = Field(..., ge=1) + engine: EnumLandingEngine + deadline_at: datetime + allowed_results: tuple[EnumLandingResultKind, ...] = CAUSE_RESULT_KINDS + push_rule: str = CAUSE_PUSH_RULE + result_rule: str = CAUSE_RESULT_RULE + instructions: str = Field(..., min_length=1) + + @model_validator(mode="after") + def _fixed_text(self) -> ModelLandingCauseBrief: + if self.brief_class is not EnumLandingBriefClass.SHARED_CAUSE: + raise ValueError("a cause brief is the shared_cause class") + if self.instructions != BRIEF_INSTRUCTIONS[self.brief_class]: + raise ValueError("a brief's instructions are fixed by its class") + if ( + self.allowed_results != CAUSE_RESULT_KINDS + or self.push_rule != CAUSE_PUSH_RULE + or self.result_rule != CAUSE_RESULT_RULE + ): + raise ValueError("a cause brief's result and push rules are fixed") + return self + + class ModelLandingAction(BaseModel): """One controller action. ``subject`` is the PR, companion or repo it acts on.""" @@ -152,14 +226,25 @@ class ModelLandingAction(BaseModel): members: tuple[ModelLandingMemberRef, ...] = () replacement: str | None = Field(default=None, pattern=PR_KEY_PATTERN) brief: ModelLandingWorkerBrief | None = None + cause_brief: ModelLandingCauseBrief | None = None + dedupe_key: str | None = Field( + default=None, + description="escalate_operator: @, one per park.", + ) @model_validator(mode="after") def _shape(self) -> ModelLandingAction: kind = self.kind if kind is EnumLandingActionKind.MERGE and self.head_sha is None: raise ValueError("merge is always pinned to a head") - if kind is EnumLandingActionKind.DISPATCH_WORKER and self.brief is None: - raise ValueError("dispatch_worker carries its brief") + if kind is EnumLandingActionKind.DISPATCH_WORKER and ( + (self.brief is None) == (self.cause_brief is None) + ): + raise ValueError("dispatch_worker carries exactly one brief") + if kind is EnumLandingActionKind.ESCALATE_OPERATOR and ( + self.dedupe_key is None or not self.members + ): + raise ValueError("escalate_operator carries its dedupe key and members") if kind is EnumLandingActionKind.COMPANION_REBUILD and ( self.rebuild_key is None or not self.members or self.target_repo is None ): @@ -174,11 +259,14 @@ def _shape(self) -> ModelLandingAction: class ModelLandingRecordedOutcome(BaseModel): - """An outcome this tick recorded (for the ledger row and the scorecard).""" + """An outcome this tick recorded (for the ledger row and the scorecard). + + ``pr`` is the subject: a PR, or a cause key for a cause worker's outcome. + """ model_config = ConfigDict(frozen=True, extra="forbid") - pr: str = Field(..., pattern=PR_KEY_PATTERN) + pr: str = Field(..., pattern=SUBJECT_PATTERN) lease_id: int | None = Field(default=None, ge=1) outcome: EnumLandingOutcome reason: EnumLandingOutcomeReason = EnumLandingOutcomeReason.NONE @@ -199,7 +287,7 @@ class ModelLandingViolation(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") - pr: str = Field(..., pattern=PR_KEY_PATTERN) + pr: str = Field(..., pattern=SUBJECT_PATTERN) lease_id: int = Field(..., ge=1) reason: EnumLandingOutcomeReason @@ -236,10 +324,14 @@ class ModelLandingDecision(BaseModel): __all__: list[str] = [ "BRIEF_INSTRUCTIONS", + "CAUSE_PUSH_RULE", + "CAUSE_RESULT_KINDS", + "CAUSE_RESULT_RULE", "PUSH_RULE", "RESULT_RULE", "VALID_RESULT_KINDS", "ModelLandingAction", + "ModelLandingCauseBrief", "ModelLandingCompanionVerdictRow", "ModelLandingDecision", "ModelLandingDegraded", diff --git a/src/omnimarket/nodes/node_pr_landing_decision_compute/models/model_landing_facts.py b/src/omnimarket/nodes/node_pr_landing_decision_compute/models/model_landing_facts.py index bde7d481c2..a6b867d52b 100644 --- a/src/omnimarket/nodes/node_pr_landing_decision_compute/models/model_landing_facts.py +++ b/src/omnimarket/nodes/node_pr_landing_decision_compute/models/model_landing_facts.py @@ -8,6 +8,11 @@ controller's own state as it last wrote it. The decision itself does no I/O and reads no clock: ``observed_at`` is the moment the snapshot describes, and deadlines and grace periods are compared against it. + +The shared-cause inputs (each red check's failure annotation, when a gate +began, the fixer HOLD rows, the floor alarm's breach set, the CLAIMs that name +a cause, RELEASE rows naming a parked cause) are all optional with empty +defaults, so a snapshot without them decides exactly as before. """ from __future__ import annotations @@ -30,14 +35,18 @@ EnumLandingSuspension, ) from omnimarket.nodes.node_pr_landing_decision_compute.models.model_landing_state import ( + CAUSE_KEY_PATTERN, KEY_PATTERN, PR_KEY_PATTERN, REPO_PATTERN, SHA_PATTERN, + SUBJECT_PATTERN, ModelLandingControllerState, ModelLandingMemberRef, ) +FIXER_HOLD_ALL = "all" + class ModelLandingBlockerRef(BaseModel): """One ref that can block a PR, with the state it was read in (R3, R8). @@ -125,6 +134,17 @@ class ModelLandingPrFacts(BaseModel): pattern=SHA_PATTERN, description="The head auto-merge is armed on, if armed.", ) + red_annotations: dict[str, str] = Field( + default_factory=dict, + description=( + "Red check name -> its failure annotation text on this head; a red " + "check absent here is unread and never clusters." + ), + ) + gate_since: datetime | None = Field( + default=None, + description="When the current gate suspension began on this head.", + ) @model_validator(mode="after") def _red_needs_class(self) -> ModelLandingPrFacts: @@ -187,7 +207,9 @@ class ModelLandingWorkerResult(BaseModel): lease_id: int = Field(..., ge=1) pr: str = Field( - ..., pattern=PR_KEY_PATTERN, description="The PR the worker was sent to." + ..., + pattern=SUBJECT_PATTERN, + description="The PR, or the cause key, the worker was sent to.", ) kind: EnumLandingResultKind head_sha: str | None = Field(default=None, pattern=SHA_PATTERN) @@ -195,6 +217,71 @@ class ModelLandingWorkerResult(BaseModel): rerun_run_id: int | None = Field(default=None, ge=1) blocker_kind: EnumLandingExternalBlockerKind | None = None blocker_ref: str | None = None + fix_ref: str | None = Field( + default=None, + pattern=PR_KEY_PATTERN, + description="cause_fix_submitted: the one fix PR at the source.", + ) + fix_head: str | None = Field( + default=None, pattern=SHA_PATTERN, description="cause_fix_submitted: its head." + ) + reason: str | None = Field(default=None, description="cause_not_shared: why.") + + +class ModelLandingCauseOwner(BaseModel): + """A live CLAIM that names a cause (never a per-PR CLAIM). + + It owns the cause whose key is ``cause``, or, with no key, the cause in + ``repo`` that carries ``check``. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + lane: str = Field(..., min_length=1) + repo: str = Field(..., pattern=REPO_PATTERN) + check: str | None = Field(default=None, min_length=1) + cause: str | None = Field(default=None, pattern=CAUSE_KEY_PATTERN) + + @model_validator(mode="after") + def _names_a_cause(self) -> ModelLandingCauseOwner: + if self.check is None and self.cause is None: + raise ValueError("a cause owner names the cause key or its check") + return self + + +class ModelLandingCauseRelease(BaseModel): + """A RELEASE row naming a parked cause; it ends a park that began before it.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + cause: str = Field(..., pattern=CAUSE_KEY_PATTERN) + at: datetime + + +class ModelLandingCauseEscalation(BaseModel): + """An operator MSG row already on the ledger for a parked cause. + + The row carries the dedupe key ``@``. The ledger + row, not the controller's state file, is the source of truth for "this park + episode was escalated": a crash between the MSG append and the state write + loses the state file's copy of the key and not the row (model finding LC-F3, + omnibase_internal#144). + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + cause: str = Field(..., pattern=CAUSE_KEY_PATTERN) + at: datetime + dedupe_key: str = Field(..., min_length=1) + + +class ModelLandingRepoChecks(BaseModel): + """The checks red on one repo's base head.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + repo: str = Field(..., pattern=REPO_PATTERN) + checks: tuple[str, ...] = () class ModelLandingRebuildAck(BaseModel): @@ -241,6 +328,51 @@ class ModelLandingPolicy(BaseModel): ), min_length=1, ) + cluster_min_members: int = Field( + default=3, ge=2, description="PRs sharing one (check, signature) to cluster." + ) + cluster_min_members_floor: int = Field( + default=2, ge=2, description="The same, in a repo under its landing floor." + ) + wait_stall_minutes: int = Field( + default=60, ge=0, description="A gate older than this joins clusters." + ) + cluster_absorb_ratio: float = Field( + default=0.8, + gt=0, + le=1, + description="A cluster this much inside a chosen cause is absorbed into it.", + ) + cluster_excluded_checks: tuple[str, ...] = Field( + default=( + "CI Summary", + "Hostile Reviewer (adversarial gate)", + "Hostile Review Gate", + ), + description="Follower checks and the reviewer pool: never clustered.", + ) + max_cause_workers: int = Field(default=4, ge=0, description="Fleet-wide.") + max_cause_workers_per_repo: int = Field(default=2, ge=0) + max_workers_per_repo: int = Field( + default=6, ge=0, description="Workers of any kind in one repo." + ) + cause_attempt_budget: int = Field( + default=2, ge=1, description="Attempts per cause per park episode." + ) + cause_spawn_failure_budget: int = Field( + default=3, ge=1, description="Exits with no result that park the cause." + ) + cause_spawn_failure_seconds: int = Field( + default=600, + ge=0, + description="An exit with no result this soon after dispatch is not an attempt.", + ) + cause_lease_seconds: int = Field(default=5400, ge=1) + cause_park_hours: int = Field(default=12, ge=1) + cause_engine_ladder: tuple[EnumLandingEngine, ...] = Field( + default=(EnumLandingEngine.CLAUDE_OPUS, EnumLandingEngine.CLAUDE_OPUS), + min_length=1, + ) class ModelLandingFacts(BaseModel): @@ -276,6 +408,28 @@ class ModelLandingFacts(BaseModel): EnumLandingEngine.CLAUDE_OPUS, EnumLandingEngine.CODEX_HIGH, ) + fixer_hold: tuple[str, ...] = Field( + default=(), + description=( + "Repos under a fixer HOLD row, or 'all': no worker is dispatched " + "there and every live lease there is revoked." + ), + ) + floor_breached_repos: tuple[str, ...] = Field( + default=(), description="Repos under their landing floor (the floor alarm)." + ) + cause_owners: tuple[ModelLandingCauseOwner, ...] = () + cause_releases: tuple[ModelLandingCauseRelease, ...] = () + cause_escalations: tuple[ModelLandingCauseEscalation, ...] = Field( + default=(), + description=( + "Operator MSG rows already on the ledger for a parked cause (LC-F3): " + "a park whose episode a row covers emits no second escalation." + ), + ) + base_red_checks: tuple[ModelLandingRepoChecks, ...] = Field( + default=(), description="Checks red on each repo's base head." + ) @model_validator(mode="after") def _keys_unique(self) -> ModelLandingFacts: @@ -290,6 +444,14 @@ def _keys_unique(self) -> ModelLandingFacts: for repo in self.drain_requested: if not _is_repo(repo): raise ValueError(f"drain_requested entry {repo!r} is not owner/name") + for repo in self.floor_breached_repos: + if not _is_repo(repo): + raise ValueError( + f"floor_breached_repos entry {repo!r} is not owner/name" + ) + for scope in self.fixer_hold: + if scope != FIXER_HOLD_ALL and not _is_repo(scope): + raise ValueError(f"fixer_hold entry {scope!r} is not owner/name or all") return self @@ -298,7 +460,10 @@ def _is_repo(value: str) -> bool: __all__: list[str] = [ + "FIXER_HOLD_ALL", "ModelLandingBlockerRef", + "ModelLandingCauseOwner", + "ModelLandingCauseRelease", "ModelLandingCompanionFacts", "ModelLandingFacts", "ModelLandingPolicy", @@ -306,6 +471,7 @@ def _is_repo(value: str) -> bool: "ModelLandingPushEvidence", "ModelLandingRebuildAck", "ModelLandingRefUpdate", + "ModelLandingRepoChecks", "ModelLandingRerunRun", "ModelLandingWorkerProbe", "ModelLandingWorkerResult", diff --git a/src/omnimarket/nodes/node_pr_landing_decision_compute/models/model_landing_state.py b/src/omnimarket/nodes/node_pr_landing_decision_compute/models/model_landing_state.py index f9d2658c09..35c687e398 100644 --- a/src/omnimarket/nodes/node_pr_landing_decision_compute/models/model_landing_state.py +++ b/src/omnimarket/nodes/node_pr_landing_decision_compute/models/model_landing_state.py @@ -6,7 +6,9 @@ state it last wrote in with the facts, and writes the ``next_state`` of the decision back before it performs any action (write-ahead): a crash after the write loses actions, never records. Every record is keyed by ``repo#pr``, and -a head is always a field, never part of a key (R7). +a head is always a field, never part of a key (R7). A shared cause is keyed by +its cause key, ``cause:/:``, and its lease lives in the +same lease space as a PR's. """ from __future__ import annotations @@ -27,6 +29,12 @@ REPO_PATTERN = r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$" SHA_PATTERN = r"^[0-9a-f]{40}$" KEY_PATTERN = r"^[0-9a-f]{64}$" +SIGNATURE_PATTERN = r"^[0-9a-f]{12}$" +CAUSE_KEY_PATTERN = r"^cause:[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+:[0-9a-f]{12}$" +SUBJECT_PATTERN = ( + r"^(?:[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+#[1-9][0-9]*" + r"|cause:[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+:[0-9a-f]{12})$" +) class ModelLandingMemberRef(BaseModel): @@ -39,11 +47,15 @@ class ModelLandingMemberRef(BaseModel): class ModelLandingLease(BaseModel): - """The one lease of one PR (R7). Released only by confirmed termination.""" + """The one lease of one PR or one cause (R7). Released only by confirmed termination. + + ``pr`` is the subject: ``repo#pr``, or a cause key for a cause worker, + whose dispatch head is the head of its first member. + """ model_config = ConfigDict(frozen=True, extra="forbid") - pr: str = Field(..., pattern=PR_KEY_PATTERN) + pr: str = Field(..., pattern=SUBJECT_PATTERN) lease_id: int = Field(..., ge=1) brief_class: EnumLandingBriefClass engine: EnumLandingEngine @@ -139,6 +151,52 @@ class ModelLandingEligibilityRerun(BaseModel): companion: str = Field(..., pattern=PR_KEY_PATTERN) +class ModelLandingCausePair(BaseModel): + """One (check, signature) a shared cause carries, with one example annotation.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + check: str = Field(..., min_length=1) + signature: str = Field(..., pattern=SIGNATURE_PATTERN) + example: str = Field( + default="", description="One member's annotation text for this pair." + ) + members: int = Field(default=0, ge=0, description="PRs sharing the pair.") + + +class ModelLandingCauseRecord(BaseModel): + """Everything the controller remembers about one shared cause between ticks. + + Upserted by cause key. ``attempts`` and ``spawn_failures`` count within one + park episode; a park ends at ``parked_until`` or on a RELEASE row naming + the cause, and the next episode starts from zero. ``escalated`` is the + dedupe key of the one operator escalation of the current park. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + key: str = Field(..., pattern=CAUSE_KEY_PATTERN) + repo: str = Field(..., pattern=REPO_PATTERN) + pairs: tuple[ModelLandingCausePair, ...] = () + members: tuple[ModelLandingMemberRef, ...] = Field(..., min_length=1) + attempts: int = Field(default=0, ge=0) + spawn_failures: int = Field(default=0, ge=0) + outcome: EnumLandingOutcome | None = None + reason: EnumLandingOutcomeReason = EnumLandingOutcomeReason.NONE + fix_ref: str | None = Field(default=None, pattern=PR_KEY_PATTERN) + fix_head: str | None = Field(default=None, pattern=SHA_PATTERN) + rerun_at: datetime | None = Field( + default=None, description="When the reruns after the fix merged were emitted." + ) + rerun_heads: tuple[ModelLandingMemberRef, ...] = Field( + default=(), description="Member heads already rerun for this cause." + ) + parked_until: datetime | None = None + escalated: str | None = Field( + default=None, description="@ of the last escalation." + ) + + class ModelLandingControllerState(BaseModel): """The controller's state file.""" @@ -156,13 +214,19 @@ class ModelLandingControllerState(BaseModel): uncovered: tuple[ModelLandingUncovered, ...] = () eligibility_reruns: tuple[ModelLandingEligibilityRerun, ...] = () close_requested: tuple[str, ...] = () + causes: tuple[ModelLandingCauseRecord, ...] = () __all__: list[str] = [ + "CAUSE_KEY_PATTERN", "KEY_PATTERN", "PR_KEY_PATTERN", "REPO_PATTERN", "SHA_PATTERN", + "SIGNATURE_PATTERN", + "SUBJECT_PATTERN", + "ModelLandingCausePair", + "ModelLandingCauseRecord", "ModelLandingControllerState", "ModelLandingEligibilityRerun", "ModelLandingLease", diff --git a/tests/unit/nodes/node_pr_landing_decision_compute/fixtures/replay_2026_10_03_omnimarket.json b/tests/unit/nodes/node_pr_landing_decision_compute/fixtures/replay_2026_10_03_omnimarket.json new file mode 100644 index 0000000000..71748d9f59 --- /dev/null +++ b/tests/unit/nodes/node_pr_landing_decision_compute/fixtures/replay_2026_10_03_omnimarket.json @@ -0,0 +1,1752 @@ +{ + "facts": { + "available_engines": [ + "claude_sonnet", + "claude_opus" + ], + "companions": [], + "drain_requested": [], + "load1": 9.669921875, + "observed_at": "2026-10-03T21:00:21Z", + "policy": { + "engine_ladder": [ + "claude_sonnet", + "claude_opus", + "claude_opus" + ], + "lease_seconds": 2700, + "load_pause_threshold": 1000000.0, + "max_workers": 18 + }, + "probes": [ + { + "group_alive": true, + "lease_id": 2251, + "tagged_alive": true + }, + { + "group_alive": true, + "lease_id": 2252, + "tagged_alive": true + }, + { + "group_alive": true, + "lease_id": 2253, + "tagged_alive": true + }, + { + "group_alive": true, + "lease_id": 2246, + "tagged_alive": true + }, + { + "group_alive": true, + "lease_id": 2250, + "tagged_alive": true + } + ], + "producer_failures": [], + "producer_release": "none", + "prs": [ + { + "auto_merge_head": null, + "blocker_refs": [ + { + "head_sha": "3a1beb08992004c653336cbb4bf8039d20364b58", + "ref": "self:omnimarket#3183", + "state": "head" + } + ], + "ci": "red", + "collaborator": true, + "created_at": "2026-10-01T12:17:37Z", + "head_sha": "3a1beb08992004c653336cbb4bf8039d20364b58", + "merge_state": "clean", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3183", + "process_fix": false, + "red_annotations": {}, + "red_checks": [ + "occ-autobind / outcome" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [] + }, + { + "auto_merge_head": "886f28a1a28cbfa211ebb43ccb98a9b33eb1c070", + "blocker_refs": [ + { + "head_sha": "886f28a1a28cbfa211ebb43ccb98a9b33eb1c070", + "ref": "self:omnimarket#3293", + "state": "head" + }, + { + "head_sha": null, + "ref": "hold:b9845bbcf08de9f9", + "state": "active" + }, + { + "head_sha": null, + "ref": "external:no-open-pr: omnibase_infra dev vendors node_projection_delegation migrations only through 0052; no open omnibase_infra PR vendors 0053_delegation_events_finish_reason_truncated.sql", + "state": "named" + }, + { + "head_sha": null, + "ref": "external:omnimarket delegation-regression-nightly run 37107272993", + "state": "named" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T06:42:19Z", + "head_sha": "886f28a1a28cbfa211ebb43ccb98a9b33eb1c070", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3293", + "process_fix": false, + "red_annotations": { + "Release Identity Gate": "Process completed with exit code 1." + }, + "red_checks": [ + "CI Summary", + "Release Identity Gate", + "Sibling Release Compatibility Gate (OMN-20381)", + "lint" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [ + "hold" + ] + }, + { + "auto_merge_head": null, + "blocker_refs": [ + { + "head_sha": "aff62b7b217dcaa35dade0c68eb415d1f742c8e5", + "ref": "self:omnimarket#3303", + "state": "head" + }, + { + "head_sha": null, + "ref": "hold:02042a47327db4eb", + "state": "active" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T07:43:09Z", + "head_sha": "aff62b7b217dcaa35dade0c68eb415d1f742c8e5", + "merge_state": "conflicting", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3303", + "process_fix": false, + "red_annotations": { + "Publish occ-autobind command": "Process completed with exit code 1." + }, + "red_checks": [ + "CI Summary", + "Publish occ-autobind command" + ], + "red_class": "cascade", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [ + "hold" + ] + }, + { + "auto_merge_head": "0ad953a750020a549bfd791f8d1f45da5ae82cc1", + "blocker_refs": [ + { + "head_sha": "0ad953a750020a549bfd791f8d1f45da5ae82cc1", + "ref": "self:omnimarket#3308", + "state": "head" + }, + { + "head_sha": "6e765a62ef3d3967f684d409565acb882ce4bad4", + "ref": "OmniNode-ai/onex_change_control#12574", + "state": "OPEN" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T10:36:46Z", + "gate_since": "2026-10-03T14:51:14Z", + "head_sha": "0ad953a750020a549bfd791f8d1f45da5ae82cc1", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3308", + "process_fix": false, + "red_annotations": { + "OCC Companion Merged Gate (OMN-15214)": "Process completed with exit code 1.\nocc-companion-merged gate: poll deadline (1500s) reached while still PENDING \u2014 failing closed. Last state: companion OCC#12574 is still OPEN \u2014 the companion must MERGE before this product PR may merge (OMN-15214). Land the companion on onex_change_control, then re-run this job.", + "OCC Preflight Dependency": "Process completed with exit code 1.\nOCC preflight concluded failure; refusing to start heavy CI.", + "call-reject-skip-token / occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12574 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1517s; failing closed", + "deploy-gate / deploy-gate": "Process completed with exit code 1.\nDEPLOY GATE FAILED: PR body has Evidence-Source but is missing Evidence-Ticket: OMN-XXXX. Add Evidence-Ticket so deploy-gate can validate the contract at the pinned OCC source.", + "occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12574 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1508s; failing closed", + "repo-evidence / dod-verify": "Process completed with exit code 1.\nOMN-17427 [dod-receipt-runner-gh-token-ac1]: bound test also passes at the merge base: the control did not fail (always-pass)", + "verify / verify": "Process completed with exit code 1.\nRECEIPT GATE FAILED: PR body contains 'Evidence-Source' but is missing an 'Evidence-Ticket: OMN-XXXX' line. Every PR with Evidence-Source must also declare Evidence-Ticket so the gate can verify the evidence is for the same ticket as the PR. Add 'Evidence-Ticket: OMN-XXXX' to the PR body." + }, + "red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Preflight Dependency", + "call-reject-skip-token / occ-preflight / eligibility", + "deploy-gate / deploy-gate", + "occ-preflight / eligibility", + "repo-evidence / dod-verify", + "verify / verify" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [ + "gate" + ] + }, + { + "auto_merge_head": "928df65150a8ffe0e2b61c637f502b6532b6c897", + "blocker_refs": [ + { + "head_sha": "928df65150a8ffe0e2b61c637f502b6532b6c897", + "ref": "self:omnimarket#3309", + "state": "head" + }, + { + "head_sha": "9eaa8f1a541139ffbaa336b0d215dbefc8852be7", + "ref": "OmniNode-ai/onex_change_control#12584", + "state": "OPEN" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T10:43:58Z", + "gate_since": "2026-10-03T18:18:33Z", + "head_sha": "928df65150a8ffe0e2b61c637f502b6532b6c897", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3309", + "process_fix": false, + "red_annotations": { + "OCC Attestation Observe (report-only, non-blocking)": "Process completed with exit code 1.", + "OCC Companion Merged Gate (OMN-15214)": "Process completed with exit code 1.\nocc-companion-merged gate: poll deadline (1500s) reached while still PENDING \u2014 failing closed. Last state: companion OCC#12584 is still OPEN \u2014 the companion must MERGE before this product PR may merge (OMN-15214). Land the companion on onex_change_control, then re-run this job.", + "OCC Preflight Dependency": "Process completed with exit code 1.\nOCC preflight concluded failure; refusing to start heavy CI.", + "call-reject-skip-token / occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12584 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1526s; failing closed", + "occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12584 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1518s; failing closed", + "repo-evidence / dod-verify": "Process completed with exit code 1.\nOMN-17427 [dod-omn17427-ac1-one-saving, dod-omn17427-ac2-minted-tenant]: bound test also passes at the merge base: the control did not fail (always-pass)" + }, + "red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Attestation Observe (report-only, non-blocking)", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Preflight Dependency", + "call-reject-skip-token / occ-preflight / eligibility", + "occ-preflight / eligibility", + "repo-evidence / dod-verify" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [ + "gate" + ] + }, + { + "auto_merge_head": null, + "blocker_refs": [ + { + "head_sha": "67043097374f2c97f24c89c0d99021324157bfea", + "ref": "self:omnimarket#3319", + "state": "head" + } + ], + "ci": "green", + "collaborator": true, + "created_at": "2026-10-03T17:03:11Z", + "head_sha": "67043097374f2c97f24c89c0d99021324157bfea", + "merge_state": "clean", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3319", + "process_fix": false, + "red_annotations": {}, + "red_checks": [], + "red_class": null, + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [ + "gate" + ] + }, + { + "auto_merge_head": "1e178ee1290dd1043d39215cb0b9778a092b976b", + "blocker_refs": [ + { + "head_sha": "1e178ee1290dd1043d39215cb0b9778a092b976b", + "ref": "self:omnimarket#3320", + "state": "head" + }, + { + "head_sha": "2144ba6333816fd8946f4b0e20366f69cebd4fec", + "ref": "OmniNode-ai/onex_change_control#12616", + "state": "OPEN" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T17:08:47Z", + "gate_since": "2026-10-03T17:34:38Z", + "head_sha": "1e178ee1290dd1043d39215cb0b9778a092b976b", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3320", + "process_fix": false, + "red_annotations": { + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)": "Process completed with exit code 1.", + "OCC Companion Observe (dry_run, non-blocking)": "Process completed with exit code 1.", + "OCC Preflight Dependency": "Process completed with exit code 1.\nOCC preflight concluded failure; refusing to start heavy CI.", + "Publish occ-autobind command": "Process completed with exit code 1.", + "call-reject-skip-token / occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12616 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1516s; failing closed", + "occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12616 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1515s; failing closed", + "repo-evidence / dod-verify": "Process completed with exit code 1.\nOMN-17427 [dod-receipt-runner-gh-token-ac1, dod-schema-fault-readiness, dod-required-bar-ac3, dod-required-bar-ac4]: bound test also passes at the merge base: the control did not fail (always-pass)", + "verify / verify": "Process completed with exit code 1." + }, + "red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Companion Observe (dry_run, non-blocking)", + "OCC Preflight Dependency", + "Publish occ-autobind command", + "call-reject-skip-token / occ-preflight / eligibility", + "occ-preflight / eligibility", + "repo-evidence / dod-verify", + "verify / verify" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [ + "gate" + ] + }, + { + "auto_merge_head": "db37112647d10deaf290860aa68e0269fe912e20", + "blocker_refs": [ + { + "head_sha": "db37112647d10deaf290860aa68e0269fe912e20", + "ref": "self:omnimarket#3321", + "state": "head" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T17:28:51Z", + "head_sha": "db37112647d10deaf290860aa68e0269fe912e20", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3321", + "process_fix": false, + "red_annotations": { + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)": "Process completed with exit code 1.", + "OCC Companion Observe (dry_run, non-blocking)": "Process completed with exit code 1.", + "OCC Preflight Dependency": "Process completed with exit code 1.\nOCC preflight concluded failure; refusing to start heavy CI.", + "call-reject-skip-token / occ-preflight / eligibility": "Process completed with exit code 1.\nPR body has no evidence-source stamp line yet (the occ-autobind mint may still be in flight) -- poll deadline (1500s) reached after 1506s; failing closed", + "occ-preflight / eligibility": "Process completed with exit code 1.\nthe occ-autobind producer TERMINALLY declined to mint a companion for this head SHA: skip:DEFER_HAND_AUTHORED \u2014 OmniNode-ai/omnimarket#3321: OCC#12592 (hand_authored, operator/omn-17427-occ-companion-omnibase-infra-4516) carries evidence for OMN-17427; OCC#12611 (hand_authored, evidence/omn-17427-omnibase-infra-4519) carries evidence for OMN-17427; OCC#12624 (hand_authored, evidence/OMN-17427-omnimarket-3324) carries evidence for OMN-17427; OCC#12623 (hand_authored, evidence/OMN-17427-omnibase_infra-4523) carries evidence for OMN-17427; OCC#12574 (hand_authored, evidence/OMN-17427-omnimarket-3308) carries evidence for OMN-17427; OCC#12595 (hand_authored, operator/omn-17427-companion-core-1869) carries evidence for OMN-17427; OCC#12584 (hand_authored, operator/omn-17427-occ-companion-omnimarket-3309) carries evidence for OMN-17427; OCC#12622 (hand_authored, evidence/OMN-17427-omnibase_infra-4524) carries evidence for OMN-17427; OCC#12616 (hand_authored, evidence/OMN-17427-omnimarket-3320) carries evidence for OMN-17427 (OMN-15247) | OCC companion NOT verified: no OCC companion verifier wired; fail-closed (cannot prove the companion was pushed). No later poll changes this verdict, so this gate stops now rather than spending its remaining budget. A hand-authored OCC companion on the OMN-15247 path is the designed remedy and is SAFE to author now -- the producer has stood down for this head SHA, so there is no in-flight mint to collide with (contrast OMN-18881). It must be signed by a different lane than the one that authored the ticket, and a new commit re-opens this gate with a fresh verdict (OMN-18647)", + "repo-evidence / dod-verify": "Process completed with exit code 1.\nOMN-17427 [dod-receipt-runner-gh-token-ac1, dod-schema-fault-readiness, dod-required-bar-ac3, dod-required-bar-ac4]: bound test also passes at the merge base: the control did not fail (always-pass)", + "verify / verify": "Process completed with exit code 1.\nRECEIPT GATE FAILED: PR body is missing required 'Evidence-Source:' line (OMN-10419). Add one of: 'Evidence-Source: OCC#' or 'Evidence-Source: '" + }, + "red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Companion Observe (dry_run, non-blocking)", + "OCC Preflight Dependency", + "call-reject-skip-token / occ-preflight / eligibility", + "occ-preflight / eligibility", + "repo-evidence / dod-verify", + "verify / verify" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [] + }, + { + "auto_merge_head": "3e6f0721e1ecf94f2704e5c0e80d77080dc86f19", + "blocker_refs": [ + { + "head_sha": "3e6f0721e1ecf94f2704e5c0e80d77080dc86f19", + "ref": "self:omnimarket#3322", + "state": "head" + }, + { + "head_sha": "de8c7741df8a1d014a58f0f7adfabf7307081b7c", + "ref": "OmniNode-ai/onex_change_control#12630", + "state": "OPEN" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T17:55:28Z", + "gate_since": "2026-10-03T18:20:55Z", + "head_sha": "3e6f0721e1ecf94f2704e5c0e80d77080dc86f19", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3322", + "process_fix": false, + "red_annotations": { + "OCC Preflight Dependency": "Process completed with exit code 1.\nOCC preflight concluded failure; refusing to start heavy CI.", + "call-reject-skip-token / occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12630 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1507s; failing closed", + "occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12630 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1515s; failing closed", + "repo-evidence / dod-verify": "Process completed with exit code 1.\nOMN-17427 [dod-receipt-runner-gh-token-ac1, dod-schema-fault-readiness, dod-required-bar-ac3, dod-required-bar-ac4]: bound test also passes at the merge base: the control did not fail (always-pass)", + "verify / verify": "Process completed with exit code 1.\nIDENTITY BINDING FAILED: neither the branch name nor any commit message references OMN-17427. branch='test/p6-task8-delegation-handler-coverage', Evidence-Ticket='OMN-17427', commit messages checked=1. Name the branch for the ticket cited in Evidence-Ticket, or \u2014 when the ticket was filed after the branch already existed \u2014 add a commit to this PR whose commit message references that ticket." + }, + "red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Preflight Dependency", + "call-reject-skip-token / occ-preflight / eligibility", + "occ-preflight / eligibility", + "repo-evidence / dod-verify", + "verify / verify" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [ + "gate" + ] + }, + { + "auto_merge_head": "969ce712802ed459170847351d8174890555ae78", + "blocker_refs": [ + { + "head_sha": "969ce712802ed459170847351d8174890555ae78", + "ref": "self:omnimarket#3323", + "state": "head" + }, + { + "head_sha": "f23b5434be2a25d407a2ed4c7df980fc60831f70", + "ref": "OmniNode-ai/onex_change_control#12625", + "state": "OPEN" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T17:57:59Z", + "gate_since": "2026-10-03T17:59:04Z", + "head_sha": "969ce712802ed459170847351d8174890555ae78", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3323", + "process_fix": false, + "red_annotations": { + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)": "Process completed with exit code 1.", + "OCC Companion Observe (dry_run, non-blocking)": "Process completed with exit code 1.", + "OCC Preflight Dependency": "Process completed with exit code 1.\nOCC preflight concluded failure; refusing to start heavy CI.", + "call-reject-skip-token / occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12625 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1518s; failing closed", + "occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12625 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1523s; failing closed", + "repo-evidence / dod-verify": "Process completed with exit code 1.\nOMN-17427 [dod-receipt-runner-gh-token-ac1, dod-schema-fault-readiness, dod-required-bar-ac3, dod-required-bar-ac4]: bound test also passes at the merge base: the control did not fail (always-pass)" + }, + "red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Companion Observe (dry_run, non-blocking)", + "OCC Preflight Dependency", + "call-reject-skip-token / occ-preflight / eligibility", + "occ-preflight / eligibility", + "repo-evidence / dod-verify" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [ + "gate" + ] + }, + { + "auto_merge_head": "15739a17a384af08087861824b9990514d64cd1e", + "blocker_refs": [ + { + "head_sha": "15739a17a384af08087861824b9990514d64cd1e", + "ref": "self:omnimarket#3324", + "state": "head" + }, + { + "head_sha": "ae8d86ff3fef4551a0133a5fd4ec37f57a8b6bec", + "ref": "OmniNode-ai/onex_change_control#12624", + "state": "OPEN" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T18:01:38Z", + "gate_since": "2026-10-03T18:02:37Z", + "head_sha": "15739a17a384af08087861824b9990514d64cd1e", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3324", + "process_fix": false, + "red_annotations": { + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)": "Process completed with exit code 1.", + "OCC Companion Observe (dry_run, non-blocking)": "Process completed with exit code 1.", + "OCC Preflight Dependency": "Process completed with exit code 1.\nOCC preflight concluded failure; refusing to start heavy CI.", + "call-reject-skip-token / occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12624 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1515s; failing closed", + "occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12624 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1516s; failing closed", + "repo-evidence / dod-verify": "Process completed with exit code 1.\nOMN-17427 [dod-receipt-runner-gh-token-ac1, dod-schema-fault-readiness, dod-required-bar-ac3, dod-required-bar-ac4]: bound test also passes at the merge base: the control did not fail (always-pass)" + }, + "red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Companion Observe (dry_run, non-blocking)", + "OCC Preflight Dependency", + "call-reject-skip-token / occ-preflight / eligibility", + "occ-preflight / eligibility", + "repo-evidence / dod-verify" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [ + "gate" + ] + }, + { + "auto_merge_head": "844b306c8f193081e8370d58dab4aa178a705b18", + "blocker_refs": [ + { + "head_sha": "844b306c8f193081e8370d58dab4aa178a705b18", + "ref": "self:omnimarket#3325", + "state": "head" + }, + { + "head_sha": "7fef379abe1e18931d0f576df744c31687e3b960", + "ref": "OmniNode-ai/onex_change_control#12626", + "state": "OPEN" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T18:03:46Z", + "gate_since": "2026-10-03T18:04:42Z", + "head_sha": "844b306c8f193081e8370d58dab4aa178a705b18", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3325", + "process_fix": false, + "red_annotations": { + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)": "Process completed with exit code 1.", + "OCC Companion Observe (dry_run, non-blocking)": "Process completed with exit code 1.", + "OCC Preflight Dependency": "Process completed with exit code 1.\nOCC preflight concluded failure; refusing to start heavy CI.", + "call-reject-skip-token / occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12626 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1510s; failing closed", + "occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12626 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1509s; failing closed", + "repo-evidence / dod-verify": "Process completed with exit code 1.\nOMN-17427 [dod-receipt-runner-gh-token-ac1, dod-schema-fault-readiness, dod-required-bar-ac3, dod-required-bar-ac4]: bound test also passes at the merge base: the control did not fail (always-pass)", + "verify / verify": "Process completed with exit code 1." + }, + "red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Companion Observe (dry_run, non-blocking)", + "OCC Preflight Dependency", + "call-reject-skip-token / occ-preflight / eligibility", + "occ-preflight / eligibility", + "repo-evidence / dod-verify", + "verify / verify" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [ + "gate" + ] + }, + { + "auto_merge_head": "916b9cde467ad58a478abe56ef6dd34024f8c78e", + "blocker_refs": [ + { + "head_sha": "916b9cde467ad58a478abe56ef6dd34024f8c78e", + "ref": "self:omnimarket#3326", + "state": "head" + }, + { + "head_sha": "fb0b9855ecd5e451895ca3d6336bfaf25f18666d", + "ref": "OmniNode-ai/onex_change_control#12635", + "state": "OPEN" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T18:17:09Z", + "gate_since": "2026-10-03T18:18:03Z", + "head_sha": "916b9cde467ad58a478abe56ef6dd34024f8c78e", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3326", + "process_fix": false, + "red_annotations": { + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)": "Process completed with exit code 1.", + "OCC Companion Observe (dry_run, non-blocking)": "Process completed with exit code 1.", + "OCC Preflight Dependency": "Process completed with exit code 1.\nOCC preflight concluded failure; refusing to start heavy CI.", + "call-reject-skip-token / occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12635 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1517s; failing closed", + "occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12635 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1509s; failing closed", + "repo-evidence / dod-verify": "Process completed with exit code 1.\nOMN-17427 [dod-omn17427-ac1-one-saving, dod-omn17427-ac2-minted-tenant]: bound test also passes at the merge base: the control did not fail (always-pass)" + }, + "red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Companion Observe (dry_run, non-blocking)", + "OCC Preflight Dependency", + "call-reject-skip-token / occ-preflight / eligibility", + "occ-preflight / eligibility", + "repo-evidence / dod-verify" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [ + "gate" + ] + }, + { + "auto_merge_head": "34b532229511a3d449dd3b705ae9f7711a781a4d", + "blocker_refs": [ + { + "head_sha": "34b532229511a3d449dd3b705ae9f7711a781a4d", + "ref": "self:omnimarket#3327", + "state": "head" + }, + { + "head_sha": "7066dcecab4635def62f462056405acc16fac020", + "ref": "OmniNode-ai/onex_change_control#12634", + "state": "OPEN" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T18:19:52Z", + "gate_since": "2026-10-03T18:45:17Z", + "head_sha": "34b532229511a3d449dd3b705ae9f7711a781a4d", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3327", + "process_fix": false, + "red_annotations": { + "OCC Preflight Dependency": "Process completed with exit code 1.\nOCC preflight concluded failure; refusing to start this workflow.", + "call-reject-skip-token / occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12634 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1514s; failing closed", + "occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12634 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1524s; failing closed", + "repo-evidence / dod-verify": "Process completed with exit code 1.\nOMN-17427 [dod-omn17427-ac1-one-saving, dod-omn17427-ac2-minted-tenant]: bound test also passes at the merge base: the control did not fail (always-pass)" + }, + "red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Preflight Dependency", + "call-reject-skip-token / occ-preflight / eligibility", + "occ-preflight / eligibility", + "repo-evidence / dod-verify" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [ + "gate" + ] + }, + { + "auto_merge_head": "8f2186647379836770b9892e35ab4cac5bb4d823", + "blocker_refs": [ + { + "head_sha": "8f2186647379836770b9892e35ab4cac5bb4d823", + "ref": "self:omnimarket#3328", + "state": "head" + }, + { + "head_sha": "34b532229511a3d449dd3b705ae9f7711a781a4d", + "ref": "OmniNode-ai/omnimarket#3327", + "state": "OPEN" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T18:24:31Z", + "head_sha": "8f2186647379836770b9892e35ab4cac5bb4d823", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3328", + "process_fix": false, + "red_annotations": { + "OCC Preflight Dependency": "Process completed with exit code 1.\nOCC preflight concluded failure; refusing to start heavy CI.", + "call-reject-skip-token / occ-preflight / eligibility": "Process completed with exit code 1.\nPR body has no evidence-source stamp line yet (the occ-autobind mint may still be in flight) -- poll deadline (1500s) reached after 1506s; failing closed", + "occ-preflight / eligibility": "Process completed with exit code 1.\nthe occ-autobind producer TERMINALLY declined to mint a companion for this head SHA: skip:DEFER_HAND_AUTHORED \u2014 OmniNode-ai/omnimarket#3328: OCC#12635 (hand_authored, evidence/OMN-17427-omnimarket-3326) carries evidence for OMN-17427; OCC#12592 (hand_authored, operator/omn-17427-occ-companion-omnibase-infra-4516) carries evidence for OMN-17427; OCC#12625 (hand_authored, evidence/OMN-17427-omnimarket-3323) carries evidence for OMN-17427; OCC#12611 (hand_authored, evidence/omn-17427-omnibase-infra-4519) carries evidence for OMN-17427; OCC#12624 (hand_authored, evidence/OMN-17427-omnimarket-3324) carries evidence for OMN-17427; OCC#12616 (hand_authored, evidence/OMN-17427-omnimarket-3320) carries evidence for OMN-17427; OCC#12595 (hand_authored, operator/omn-17427-companion-core-1869) carries evidence for OMN-17427; OCC#12574 (hand_authored, evidence/OMN-17427-omnimarket-3308) carries evidence for OMN-17427; OCC#12629 (hand_authored, operator/omn-17427-occ-companion-omnibase-infra-4525) carries evidence for OMN-17427 (OMN-15247) | OCC companion NOT verified: no OCC companion verifier wired; fail-closed (cannot prove the companion was pushed). No later poll changes this verdict, so this gate stops now rather than spending its remaining budget. A hand-authored OCC companion on the OMN-15247 path is the designed remedy and is SAFE to author now -- the producer has stood down for this head SHA, so there is no in-flight mint to collide with (contrast OMN-18881). It must be signed by a different lane than the one that authored the ticket, and a new commit re-opens this gate with a fresh verdict (OMN-18647)", + "repo-evidence / dod-verify": "Process completed with exit code 1.\nOMN-17427 [dod-omn17427-ac1-one-saving, dod-omn17427-ac2-minted-tenant]: bound test also passes at the merge base: the control did not fail (always-pass)", + "verify / verify": "Process completed with exit code 1.\nRECEIPT GATE FAILED: PR body is missing required 'Evidence-Source:' line (OMN-10419). Add one of: 'Evidence-Source: OCC#' or 'Evidence-Source: '" + }, + "red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Preflight Dependency", + "call-reject-skip-token / occ-preflight / eligibility", + "occ-preflight / eligibility", + "repo-evidence / dod-verify", + "verify / verify" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [] + }, + { + "auto_merge_head": "b9f9a38c5dda0540a59af458475f0258e3801d15", + "blocker_refs": [ + { + "head_sha": "b9f9a38c5dda0540a59af458475f0258e3801d15", + "ref": "self:omnimarket#3330", + "state": "head" + }, + { + "head_sha": "e9eaffc1b326af36534ac00d54a16d2c4394f21c", + "ref": "OmniNode-ai/onex_change_control#12642", + "state": "OPEN" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T18:59:50Z", + "gate_since": "2026-10-03T19:01:18Z", + "head_sha": "b9f9a38c5dda0540a59af458475f0258e3801d15", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3330", + "process_fix": false, + "red_annotations": { + "OCC Companion Observe (dry_run, non-blocking)": "Process completed with exit code 1.", + "OCC Preflight Dependency": "Process completed with exit code 1.\nOCC preflight concluded failure; refusing to start heavy CI.", + "call-reject-skip-token / occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12642 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1516s; failing closed", + "occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12642 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1509s; failing closed", + "repo-evidence / dod-verify": "Process completed with exit code 1.\nOMN-17427 [dod-omn17427-ac1-one-saving, dod-omn17427-ac2-minted-tenant]: bound test also passes at the merge base: the control did not fail (always-pass)" + }, + "red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Companion Observe (dry_run, non-blocking)", + "OCC Preflight Dependency", + "call-reject-skip-token / occ-preflight / eligibility", + "occ-preflight / eligibility", + "repo-evidence / dod-verify" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [ + "gate" + ] + }, + { + "auto_merge_head": "81eda89b1ec087fd9e0a53c53445f3056bcdcc9b", + "blocker_refs": [ + { + "head_sha": "81eda89b1ec087fd9e0a53c53445f3056bcdcc9b", + "ref": "self:omnimarket#3332", + "state": "head" + }, + { + "head_sha": "b7275ec3e63abce68930e227d1a1965e43761b1f", + "ref": "OmniNode-ai/onex_change_control#12649", + "state": "OPEN" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T19:14:34Z", + "gate_since": "2026-10-03T19:16:11Z", + "head_sha": "81eda89b1ec087fd9e0a53c53445f3056bcdcc9b", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3332", + "process_fix": false, + "red_annotations": { + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)": "Process completed with exit code 1.", + "OCC Companion Observe (dry_run, non-blocking)": "Process completed with exit code 1.", + "OCC Preflight Dependency": "Process completed with exit code 1.\nOCC preflight concluded failure; refusing to start heavy CI.", + "call-reject-skip-token / occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12649 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1509s; failing closed", + "repo-evidence / dod-verify": "Process completed with exit code 1.\nOMN-17427 [dod-omn17427-ac1-one-saving, dod-omn17427-ac2-minted-tenant]: bound test also passes at the merge base: the control did not fail (always-pass)" + }, + "red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Companion Observe (dry_run, non-blocking)", + "OCC Preflight Dependency", + "call-reject-skip-token / occ-preflight / eligibility", + "repo-evidence / dod-verify" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [ + "gate" + ] + }, + { + "auto_merge_head": "eab52e8bad39e88a03493ab175a7c07750e36a56", + "blocker_refs": [ + { + "head_sha": "eab52e8bad39e88a03493ab175a7c07750e36a56", + "ref": "self:omnimarket#3333", + "state": "head" + }, + { + "head_sha": "961554b227aaa7de6493a46d6dac4df274efacab", + "ref": "OmniNode-ai/onex_change_control#12651", + "state": "OPEN" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T19:22:20Z", + "gate_since": "2026-10-03T19:24:22Z", + "head_sha": "eab52e8bad39e88a03493ab175a7c07750e36a56", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3333", + "process_fix": false, + "red_annotations": { + "OCC Preflight Dependency": "Process completed with exit code 1.\nOCC preflight concluded failure; refusing to start heavy CI.", + "call-reject-skip-token / occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12651 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1513s; failing closed", + "occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12651 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1514s; failing closed", + "repo-evidence / dod-verify": "Process completed with exit code 1.\nno test-side change: the bound tests cannot be shown to fail at the merge base" + }, + "red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Preflight Dependency", + "call-reject-skip-token / occ-preflight / eligibility", + "occ-preflight / eligibility", + "repo-evidence / dod-verify" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [ + "gate" + ] + }, + { + "auto_merge_head": "212c0ee2b16bcc404aa3176bb0c879931a494858", + "blocker_refs": [ + { + "head_sha": "212c0ee2b16bcc404aa3176bb0c879931a494858", + "ref": "self:omnimarket#3334", + "state": "head" + }, + { + "head_sha": "29a0899c1eda30a80f143d9b48ee4495fe874779", + "ref": "OmniNode-ai/onex_change_control#12648", + "state": "OPEN" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T19:26:30Z", + "gate_since": "2026-10-03T19:28:07Z", + "head_sha": "212c0ee2b16bcc404aa3176bb0c879931a494858", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3334", + "process_fix": false, + "red_annotations": { + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)": "Process completed with exit code 1.", + "OCC Companion Observe (dry_run, non-blocking)": "Process completed with exit code 1.", + "OCC Preflight Dependency": "Process completed with exit code 1.\nOCC preflight concluded failure; refusing to start heavy CI.", + "call-reject-skip-token / occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12648 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1510s; failing closed", + "occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12648 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1525s; failing closed", + "repo-evidence / dod-verify": "Process completed with exit code 1.\nOMN-17427 [dod-omn17427-ac1-one-saving, dod-omn17427-ac2-minted-tenant]: bound test also passes at the merge base: the control did not fail (always-pass)", + "verify / verify": "Process completed with exit code 1." + }, + "red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Companion Observe (dry_run, non-blocking)", + "OCC Preflight Dependency", + "call-reject-skip-token / occ-preflight / eligibility", + "occ-preflight / eligibility", + "repo-evidence / dod-verify", + "verify / verify" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [ + "gate" + ] + }, + { + "auto_merge_head": "987730ddb6961f2895a04e11c049ec2069658a5c", + "blocker_refs": [ + { + "head_sha": "987730ddb6961f2895a04e11c049ec2069658a5c", + "ref": "self:omnimarket#3335", + "state": "head" + }, + { + "head_sha": "c23ca809dada9dc56c2988aaca8a74236937b7da", + "ref": "OmniNode-ai/onex_change_control#12652", + "state": "OPEN" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T19:26:37Z", + "gate_since": "2026-10-03T19:29:46Z", + "head_sha": "987730ddb6961f2895a04e11c049ec2069658a5c", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3335", + "process_fix": false, + "red_annotations": { + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)": "Process completed with exit code 1.", + "OCC Companion Observe (dry_run, non-blocking)": "Process completed with exit code 1.", + "OCC Preflight Dependency": "Process completed with exit code 1.\nOCC preflight concluded failure; refusing to start heavy CI.", + "call-reject-skip-token / occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12652 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1524s; failing closed", + "repo-evidence / dod-verify": "Process completed with exit code 1.\nOMN-17427 [dod-omn17427-ac1-one-saving, dod-omn17427-ac2-minted-tenant]: bound test also passes at the merge base: the control did not fail (always-pass)" + }, + "red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Companion Observe (dry_run, non-blocking)", + "OCC Preflight Dependency", + "call-reject-skip-token / occ-preflight / eligibility", + "repo-evidence / dod-verify" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [ + "gate" + ] + }, + { + "auto_merge_head": "9d172cfdd651c26a3fca876f82897c2735cb59fe", + "blocker_refs": [ + { + "head_sha": "9d172cfdd651c26a3fca876f82897c2735cb59fe", + "ref": "self:omnimarket#3337", + "state": "head" + }, + { + "head_sha": "3433077450c0c4ad0c9128e698ca68641c0e93f8", + "ref": "OmniNode-ai/onex_change_control#12647", + "state": "OPEN" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T19:32:49Z", + "gate_since": "2026-10-03T19:34:55Z", + "head_sha": "9d172cfdd651c26a3fca876f82897c2735cb59fe", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3337", + "process_fix": false, + "red_annotations": { + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)": "Process completed with exit code 1.", + "OCC Companion Observe (dry_run, non-blocking)": "Process completed with exit code 1.", + "OCC Preflight Dependency": "Process completed with exit code 1.\nOCC preflight concluded failure; refusing to start heavy CI.", + "call-reject-skip-token / occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12647 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1510s; failing closed", + "occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12647 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1510s; failing closed", + "repo-evidence / dod-verify": "Process completed with exit code 1.\nOMN-17427 [dod-omn17427-ac1-one-saving, dod-omn17427-ac2-minted-tenant]: bound test also passes at the merge base: the control did not fail (always-pass)" + }, + "red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Companion Observe (dry_run, non-blocking)", + "OCC Preflight Dependency", + "call-reject-skip-token / occ-preflight / eligibility", + "occ-preflight / eligibility", + "repo-evidence / dod-verify" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [ + "gate" + ] + }, + { + "auto_merge_head": "3356b1b7b1833580b03aea737673a89f70afde64", + "blocker_refs": [ + { + "head_sha": "3356b1b7b1833580b03aea737673a89f70afde64", + "ref": "self:omnimarket#3338", + "state": "head" + }, + { + "head_sha": "551a057c48112b347a188689e1987503280180a1", + "ref": "OmniNode-ai/onex_change_control#12650", + "state": "OPEN" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T19:42:25Z", + "head_sha": "3356b1b7b1833580b03aea737673a89f70afde64", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3338", + "process_fix": false, + "red_annotations": { + "OCC Preflight Dependency": "Process completed with exit code 1.\nOCC preflight concluded failure; refusing to start this workflow.", + "call-reject-skip-token / occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12650 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1518s; failing closed", + "occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12650 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1516s; failing closed", + "product-readiness / evaluate": "Process completed with exit code 1.", + "reason-graph": "Process completed with exit code 1.\nROOT PRODUCT_FAILED \u2014 product defect (enforcing; NON-required, does not block merge)", + "repo-evidence / dod-verify": "Process completed with exit code 1.\nOMN-17427 [dod-omn17427-ac1-one-saving, dod-omn17427-ac2-minted-tenant]: bound test also passes at the merge base: the control did not fail (always-pass)", + "tests+coverage (shadow)": "Process completed with exit code 1." + }, + "red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Preflight Dependency", + "call-reject-skip-token / occ-preflight / eligibility", + "occ-preflight / eligibility", + "product-readiness / evaluate", + "reason-graph", + "repo-evidence / dod-verify", + "tests+coverage (shadow)" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [] + }, + { + "auto_merge_head": "1497a0498501b5edfcc7657a6716cc5501246c67", + "blocker_refs": [ + { + "head_sha": "1497a0498501b5edfcc7657a6716cc5501246c67", + "ref": "self:omnimarket#3339", + "state": "head" + }, + { + "head_sha": "cd2f4d6328314810cf88144c65f27a980c228c3b", + "ref": "OmniNode-ai/onex_change_control#12655", + "state": "OPEN" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T19:48:41Z", + "gate_since": "2026-10-03T19:50:46Z", + "head_sha": "1497a0498501b5edfcc7657a6716cc5501246c67", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3339", + "process_fix": false, + "red_annotations": { + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)": "Process completed with exit code 1.", + "OCC Companion Observe (dry_run, non-blocking)": "Process completed with exit code 1.", + "OCC Preflight Dependency": "Process completed with exit code 1.\nOCC preflight concluded failure; refusing to start heavy CI.", + "call-reject-skip-token / occ-preflight / eligibility": "Process completed with exit code 1.\ncompanion OCC#12655 is still OPEN -- it must merge before this PR may merge -- poll deadline (1500s) reached after 1509s; failing closed", + "repo-evidence / dod-verify": "Process completed with exit code 1.\nOMN-17427 [dod-omn17427-ac1-one-saving, dod-omn17427-ac2-minted-tenant]: bound test also passes at the merge base: the control did not fail (always-pass)" + }, + "red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Companion Observe (dry_run, non-blocking)", + "OCC Preflight Dependency", + "call-reject-skip-token / occ-preflight / eligibility", + "repo-evidence / dod-verify" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [ + "gate" + ] + }, + { + "auto_merge_head": "15506bb9bf82f9986a870229f2d2ea8b1a04e648", + "blocker_refs": [ + { + "head_sha": "15506bb9bf82f9986a870229f2d2ea8b1a04e648", + "ref": "self:omnimarket#3340", + "state": "head" + }, + { + "head_sha": "cbfe9db57a902d5aa18e5db98b7ecf38f361721d", + "ref": "OmniNode-ai/onex_change_control#12656", + "state": "OPEN" + } + ], + "ci": "red", + "collaborator": false, + "created_at": "2026-10-03T20:09:46Z", + "gate_since": "2026-10-03T20:12:19Z", + "head_sha": "15506bb9bf82f9986a870229f2d2ea8b1a04e648", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3340", + "process_fix": false, + "red_annotations": { + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)": "Process completed with exit code 1.", + "OCC Companion Observe (dry_run, non-blocking)": "Process completed with exit code 1.", + "OCC Preflight Dependency": "Process completed with exit code 1.\nOCC preflight concluded failure; refusing to start heavy CI.", + "repo-evidence / dod-verify": "Process completed with exit code 1.\nOMN-17427 [dod-omn17427-ac1-one-saving, dod-omn17427-ac2-minted-tenant]: bound test also passes at the merge base: the control did not fail (always-pass)" + }, + "red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Companion Author (mode from OMNI_OCC_AUTOAUTHOR_MODE, default dry_run)", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Companion Observe (dry_run, non-blocking)", + "OCC Preflight Dependency", + "repo-evidence / dod-verify" + ], + "red_class": "product", + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [ + "gate" + ] + }, + { + "auto_merge_head": "aea29853114d5a70f7ae34fb7488f0262a0a14dc", + "blocker_refs": [ + { + "head_sha": "aea29853114d5a70f7ae34fb7488f0262a0a14dc", + "ref": "self:omnimarket#3341", + "state": "head" + } + ], + "ci": "pending", + "collaborator": false, + "created_at": "2026-10-03T20:10:22Z", + "head_sha": "aea29853114d5a70f7ae34fb7488f0262a0a14dc", + "merge_state": "unknown", + "open_parents": [], + "parents": [], + "pr": "OmniNode-ai/omnimarket#3341", + "process_fix": false, + "red_annotations": {}, + "red_checks": [], + "red_class": null, + "ref_updates": [], + "rerun_runs": [], + "runtime": true, + "state": "open", + "suspensions": [] + } + ], + "rebuild_acks": [], + "results": [], + "state": { + "close_requested": [], + "draining_repos": [], + "eligibility_reruns": [], + "last_tick": 724, + "leases": [ + { + "brief_class": "runtime", + "deadline_at": "2026-10-03T21:32:59Z", + "dispatch_head": "298b64b4ab21676ff9712de3cae8e10faefbfb9e", + "dispatch_red_checks": [ + "runner-image-build-smoke" + ], + "dispatched_at": "2026-10-03T20:47:59Z", + "engine": "claude_sonnet", + "kill_sent_tick": null, + "last_seen_head": "298b64b4ab21676ff9712de3cae8e10faefbfb9e", + "lease_id": 2251, + "pr": "OmniNode-ai/omnibase_infra#4534", + "result_recorded_at": null, + "revoked": false, + "seen_heads": [ + "298b64b4ab21676ff9712de3cae8e10faefbfb9e" + ], + "stuck": false + }, + { + "brief_class": "runtime", + "deadline_at": "2026-10-03T21:41:21Z", + "dispatch_head": "3356b1b7b1833580b03aea737673a89f70afde64", + "dispatch_red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Preflight Dependency", + "call-reject-skip-token / occ-preflight / eligibility", + "occ-preflight / eligibility", + "product-readiness / evaluate", + "reason-graph", + "repo-evidence / dod-verify", + "tests+coverage (shadow)" + ], + "dispatched_at": "2026-10-03T20:56:21Z", + "engine": "claude_sonnet", + "kill_sent_tick": null, + "last_seen_head": "3356b1b7b1833580b03aea737673a89f70afde64", + "lease_id": 2252, + "pr": "OmniNode-ai/omnimarket#3338", + "result_recorded_at": null, + "revoked": false, + "seen_heads": [ + "3356b1b7b1833580b03aea737673a89f70afde64" + ], + "stuck": false + }, + { + "brief_class": "conflict_mechanical", + "deadline_at": "2026-10-03T21:41:21Z", + "dispatch_head": "c3aabd2c36df1589ad3adf53fc128f763f508f84", + "dispatch_red_checks": [], + "dispatched_at": "2026-10-03T20:56:21Z", + "engine": "claude_sonnet", + "kill_sent_tick": null, + "last_seen_head": "c3aabd2c36df1589ad3adf53fc128f763f508f84", + "lease_id": 2253, + "pr": "OmniNode-ai/onex_change_control#12643", + "result_recorded_at": null, + "revoked": false, + "seen_heads": [ + "c3aabd2c36df1589ad3adf53fc128f763f508f84" + ], + "stuck": false + }, + { + "brief_class": "real_red", + "deadline_at": "2026-10-03T21:21:14Z", + "dispatch_head": "3433077450c0c4ad0c9128e698ca68641c0e93f8", + "dispatch_red_checks": [], + "dispatched_at": "2026-10-03T20:36:14Z", + "engine": "claude_opus", + "kill_sent_tick": null, + "last_seen_head": "3433077450c0c4ad0c9128e698ca68641c0e93f8", + "lease_id": 2246, + "pr": "OmniNode-ai/onex_change_control#12647", + "result_recorded_at": null, + "revoked": false, + "seen_heads": [ + "3433077450c0c4ad0c9128e698ca68641c0e93f8" + ], + "stuck": false + }, + { + "brief_class": "real_red", + "deadline_at": "2026-10-03T21:32:59Z", + "dispatch_head": "9c9013f456a640f376a4f5bace110a45d8db4ff8", + "dispatch_red_checks": [ + "CI Summary", + "Pre-commit" + ], + "dispatched_at": "2026-10-03T20:47:59Z", + "engine": "claude_sonnet", + "kill_sent_tick": null, + "last_seen_head": "cbfe9db57a902d5aa18e5db98b7ecf38f361721d", + "lease_id": 2250, + "pr": "OmniNode-ai/onex_change_control#12656", + "result_recorded_at": null, + "revoked": false, + "seen_heads": [ + "9c9013f456a640f376a4f5bace110a45d8db4ff8", + "cbfe9db57a902d5aa18e5db98b7ecf38f361721d" + ], + "stuck": false + } + ], + "next_lease_id": 2254, + "observe_only_repos": [], + "rebuilds": [], + "records": [ + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "", + "ladder_index": 0, + "outcome": null, + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnibase_infra#4534", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [ + "call-reject-skip-token / scan / reject-skip-gate-token", + "repo-evidence / dod-verify" + ], + "awaiting_head": null, + "blocker_fingerprint": "2bf0814c21829bd726dd01a0eb1307c7c432b334b2711801ac06453556df28f4", + "ladder_index": 0, + "outcome": "external_blocker", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3293", + "reason": "none", + "rerun_heads": [ + "a636f30f7fb25d6e9b729b3ee870f6a9878e8048" + ], + "update_heads": [] + }, + { + "attempt_red_checks": [ + "Publish occ-autobind command" + ], + "awaiting_head": "aff62b7b217dcaa35dade0c68eb415d1f742c8e5", + "blocker_fingerprint": "83578f0a93aa5e317eb430752a9c8a7cb48f63225fb00b2f23b0eed5be499186", + "ladder_index": 0, + "outcome": "fix_submitted", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3303", + "reason": "none", + "rerun_heads": [ + "aff62b7b217dcaa35dade0c68eb415d1f742c8e5" + ], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "526bd091fec127e957ff9879d11014b8d1431fc23acc0b9ad4169ce0ced867ea", + "ladder_index": 0, + "outcome": "external_blocker", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3308", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [ + "CI Summary", + "Coverage Sweep Gate", + "OCC Companion Merged Gate (OMN-15214)", + "OCC Preflight Dependency", + "call-reject-skip-token / occ-preflight / eligibility", + "occ-preflight / eligibility" + ], + "awaiting_head": "928df65150a8ffe0e2b61c637f502b6532b6c897", + "blocker_fingerprint": "c02567375faa278a65c17c0f01e1155c03fe9fae2cbc5fb9f761c01076d5f007", + "ladder_index": 0, + "outcome": "fix_submitted", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3309", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "25bca50ef61cbf5e41bb658061af415e684ee46e3a8837c6edb61b8cd0645f47", + "ladder_index": 0, + "outcome": "external_blocker", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3320", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "c06b3813b5360762c0aff48b96415ba48ffc31997a410504666772b6f1a24994", + "ladder_index": 3, + "outcome": "timed_out", + "parked_fingerprint": "c06b3813b5360762c0aff48b96415ba48ffc31997a410504666772b6f1a24994", + "parked_head": "db37112647d10deaf290860aa68e0269fe912e20", + "pr": "OmniNode-ai/omnimarket#3321", + "reason": "exited", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "a6eebc5af0062255fb75acd278edbad4086da739bced36754e147a8d1adc08c7", + "ladder_index": 0, + "outcome": "external_blocker", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3322", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "3067fadf8a1f32c7896e02460e319548b7b9587cbe6a680870f449f4d51ec1e3", + "ladder_index": 0, + "outcome": "external_blocker", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3323", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "a0f50d4268591f9137a733a32641631e1c18083bd18dcdae7a84fc3bbb67473d", + "ladder_index": 0, + "outcome": "external_blocker", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3324", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "5df77324669b359f3450b8903277913b1e3c583c26d3b8fb7bac38a3ab3d5343", + "ladder_index": 0, + "outcome": "external_blocker", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3325", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "3f005727700b5edf84d7141062b0726ac75bb14e14a052795843b19f2b4f4baf", + "ladder_index": 0, + "outcome": "external_blocker", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3326", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "03efafb5c87cb8bccb0f9cc1619eb1b7645111889c4cd2b1a46a42282b26a708", + "ladder_index": 0, + "outcome": "external_blocker", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3327", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "05d2c3637bfaa08385efe0fb425727a801348dd746cd77ee351829a2edfdd589", + "ladder_index": 0, + "outcome": "external_blocker", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3328", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "be73132ac06083ab1d76c5d8f86597067e3b539251648d14ec02e2936cf59d2a", + "ladder_index": 0, + "outcome": "external_blocker", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3330", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "4c51934239f5a215ec1fd09da86d8702e967c6b1abe1c9a65322c5ec490c7ec0", + "ladder_index": 0, + "outcome": "external_blocker", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3332", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "91164f2ea390a973465e22ff24ebaa9daab0998e391e06bfced8f3537f231e0e", + "ladder_index": 0, + "outcome": "external_blocker", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3333", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "fe3f2d9988689e2eff7123a7557572694e3b0401e1361a7c5184447dad6f78b0", + "ladder_index": 0, + "outcome": "external_blocker", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3334", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "5f55f2ea48ffdcaae116bfa5f0884ee52b7d4c8a33db605f658895423ca76245", + "ladder_index": 0, + "outcome": "external_blocker", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3335", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "aba3dc177be182b19b2897b05c33e7532d655bbfd272be3869852bdabbde7561", + "ladder_index": 0, + "outcome": "external_blocker", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3337", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "96763ee384784ebd172c38d70da562e196261edc574cf924bb0c98e80ce79456", + "ladder_index": 0, + "outcome": "external_blocker", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3338", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "3ab372bcca945ce0d95ff2147b8dcfaba903f454b38e2bb644b45d1f90455448", + "ladder_index": 0, + "outcome": "external_blocker", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3339", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "792c99f0947c920bb8694de0af315425a01d264f7bfb4f9d8e6d11e29393390f", + "ladder_index": 0, + "outcome": "external_blocker", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3340", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "5d82fa1cdac282a2c63e98920b7b4500c8824c71a4f2402a533cb87daca0eeaf", + "ladder_index": 0, + "outcome": "external_blocker", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/omnimarket#3341", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "fb9b3505618cd88dc37a5c1fbb4b98ae455f12e01449400d83ce77aa8b29f14a", + "ladder_index": 0, + "outcome": "fix_submitted", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/onex_change_control#12643", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "82adfd88084188212a6331e0cc0d1f73810bad24b80302e72bbbd8680ebb7794", + "ladder_index": 1, + "outcome": "invalid", + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/onex_change_control#12647", + "reason": "head_mismatch", + "rerun_heads": [], + "update_heads": [] + }, + { + "attempt_red_checks": [], + "awaiting_head": null, + "blocker_fingerprint": "", + "ladder_index": 0, + "outcome": null, + "parked_fingerprint": "", + "parked_head": null, + "pr": "OmniNode-ai/onex_change_control#12656", + "reason": "none", + "rerun_heads": [], + "update_heads": [] + } + ], + "revoked_lease_ids": [ + 1968, + 1969, + 1983 + ], + "token_holder": null, + "uncovered": [] + }, + "tick": 725 + }, + "provenance": { + "floor_breached_repos": "not in the snapshot: read as no breach", + "gate_since": "per gate-suspended PR: completed_at of the first-created (lowest id) failing check-run copy of its red checks on the snapshot head, read from the GitHub check-runs API by the ids the annotation read carries", + "observed_at": "2026-10-03T21:00:21Z", + "red_annotations": "per red check: the failure-level annotations (at most 3, in order, newline-joined) of the latest failing check-run copy on the snapshot head, read from the GitHub check rollup at about 21:05Z; a red check with no failing copy in that read is absent (unread)", + "scrub": "author handles in annotation text (branch prefixes) replaced by role names", + "sha256": { + "landing-facts-snapshot.json": "31610a0cd3c6052361126554bdb162bc9df33f0eb63da7b6a4faefbde83784ec", + "omnimarket-red-annotations.json": "776bbf7a1624c24e883ae9c37bb5ad24240c734b976e31ddfbf5e85ea1b3d303" + }, + "source": "landing controller facts snapshot of tick 725 (input of the tick), trimmed to the omnimarket PRs, the live leases and their probes, and the records of those subjects", + "tick": 725 + } +} diff --git a/tests/unit/nodes/node_pr_landing_decision_compute/test_landing_decision_causes.py b/tests/unit/nodes/node_pr_landing_decision_compute/test_landing_decision_causes.py new file mode 100644 index 0000000000..a532177544 --- /dev/null +++ b/tests/unit/nodes/node_pr_landing_decision_compute/test_landing_decision_causes.py @@ -0,0 +1,1411 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Shared red causes: one bounded cause worker for many red PRs. + +Selectors (one ``-k`` each): ``replay_tick722`` replays the landing controller's +snapshot of 2026-10-03 (one omnimarket cause, at least 16 members, no per-PR +dispatch for a member, omnimarket#3321 inside); ``cause_permutation`` checks +the decision is byte-identical under input permutation; ``cause_scenario`` +runs S21 to S28 (the cause scenarios of the landing controller model, taken +from the scenario text of the model task while its TLC verdict is pending); +``budget``, ``fixer_hold`` and ``escalated_singleton`` pin the bounds. + +The replay fixture ``fixtures/replay_2026_10_03_omnimarket.json`` is the +controller's facts snapshot of tick 725 (observed 2026-10-03T21:00:21Z, the +input of that tick; the same cluster as tick 722), trimmed to the 25 omnimarket +PRs, the 5 live leases with their probes, and the records of those subjects. +Two fields the snapshot does not carry were derived, never invented: + +* ``red_annotations``: for each red check, the failure-level annotations (at + most 3, newline-joined, in order) of the latest failing check-run copy on + the snapshot head, from the GitHub check rollup read at about 21:05Z. A red + check with no failing copy in that read (``CI Summary``, ``Coverage Sweep + Gate`` and ``OCC Companion Merged Gate`` past the rollup's first 100 + contexts) is absent, so it reads ``unread`` and never clusters. +* ``gate_since``: for each ``gate``-suspended PR, the ``completed_at`` of the + first-created (lowest id) failing check-run copy of its red checks on the + snapshot head, read from the GitHub check-runs API by the ids the rollup read + carries: the moment that head first read red. The controller's tick tail + confirms every such PR in its companion wait from tick 721 (20:36:14Z), so + the true wait is at least 24 minutes and at most this bound; + ``test_replay_tick722_lower_bound_gate_still_forms_a_cause`` replays the + lower bound too. + +Author handles inside annotation text (branch prefixes) are replaced by role +names; the floor alarm's breach set is not in the snapshot and reads as no +breach. +""" + +from __future__ import annotations + +import hashlib +import json +import random +import re +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import pytest + +from omnimarket.nodes.node_pr_landing_decision_compute.handlers.handler_pr_landing_decision import ( + decide_landing, +) +from omnimarket.nodes.node_pr_landing_decision_compute.models.model_landing_decision import ( + ModelLandingAction, + ModelLandingDecision, +) +from omnimarket.nodes.node_pr_landing_decision_compute.models.model_landing_facts import ( + ModelLandingFacts, +) +from omnimarket.nodes.node_pr_landing_decision_compute.models.model_landing_state import ( + ModelLandingControllerState, +) + +FIXTURE = Path(__file__).parent / "fixtures" / "replay_2026_10_03_omnimarket.json" +OMNIMARKET = "OmniNode-ai/omnimarket" +REPO = "acme/app" +OTHER = "acme/lib" +EPOCH = datetime(2026, 10, 3, 12, 0, tzinfo=UTC) +TICK = timedelta(seconds=300) +CHECK = "repo-evidence / dod-verify" +ANN = ( + "Process completed with exit code 1.\n" + "OMN-17427 [dod-a, dod-b]: bound test also passes at the merge base: the " + "control did not fail (always-pass)" +) +EMPTY_FINGERPRINT = hashlib.sha256(b"").hexdigest() + + +def sha(subject: str, n: int) -> str: + return hashlib.sha1(f"{subject}@{n}".encode()).hexdigest() + + +def key_for(check: str = CHECK, text: str = ANN, repo: str = REPO) -> str: + from omnimarket.nodes.node_pr_landing_decision_compute.handlers.handler_pr_landing_decision import ( + cause_key, + normalize_signature, + ) + + return cause_key(repo, normalize_signature(check, text)) + + +@dataclass +class Pr: + number: int + repo: str = REPO + head: int = 1 + state: str = "open" + ci: str = "red" + red_class: str = "product" + red_checks: tuple[str, ...] = (CHECK,) + annotations: dict[str, str] = field(default_factory=lambda: {CHECK: ANN}) + suspensions: tuple[str, ...] = () + gate_since: datetime | None = None + process_fix: bool = False + rerun_runs: list[dict[str, Any]] = field(default_factory=list) + + @property + def key(self) -> str: + return f"{self.repo}#{self.number}" + + @property + def head_sha(self) -> str: + return sha(self.key, self.head) + + def facts(self) -> dict[str, Any]: + data: dict[str, Any] = { + "pr": self.key, + "head_sha": self.head_sha, + "state": self.state, + "ci": self.ci, + "created_at": (EPOCH - timedelta(days=1, minutes=-self.number)).isoformat(), + "suspensions": list(self.suspensions), + "process_fix": self.process_fix, + "rerun_runs": list(self.rerun_runs), + } + if self.ci == "red": + data["red_class"] = self.red_class + data["red_checks"] = list(self.red_checks) + data["red_annotations"] = dict(self.annotations) + if self.gate_since is not None: + data["gate_since"] = self.gate_since.isoformat() + return data + + +def green(number: int, **kw: Any) -> Pr: + return Pr(number, ci="green", red_checks=(), annotations={}, **kw) + + +def parked_record(pr: Pr) -> dict[str, Any]: + """The record of a PR parked escalation_exhausted on its head (R1).""" + return { + "pr": pr.key, + "ladder_index": 3, + "outcome": "timed_out", + "reason": "exited", + "parked_head": pr.head_sha, + "parked_fingerprint": EMPTY_FINGERPRINT, + } + + +class CauseWorld: + """GitHub truth, worker processes and the controller state, tick by tick.""" + + def __init__( + self, + prs: list[Pr], + policy: dict[str, Any] | None = None, + records: list[dict[str, Any]] | None = None, + **extra: Any, + ) -> None: + self.prs = {p.key: p for p in prs} + self.policy = {"max_workers": 18, **(policy or {})} + self.state = ModelLandingControllerState.model_validate( + {"records": records or []} + ) + self.extra: dict[str, Any] = dict(extra) + self.tick = 0 + self.now = EPOCH + self.alive: dict[int, bool] = {} + self.kill_works: dict[int, bool] = {} + self.results: list[dict[str, Any]] = [] + self.crash_spawn = False + self.decisions: list[ModelLandingDecision] = [] + + def facts(self) -> ModelLandingFacts: + probes = [ + { + "lease_id": le.lease_id, + "group_alive": self.alive.get(le.lease_id, False), + "tagged_alive": self.alive.get(le.lease_id, False), + } + for le in self.state.leases + ] + return ModelLandingFacts.model_validate( + { + "tick": self.tick, + "observed_at": self.now.isoformat(), + "policy": self.policy, + "state": self.state.model_dump(mode="json"), + "prs": [self.prs[k].facts() for k in sorted(self.prs)], + "probes": probes, + "results": self.results, + **self.extra, + } + ) + + def step(self, advance: timedelta = TICK) -> ModelLandingDecision: + self.tick += 1 + self.now += advance + facts = self.facts() + decision = decide_landing(facts) + assert decide_landing(facts).model_dump_json() == decision.model_dump_json() + self._check(facts, decision) + self.results = [] + self.state = decision.next_state + for action in decision.actions: + self._perform(action) + self.decisions.append(decision) + return decision + + def _check(self, facts: ModelLandingFacts, decision: ModelLandingDecision) -> None: + state = decision.next_state + subjects = [le.pr for le in state.leases] + assert len(subjects) == len(set(subjects)), ( + f"two leases on one subject {subjects}" + ) + released = {le.lease_id for le in facts.state.leases} - { + le.lease_id for le in state.leases + } + for lease_id in released: + assert not self.alive.get(lease_id, False), ( + f"lease {lease_id} released alive" + ) + leased = {le.pr for le in state.leases} + held: set[str] = set() + for rec in state.causes: + not_shared = ( + rec.outcome is not None and rec.outcome.value == "cause_not_shared" + ) + if rec.key in leased and not not_shared: + held |= {m.pr for m in rec.members} + for action in decision.actions: + if action.kind.value == "dispatch_worker" and not is_cause_subject( + action.subject + ): + assert action.subject not in held, ( + f"per-PR dispatch of {action.subject}, a member of a leased cause" + ) + + def _perform(self, action: ModelLandingAction) -> None: + kind = action.kind.value + if kind == "dispatch_worker": + assert action.lease_id is not None + self.alive[action.lease_id] = not self.crash_spawn + elif kind == "kill_worker": + assert action.lease_id is not None + if self.kill_works.get(action.lease_id, True): + self.alive[action.lease_id] = False + elif kind == "rerun": + pr = self.prs[action.subject] + pr.ci = "pending" + pr.rerun_runs.append( + { + "run_id": 9000 + len(pr.rerun_runs) + 1, + "head_sha": pr.head_sha, + "created_at": self.now.isoformat(), + } + ) + elif kind == "merge": + pr = self.prs[action.subject] + if pr.ci == "green" and pr.head_sha == action.head_sha: + pr.state = "merged" + + def lease(self, subject: str) -> Any: + return next(le for le in self.state.leases if le.pr == subject) + + def result(self, subject: str, kind: str, **fields: Any) -> int: + lease_id = int(self.lease(subject).lease_id) + self.results.append( + {"lease_id": lease_id, "pr": subject, "kind": kind, **fields} + ) + return lease_id + + def exit(self, subject: str) -> None: + self.alive[self.lease(subject).lease_id] = False + + def cause(self, key: str) -> Any: + return next(c for c in self.state.causes if c.key == key) + + +def is_cause_subject(subject: str) -> bool: + return subject.startswith("cause:") + + +def dispatched(decision: ModelLandingDecision) -> list[str]: + return [a.subject for a in decision.actions if a.kind.value == "dispatch_worker"] + + +def cause_dispatched(decision: ModelLandingDecision) -> list[str]: + return [s for s in dispatched(decision) if is_cause_subject(s)] + + +def per_pr_dispatched(decision: ModelLandingDecision) -> list[str]: + return [s for s in dispatched(decision) if not is_cause_subject(s)] + + +def actions_of(decision: ModelLandingDecision, kind: str) -> list[ModelLandingAction]: + return [a for a in decision.actions if a.kind.value == kind] + + +def member_prs(action: ModelLandingAction) -> list[str]: + assert action.cause_brief is not None + return [m.pr for m in action.cause_brief.members] + + +def recorded(decision: ModelLandingDecision) -> list[str]: + return sorted( + f"{r.pr} {r.outcome.value} {r.reason.value}" for r in decision.recorded_outcomes + ) + + +def three() -> list[Pr]: + return [Pr(1), Pr(2), Pr(3)] + + +# ------------------------------------------------------------------ signature +@pytest.mark.unit +def test_normalize_signature_replaces_what_varies_per_pr() -> None: + from omnimarket.nodes.node_pr_landing_decision_compute.handlers.handler_pr_landing_decision import ( + UNREAD, + normalize_annotation, + normalize_signature, + ) + + a = ( + "companion OCC#12642 is still OPEN -- it must merge before this PR may " + "merge -- poll deadline (1500s) reached after 1508s at 2026-10-03T20:36:30Z " + "on 3a1beb08992004c653336cbb4bf8039d20364b58" + ) + b = ( + "companion OCC#12574 is still OPEN -- it must merge before this PR may " + "merge -- poll deadline (1500s) reached after 1531s at 2026-10-03T19:01:18Z " + "on 0ad953a7f00d" + ) + assert normalize_annotation(a) == normalize_annotation(b) + assert "#" in normalize_annotation(a) + assert "" in normalize_annotation(a) + assert "" in normalize_annotation(a) + check = "occ-preflight / eligibility" + assert normalize_signature(check, a) == normalize_signature(check, b) + assert re.fullmatch(r"[0-9a-f]{12}", normalize_signature(check, a)) + assert normalize_signature("other check", a) != normalize_signature(check, a) + ids_one = "OMN-17427 [dod-x-ac1, dod-x-ac2]: bound test also passes" + ids_two = "OMN-18000 [dod-y-ac3]: bound test also passes" + assert normalize_signature(CHECK, ids_one) == normalize_signature(CHECK, ids_two) + generic = "Process completed with exit code 1.\nreal failure line" + assert normalize_annotation(generic) == "real failure line" + assert normalize_signature(CHECK, None) == UNREAD + assert normalize_signature(CHECK, "") == UNREAD + assert normalize_signature(CHECK, "Process completed with exit code 2.") == UNREAD + assert len(normalize_annotation("word " * 200)) == 300 + + +# --------------------------------------------------------------------- replay +def _replay_facts() -> ModelLandingFacts: + doc = json.loads(FIXTURE.read_text()) + return ModelLandingFacts.model_validate(doc["facts"]) + + +def _members_of_only_cause(decision: ModelLandingDecision) -> tuple[str, list[str]]: + causes = [ + a + for a in actions_of(decision, "dispatch_worker") + if is_cause_subject(a.subject) + ] + assert len(causes) == 1, [a.subject for a in causes] + return causes[0].subject, member_prs(causes[0]) + + +@pytest.mark.unit +def test_replay_tick722_one_omnimarket_cause_covers_the_cluster() -> None: + """AC1: one cause, at least 16 members, #3321 inside, no per-PR dispatch for a member.""" + decision = decide_landing(_replay_facts()) + key, members = _members_of_only_cause(decision) + assert key.startswith(f"cause:{OMNIMARKET}:") + assert len(members) >= 16 + assert f"{OMNIMARKET}#3321" in members + assert not set(per_pr_dispatched(decision)) & set(members) + # pinned by this replay: the dod-verify always-pass pair leads, 19 members + assert key == "cause:OmniNode-ai/omnimarket:52c59723e1ad" + assert len(members) == 19 + + +@pytest.mark.unit +def test_replay_tick722_cause_dispatch_comes_first_with_one_bounded_lease() -> None: + facts = _replay_facts() + decision = decide_landing(facts) + key, members = _members_of_only_cause(decision) + dispatches = dispatched(decision) + assert dispatches[0] == key + leases = [le for le in decision.next_state.leases if le.pr == key] + assert len(leases) == 1 + assert leases[0].deadline_at - facts.observed_at == timedelta(seconds=5400) + assert leases[0].engine.value == "claude_opus" + brief = decision.actions[ + [a.subject for a in decision.actions].index(key) + ].cause_brief + assert brief is not None + assert brief.pairs[0].check == CHECK + assert {p.check for p in brief.pairs} >= {CHECK, "OCC Preflight Dependency"} + # the live per-PR lease on a member runs to its own deadline + assert any(le.pr == f"{OMNIMARKET}#3338" for le in decision.next_state.leases) + assert f"{OMNIMARKET}#3338" in members + # a gate younger than the stall window stays out of the cause + assert f"{OMNIMARKET}#3340" not in members + + +@pytest.mark.unit +def test_replay_tick722_lower_bound_gate_still_forms_a_cause() -> None: + """With every gate dated at the tick tail's first sighting (20:36:14Z), only + the PRs with no gate cluster: still one cause, holding #3321.""" + facts = _replay_facts() + first_seen = datetime(2026, 10, 3, 20, 36, 14, tzinfo=UTC) + prs = tuple( + p.model_copy(update={"gate_since": first_seen}) if p.gate_since else p + for p in facts.prs + ) + decision = decide_landing(facts.model_copy(update={"prs": prs})) + _, members = _members_of_only_cause(decision) + assert f"{OMNIMARKET}#3321" in members + assert all(not next(p for p in prs if p.pr == m).suspensions for m in members) + assert not set(per_pr_dispatched(decision)) & set(members) + + +@pytest.mark.unit +def test_replay_tick722_fixture_carries_no_private_paths_or_handles() -> None: + text = FIXTURE.read_text() + assert not re.search(r"/(Users|home|root)/", text) + assert not re.search(r"\bh20\d\b", text) + prefixes = set(re.findall(r"\b([A-Za-z0-9_.-]+)/[A-Za-z0-9_.-]+", text)) + assert prefixes <= {"OmniNode-ai", "evidence", "operator", "test"}, prefixes + + +# ---------------------------------------------------------------- permutation +def _shuffled(items: tuple[Any, ...], how: str) -> tuple[Any, ...]: + out = list(items) + if how == "reversed": + out.reverse() + elif how == "rotated": + out = out[len(out) // 2 :] + out[: len(out) // 2] + elif how.startswith("shuffle"): + random.Random(int(how.split("-")[1])).shuffle(out) + return tuple(out) + + +def _permuted(facts: ModelLandingFacts, how: str) -> ModelLandingFacts: + state = facts.state + prs = tuple( + p.model_copy( + update={ + "red_annotations": dict(reversed(list(p.red_annotations.items()))), + "red_checks": tuple(reversed(p.red_checks)), + } + ) + for p in facts.prs + ) + return facts.model_copy( + update={ + "prs": _shuffled(prs, how), + "probes": _shuffled(facts.probes, how), + "results": _shuffled(facts.results, how), + "cause_owners": _shuffled(facts.cause_owners, how), + "cause_releases": _shuffled(facts.cause_releases, how), + "fixer_hold": _shuffled(facts.fixer_hold, how), + "floor_breached_repos": _shuffled(facts.floor_breached_repos, how), + "state": state.model_copy( + update={ + "leases": _shuffled(state.leases, how), + "records": _shuffled(state.records, how), + "causes": _shuffled(state.causes, how), + } + ), + } + ) + + +def _midway_facts() -> ModelLandingFacts: + """A two-repo world one tick after its causes dispatched, a result pending.""" + world = CauseWorld( + [ + *three(), + Pr(4), + Pr(1, repo=OTHER), + Pr(2, repo=OTHER), + Pr(7, annotations={CHECK: "x"}), + ], + floor_breached_repos=[OTHER], + ) + world.step() + world.result(key_for(), "cause_not_shared", reason="coincidence") + world.tick += 1 + world.now += TICK + return world.facts() + + +PERMUTATIONS = ["reversed", "rotated", "shuffle-1", "shuffle-2", "shuffle-3"] + + +@pytest.mark.unit +@pytest.mark.parametrize("how", PERMUTATIONS) +@pytest.mark.parametrize("source", ["replay", "midway"]) +def test_cause_permutation(source: str, how: str) -> None: + """AC2: any permutation of the facts gives a byte-identical decision.""" + facts = _replay_facts() if source == "replay" else _midway_facts() + first = decide_landing(facts).model_dump_json() + assert decide_landing(_permuted(facts, how)).model_dump_json() == first + assert ( + decide_landing( + ModelLandingFacts.model_validate_json(facts.model_dump_json()) + ).model_dump_json() + == first + ) + + +# ------------------------------------------------------------------ scenarios +@pytest.mark.unit +def test_cause_scenario_s21_cluster_of_three_gets_one_lease_and_no_per_pr_dispatch() -> ( + None +): + world = CauseWorld(three()) + decision = world.step() + key = key_for() + assert dispatched(decision) == [key] + (action,) = decision.actions + assert action.cause_brief is not None + assert action.cause_brief.brief_class.value == "shared_cause" + assert action.cause_brief.attempt == 1 + assert action.cause_brief.engine.value == "claude_opus" + assert member_prs(action) == [f"{REPO}#1", f"{REPO}#2", f"{REPO}#3"] + lease = world.lease(key) + assert lease.deadline_at - lease.dispatched_at == timedelta(seconds=5400) + assert world.step().actions == () # members stay suppressed under the lease + # two red PRs do not cluster: each gets its own worker, as before + pair = CauseWorld([Pr(1), Pr(2)]) + assert sorted(dispatched(pair.step())) == [f"{REPO}#1", f"{REPO}#2"] + + +@pytest.mark.unit +def test_cause_scenario_s22_fix_merges_members_rerun_and_go_green() -> None: + fix = Pr(99, ci="pending", red_checks=(), annotations={}, process_fix=True) + world = CauseWorld([*three(), fix]) + key = key_for() + assert dispatched(world.step()) == [key] + world.result(key, "cause_fix_submitted", fix_ref=fix.key, fix_head=fix.head_sha) + decision = world.step() + assert recorded(decision) == [f"{key} cause_fix_submitted none"] + rec = world.cause(key) + assert (rec.attempts, rec.fix_ref) == (1, fix.key) + assert per_pr_dispatched(decision) == [] + fix.ci = "green" + decision = world.step() # the controller lands the fix PR like any PR + assert [a.subject for a in actions_of(decision, "merge")] == [fix.key] + assert per_pr_dispatched(decision) == [] + decision = world.step() # merged: one rerun per member head + reruns = actions_of(decision, "rerun") + assert sorted((a.subject, a.head_sha) for a in reruns) == sorted( + (p.key, p.head_sha) for p in three() + ) + assert actions_of(world.step(), "rerun") == [] # keyed by cause and head + for n in (1, 2, 3): + world.prs[f"{REPO}#{n}"].ci = "green" + decision = world.step() + assert sorted(a.subject for a in actions_of(decision, "merge")) == [ + f"{REPO}#1", + f"{REPO}#2", + f"{REPO}#3", + ] + assert world.state.causes == () + assert all(world.prs[f"{REPO}#{n}"].state == "merged" for n in (1, 2, 3)) + + +@pytest.mark.unit +def test_cause_scenario_s23_two_failed_attempts_park_with_one_escalation_then_fallback() -> ( + None +): + world = CauseWorld(three()) + key = key_for() + world.step() + world.result( + key, "external_blocker", blocker_kind="upstream_open", blocker_ref="acme/lib#7" + ) + assert recorded(world.step()) == [f"{key} external_blocker none"] + assert actions_of(world.step(), "kill_worker") + decision = world.step() # released; the cause is still present: attempt 2 + assert dispatched(decision) == [key] + assert decision.actions[0].cause_brief is not None + assert decision.actions[0].cause_brief.attempt == 2 + world.step(advance=timedelta(seconds=5400)) # deadline with the process alive + decision = world.step() + assert f"{key} timed_out deadline" in recorded(decision) + escalations = actions_of(decision, "escalate_operator") + assert len(escalations) == 1 + rec = world.cause(key) + assert rec.attempts == 2 + assert rec.parked_until == world.now + timedelta(hours=12) + assert escalations[0].dedupe_key == f"{key}@{rec.parked_until.isoformat()}" + assert [m.pr for m in escalations[0].members] == [p.key for p in three()] + assert cause_dispatched(decision) == [] + assert sorted(per_pr_dispatched(decision)) == [p.key for p in three()] # fallback + assert [(d.reason.value, d.subject) for d in decision.degraded] == [ + ("cause_exhausted", key) + ] + for _ in range(3): # the members' workers run; the park keeps its one MSG + later = world.step() + assert later.actions == () + assert [(d.reason.value, d.subject) for d in later.degraded] == [ + ("cause_exhausted", key) + ] + total = sum(len(actions_of(d, "escalate_operator")) for d in world.decisions) + assert total == 1 + + +@pytest.mark.unit +def test_cause_scenario_s24_fixer_hold_revokes_kills_and_confirms_with_no_dispatch() -> ( + None +): + late = Pr(5, ci="pending", red_checks=(), annotations={}) + world = CauseWorld([*three(), late]) + key = key_for() + world.step() + lease_id = world.lease(key).lease_id + world.extra["fixer_hold"] = [REPO] + late.ci = "green" + decision = world.step() + assert [a.lease_id for a in actions_of(decision, "kill_worker")] == [lease_id] + assert world.lease(key).revoked + assert [a.subject for a in actions_of(decision, "merge")] == [late.key] + assert dispatched(decision) == [] + decision = world.step() # confirmed terminated: released, nothing dispatched + assert recorded(decision) == [f"{key} timed_out revoked"] + assert world.state.leases == () + assert dispatched(decision) == [] + assert world.cause(key).attempts == 0 # a revoke spends nothing + assert dispatched(world.step()) == [] + world.extra["fixer_hold"] = [] + decision = world.step() + assert dispatched(decision) == [key] + assert decision.actions[0].cause_brief is not None + assert decision.actions[0].cause_brief.attempt == 1 + + +@pytest.mark.unit +def test_cause_scenario_s25_per_pr_claims_do_not_block_a_cause_claim_does() -> None: + owned = ("owned",) + prs = [ + Pr(1, suspensions=owned), + Pr(2, suspensions=owned), + Pr(3), + green(4, suspensions=owned), + ] + world = CauseWorld(prs) + decision = world.step() + assert dispatched(decision) == [key_for()] + assert member_prs(decision.actions[0]) == [f"{REPO}#1", f"{REPO}#2", f"{REPO}#3"] + assert actions_of(decision, "merge") == [] # an owned PR is never merged here + for owner in ( + {"lane": "cause-lane", "repo": REPO, "check": CHECK}, + {"lane": "cause-lane", "repo": REPO, "cause": key_for()}, + ): + claimed = CauseWorld( + [Pr(1, suspensions=owned), Pr(2, suspensions=owned), Pr(3)], + cause_owners=[owner], + ) + assert claimed.step().actions == () # no cause worker, no per-PR worker + unrelated = CauseWorld( + three(), cause_owners=[{"lane": "x", "repo": REPO, "check": "lint"}] + ) + assert dispatched(unrelated.step()) == [key_for()] + + +@pytest.mark.unit +def test_cause_scenario_s26_an_escalated_pr_alone_becomes_a_cause() -> None: + pr = Pr(1) + world = CauseWorld([pr], records=[parked_record(pr)]) + decision = world.step() + assert dispatched(decision) == [key_for()] + assert member_prs(decision.actions[0]) == [pr.key] + assert ("escalation_exhausted", pr.key) in [ + (d.reason.value, d.subject) for d in decision.degraded + ] + assert world.step().actions == () + + +@pytest.mark.unit +def test_cause_scenario_s27_cause_not_shared_returns_members_to_per_pr_rung_zero() -> ( + None +): + prs = three() + world = CauseWorld(prs, records=[parked_record(prs[2])]) + key = key_for() + assert dispatched(world.step()) == [key] + world.result( + key, "cause_not_shared", reason="the annotation matches by coincidence" + ) + decision = world.step() + assert recorded(decision) == [f"{key} cause_not_shared none"] + per_pr = actions_of(decision, "dispatch_worker") + assert sorted(a.subject for a in per_pr) == [p.key for p in prs] + for action in per_pr: + assert action.brief is not None + assert action.brief.engine.value == "claude_sonnet" + assert action.brief.brief_class.value == "real_red" + records = {r.pr: r for r in world.state.records} + assert all(records[p.key].ladder_index == 0 for p in prs) + assert records[prs[2].key].parked_head is None + for _ in range(3): # blocked for this member set: no cause again + assert not [s for s in dispatched(world.step()) if is_cause_subject(s)] + prs[0].head += 1 # a member's head moves + decision = world.step() + assert key in dispatched(decision) + + +@pytest.mark.unit +def test_cause_scenario_s28_crash_between_state_write_and_spawn_leaves_one_lease() -> ( + None +): + world = CauseWorld(three()) + key = key_for() + world.crash_spawn = True + world.step() # next state written with the lease, the spawn never happened + first = world.lease(key).lease_id + world.crash_spawn = False + decision = world.step() + assert recorded(decision) == [f"{key} timed_out exited"] + assert [le.pr for le in world.state.leases] == [key] + assert world.lease(key).lease_id == first + 1 + rec = world.cause(key) + assert (rec.attempts, rec.spawn_failures) == (0, 1) + assert world.step().actions == () + assert [le.pr for le in world.state.leases] == [key] + + +# --------------------------------------------------------------------- budget +def _park_by_spawn_failures(world: CauseWorld, key: str) -> ModelLandingDecision: + world.crash_spawn = True + world.step() + world.step() + world.step() + world.crash_spawn = False # the park tick's own workers (the fallback) do spawn + return world.step() + + +@pytest.mark.unit +def test_budget_three_spawn_failures_park_the_cause_and_members_fall_back() -> None: + world = CauseWorld(three()) + key = key_for() + decision = _park_by_spawn_failures(world, key) + rec = world.cause(key) + assert (rec.attempts, rec.spawn_failures) == (0, 3) + assert len(actions_of(decision, "escalate_operator")) == 1 + assert cause_dispatched(decision) == [] + assert sorted(per_pr_dispatched(decision)) == [p.key for p in three()] # fallback + assert world.step().actions == () # parked: no cause worker, members working + + +@pytest.mark.unit +def test_budget_an_exit_after_the_spawn_window_spends_an_attempt() -> None: + world = CauseWorld(three()) + key = key_for() + world.step() + world.exit(key) + decision = world.step(advance=timedelta(seconds=900)) + assert recorded(decision) == [f"{key} timed_out exited"] + assert decision.actions[0].cause_brief is not None + assert decision.actions[0].cause_brief.attempt == 2 + assert world.cause(key).spawn_failures == 0 + + +@pytest.mark.unit +def test_budget_an_unverified_fix_claim_spends_the_attempt() -> None: + world = CauseWorld(three()) + key = key_for() + world.step() + world.result(key, "cause_fix_submitted", fix_ref=f"{REPO}#404", fix_head="a" * 40) + decision = world.step() + assert recorded(decision) == [f"{key} invalid unverified_claim"] + assert [v.pr for v in decision.violations] == [key] + assert world.cause(key).attempts == 1 + assert world.cause(key).fix_ref is None + + +@pytest.mark.unit +def test_budget_release_row_ends_the_park_early_and_a_stale_one_does_not() -> None: + world = CauseWorld(three()) + key = key_for() + _park_by_spawn_failures(world, key) + world.extra["cause_releases"] = [ + {"cause": key, "at": (world.now - timedelta(hours=1)).isoformat()} + ] + assert dispatched(world.step()) == [] + world.extra["cause_releases"] = [{"cause": key, "at": world.now.isoformat()}] + decision = world.step() + assert dispatched(decision) == [key] + assert decision.actions[0].cause_brief is not None + assert decision.actions[0].cause_brief.attempt == 1 + + +@pytest.mark.unit +def test_budget_park_ends_after_twelve_hours() -> None: + world = CauseWorld(three()) + key = key_for() + _park_by_spawn_failures(world, key) + assert dispatched(world.step(advance=timedelta(hours=11))) == [] + assert dispatched(world.step(advance=timedelta(hours=1))) == [key] + + +# ----------------------------------------------------------------- fixer hold +@pytest.mark.unit +def test_fixer_hold_repo_scope_revokes_every_lease_there_and_dispatches_nothing() -> ( + None +): + lone = Pr(7, annotations={CHECK: "a failure of its own"}) + elsewhere = Pr(1, repo=OTHER) + world = CauseWorld([*three(), lone, elsewhere]) + first = world.step() + assert sorted(dispatched(first)) == sorted([key_for(), lone.key, elsewhere.key]) + world.extra["fixer_hold"] = [REPO] + decision = world.step() + killed = {a.subject for a in actions_of(decision, "kill_worker")} + assert killed == {key_for(), lone.key} + decision = world.step() + assert {le.pr for le in world.state.leases} == {elsewhere.key} + assert dispatched(decision) == [] + world.extra["fixer_hold"] = [] + assert sorted(dispatched(world.step())) == sorted([key_for(), lone.key]) + + +@pytest.mark.unit +def test_fixer_hold_all_scope_still_merges_and_reruns() -> None: + cascade = Pr(8, red_class="cascade", annotations={}) + world = CauseWorld([*three(), green(9), cascade], fixer_hold=["all"]) + decision = world.step() + assert dispatched(decision) == [] + assert [a.subject for a in actions_of(decision, "merge")] == [f"{REPO}#9"] + assert [a.subject for a in actions_of(decision, "rerun")] == [cascade.key] + + +# -------------------------------------------------------- escalated singleton +@pytest.mark.unit +def test_escalated_singleton_dispatches_one_cause_for_a_parked_pr() -> None: + pr = Pr(1) + world = CauseWorld([pr, Pr(2, repo=OTHER)], records=[parked_record(pr)]) + decision = world.step() + assert key_for() in dispatched(decision) + assert pr.key not in per_pr_dispatched(decision) + assert len([s for s in dispatched(decision) if is_cause_subject(s)]) == 1 + + +@pytest.mark.unit +def test_escalated_singleton_is_not_minted_for_a_pr_a_cluster_covers() -> None: + prs = three() + world = CauseWorld(prs, records=[parked_record(prs[0])]) + decision = world.step() + assert dispatched(decision) == [key_for()] + assert prs[0].key in member_prs(decision.actions[0]) + + +@pytest.mark.unit +def test_escalated_singleton_needs_a_readable_annotation() -> None: + pr = Pr(1, annotations={}) + world = CauseWorld([pr], records=[parked_record(pr)]) + decision = world.step() + assert decision.actions == () + assert [(d.reason.value, d.subject) for d in decision.degraded] == [ + ("escalation_exhausted", pr.key) + ] + + +# ------------------------------------------------------------------- clusters +@pytest.mark.unit +def test_gate_joins_clusters_only_after_the_stall_window() -> None: + def gated(minutes: int) -> list[Pr]: + since = EPOCH + TICK - timedelta(minutes=minutes) + return [Pr(n, suspensions=("gate",), gate_since=since) for n in (1, 2, 3)] + + assert CauseWorld(gated(59)).step().actions == () + assert CauseWorld(gated(60)).step().actions == () + assert dispatched(CauseWorld(gated(61)).step()) == [key_for()] + undated = [Pr(n, suspensions=("gate",)) for n in (1, 2, 3)] + assert CauseWorld(undated).step().actions == () + + +@pytest.mark.unit +@pytest.mark.parametrize("suspension", ["hold", "draft", "do_not_land"]) +def test_person_holds_drafts_and_do_not_land_never_join(suspension: str) -> None: + world = CauseWorld([Pr(1), Pr(2), Pr(3, suspensions=(suspension,))]) + decision = world.step() + assert not [s for s in dispatched(decision) if is_cause_subject(s)] + assert sorted(dispatched(decision)) == [f"{REPO}#1", f"{REPO}#2"] + + +@pytest.mark.unit +def test_follower_checks_reviewer_pool_and_unread_never_cluster() -> None: + for check in ("CI Summary", "Hostile Review Gate"): + prs = [Pr(n, red_checks=(check,), annotations={check: ANN}) for n in (1, 2, 3)] + assert not [ + s for s in dispatched(CauseWorld(prs).step()) if is_cause_subject(s) + ] + unread = [Pr(n, annotations={}) for n in (1, 2, 3)] + assert not [s for s in dispatched(CauseWorld(unread).step()) if is_cause_subject(s)] + + +@pytest.mark.unit +def test_a_rerunnable_red_reruns_before_it_clusters() -> None: + prs = [Pr(n, red_class="cascade") for n in (1, 2, 3)] + decision = CauseWorld(prs).step() + assert dispatched(decision) == [] + assert sorted(a.subject for a in actions_of(decision, "rerun")) == [ + p.key for p in prs + ] + + +@pytest.mark.unit +def test_an_absorbed_cluster_carries_its_pairs_into_one_cause() -> None: + lead = "occ-preflight / eligibility" + lead_text = "companion OCC#1 is still OPEN" + prs = [] + for n in range(1, 6): + checks = {lead: lead_text} + if n <= 4: + checks[CHECK] = ANN + prs.append(Pr(n, red_checks=tuple(checks), annotations=checks)) + decision = CauseWorld(prs).step() + (action,) = actions_of(decision, "dispatch_worker") + assert action.subject == key_for(lead, lead_text) + assert action.cause_brief is not None + assert [p.check for p in action.cause_brief.pairs] == [lead, CHECK] + assert len(member_prs(action)) == 5 + + +@pytest.mark.unit +def test_a_disjoint_cluster_is_its_own_cause() -> None: + other = "lint" + prs = [*three()] + [ + Pr(n, red_checks=(other,), annotations={other: "E501 line too long"}) + for n in (4, 5, 6) + ] + decision = CauseWorld(prs).step() + assert sorted(dispatched(decision)) == sorted( + [key_for(), key_for(other, "E501 line too long")] + ) + + +@pytest.mark.unit +def test_cause_caps_four_fleet_wide_two_per_repo() -> None: + prs: list[Pr] = [] + for repo in ("acme/a", "acme/b", "acme/c"): + for c in range(3): + check = f"check-{c}" + prs += [ + Pr( + 10 * c + n, + repo=repo, + red_checks=(check,), + annotations={check: "boom"}, + ) + for n in (1, 2, 3) + ] + decision = CauseWorld(prs).step() + causes = [s for s in dispatched(decision) if is_cause_subject(s)] + assert len(causes) == 4 + per_repo: dict[str, int] = {} + for key in causes: + repo = key.split(":")[1] + per_repo[repo] = per_repo.get(repo, 0) + 1 + assert max(per_repo.values()) == 2 + leases_per_repo: dict[str, int] = {} + for lease in decision.next_state.leases: + repo = ( + lease.pr.split(":")[1] + if is_cause_subject(lease.pr) + else lease.pr.split("#")[0] + ) + leases_per_repo[repo] = leases_per_repo.get(repo, 0) + 1 + assert max(leases_per_repo.values()) <= 6 + + +@pytest.mark.unit +def test_per_repo_cap_six_workers_of_any_kind() -> None: + prs = [Pr(n, annotations={CHECK: f"failure number {'x' * n}"}) for n in range(1, 9)] + decision = CauseWorld(prs).step() + assert len(dispatched(decision)) == 6 + + +@pytest.mark.unit +def test_floor_breached_repo_clusters_at_two_and_dispatches_first() -> None: + floor = [Pr(1, repo=OTHER), Pr(2, repo=OTHER)] + big = [Pr(n) for n in range(1, 6)] + world = CauseWorld( + [*floor, *big], policy={"max_cause_workers": 1}, floor_breached_repos=[OTHER] + ) + decision = world.step() + causes = [s for s in dispatched(decision) if is_cause_subject(s)] + assert causes == [key_for(repo=OTHER)] + + +@pytest.mark.unit +def test_cause_dispatch_precedes_per_pr_dispatch_in_a_tick() -> None: + lone = Pr(1, repo=OTHER, annotations={CHECK: "unrelated"}) + decision = CauseWorld([*three(), lone]).step() + assert dispatched(decision) == [key_for(), lone.key] + + +@pytest.mark.unit +def test_cause_brief_text_and_rules_are_fixed() -> None: + from pydantic import ValidationError + + from omnimarket.nodes.node_pr_landing_decision_compute.models.model_landing_decision import ( + BRIEF_INSTRUCTIONS, + ModelLandingCauseBrief, + ) + + decision = CauseWorld(three()).step() + brief = decision.actions[0].cause_brief + assert brief is not None + data = brief.model_dump() + assert data["instructions"] == BRIEF_INSTRUCTIONS[brief.brief_class] + assert [k.value for k in brief.allowed_results] == [ + "cause_fix_submitted", + "cause_not_shared", + "external_blocker", + ] + with pytest.raises(ValidationError): + ModelLandingCauseBrief( + **{**data, "instructions": data["instructions"] + " and more"} + ) + with pytest.raises(ValidationError): + ModelLandingCauseBrief(**{**data, "push_rule": "push to every member branch"}) + + +# ------------------------------------------- T1 model findings LC-F2 and LC-F3 +# omnibase_internal#144 (tla/landing_controller, S21 to S28 reached, every +# mutant caught) found two gaps in the plan's section 2.4. LC-F2 (mutant +# M_P11_late_exit_uncounted): an exit with no result after the spawn window and +# before the deadline counted against no budget, so a cause whose worker keeps +# dying was redispatched without bound. LC-F3 (mutant M_P14_sidecar_dedupe): a +# dedupe key kept only in the controller's state escalated twice after a crash +# between the operator MSG and the state write. The ledger row is the source. +def _exit_mid_run( + world: CauseWorld, key: str, after: int = 900 +) -> ModelLandingDecision: + world.exit(key) + return world.step(advance=timedelta(seconds=after)) + + +@pytest.mark.unit +def test_lc_f2_a_worker_that_keeps_dying_mid_run_parks_after_two_attempts() -> None: + world = CauseWorld(three()) + key = key_for() + world.step() + first = _exit_mid_run(world, key) + assert recorded(first) == [f"{key} timed_out exited"] + assert dispatched(first) == [key] + second = _exit_mid_run(world, key) + assert recorded(second) == [f"{key} timed_out exited"] + assert cause_dispatched(second) == [] + assert len(actions_of(second, "escalate_operator")) == 1 + rec = world.cause(key) + assert (rec.attempts, rec.spawn_failures) == (2, 0) + assert rec.parked_until == world.now + timedelta(hours=12) + for _ in range(4): + later = world.step() + assert cause_dispatched(later) == [] + assert actions_of(later, "escalate_operator") == [] + total = sum(len(actions_of(d, "escalate_operator")) for d in world.decisions) + assert total == 1 + + +@pytest.mark.unit +def test_lc_f2_the_spawn_window_edge_splits_a_spawn_failure_from_an_attempt() -> None: + inside = CauseWorld(three()) + key = key_for() + inside.step() + _exit_mid_run(inside, key, after=599) + assert (inside.cause(key).attempts, inside.cause(key).spawn_failures) == (0, 1) + + at_edge = CauseWorld(three()) + at_edge.step() + _exit_mid_run(at_edge, key, after=600) + assert (at_edge.cause(key).attempts, at_edge.cause(key).spawn_failures) == (1, 0) + + +@pytest.mark.unit +def test_lc_f2_exits_before_and_after_the_window_each_count_toward_one_budget() -> None: + world = CauseWorld(three()) + key = key_for() + world.step() + _exit_mid_run(world, key, after=300) # a spawn failure, not an attempt + assert (world.cause(key).attempts, world.cause(key).spawn_failures) == (0, 1) + _exit_mid_run(world, key, after=1800) # a late exit: an attempt + assert (world.cause(key).attempts, world.cause(key).spawn_failures) == (1, 1) + decision = _exit_mid_run(world, key, after=1800) # the second attempt parks it + assert (world.cause(key).attempts, world.cause(key).spawn_failures) == (2, 1) + assert len(actions_of(decision, "escalate_operator")) == 1 + assert cause_dispatched(decision) == [] + + +def _park_tick(world: CauseWorld, key: str) -> tuple[ModelLandingControllerState, Any]: + before = world.state + world.exit(key) + decision = world.step(advance=timedelta(seconds=900)) + escalations = actions_of(decision, "escalate_operator") + assert len(escalations) == 1 + return before, escalations[0] + + +def _ledger_row(world: CauseWorld, key: str, action: Any) -> dict[str, Any]: + return { + "cause": key, + "at": world.now.isoformat(), + "dedupe_key": action.dedupe_key, + } + + +@pytest.mark.unit +def test_lc_f3_a_crash_between_the_msg_and_the_state_write_escalates_once() -> None: + world = CauseWorld(three()) + key = key_for() + world.step() + _exit_mid_run(world, key) + before, action = _park_tick(world, key) + row = _ledger_row(world, key, action) + world.state = before # the controller died before it wrote the next state + world.extra["cause_escalations"] = [row] + decision = world.step() + assert actions_of(decision, "escalate_operator") == [] + assert cause_dispatched(decision) == [] + rec = world.cause(key) + assert rec.parked_until == datetime.fromisoformat(row["at"]) + timedelta(hours=12) + assert rec.escalated == row["dedupe_key"] + total = sum(len(actions_of(d, "escalate_operator")) for d in world.decisions) + assert total == 1 # the model's P14: one MSG for the park episode + + +@pytest.mark.unit +def test_lc_f3_without_a_ledger_row_the_same_crash_still_escalates() -> None: + """Positive control: the row, and only the row, suppresses the second MSG.""" + world = CauseWorld(three()) + key = key_for() + world.step() + _exit_mid_run(world, key) + before, _ = _park_tick(world, key) + world.state = before + decision = world.step() + assert len(actions_of(decision, "escalate_operator")) == 1 + + +@pytest.mark.unit +def test_lc_f3_two_crashes_in_a_row_still_escalate_once() -> None: + world = CauseWorld(three()) + key = key_for() + world.step() + _exit_mid_run(world, key) + before, action = _park_tick(world, key) + world.extra["cause_escalations"] = [_ledger_row(world, key, action)] + for _ in range(2): + world.state = before + assert actions_of(world.step(), "escalate_operator") == [] + assert world.cause(key).escalated == action.dedupe_key + + +@pytest.mark.unit +def test_lc_f3_a_row_older_than_the_park_does_not_cover_a_new_episode() -> None: + world = CauseWorld(three()) + key = key_for() + world.step() + _exit_mid_run(world, key) + before, action = _park_tick(world, key) + stale = _ledger_row(world, key, action) + stale["at"] = (world.now - timedelta(hours=13)).isoformat() + world.state = before + world.extra["cause_escalations"] = [stale] + assert len(actions_of(world.step(), "escalate_operator")) == 1 + + +@pytest.mark.unit +def test_lc_f3_a_release_after_the_row_starts_a_new_episode() -> None: + world = CauseWorld(three()) + key = key_for() + world.step() + _exit_mid_run(world, key) + before, action = _park_tick(world, key) + row = _ledger_row(world, key, action) + world.state = before + world.extra["cause_escalations"] = [row] + world.extra["cause_releases"] = [ + {"cause": key, "at": (world.now + timedelta(minutes=1)).isoformat()} + ] + assert ( + len(actions_of(world.step(advance=timedelta(minutes=2)), "escalate_operator")) + == 1 + ) + + +@pytest.mark.unit +def test_lc_f3_rows_for_other_causes_and_their_order_change_nothing() -> None: + world = CauseWorld(three()) + key = key_for() + other = key_for(check="build / unit", text="boom") + world.step() + _exit_mid_run(world, key) + before, action = _park_tick(world, key) + rows = [ + _ledger_row(world, key, action), + {"cause": other, "at": world.now.isoformat(), "dedupe_key": f"{other}@x"}, + ] + world.state = before + world.extra["cause_escalations"] = rows + first = decide_landing(world.facts()).model_dump_json() + world.extra["cause_escalations"] = list(reversed(rows)) + assert decide_landing(world.facts()).model_dump_json() == first + assert actions_of(world.step(), "escalate_operator") == [] + + +# ------------------------------------------------ parked cause: per-PR fallback +# Operator ruling: a parked cause no longer holds its members. Red PRs keep +# getting fixed per PR while the cause waits out its park; a live cause lease, +# an open fix PR or an owner still hold them. +def _park_by_two_failed_attempts(world: CauseWorld, key: str) -> ModelLandingDecision: + """The s23 path: two attempts that fail, the second at its deadline.""" + world.step() + world.result( + key, "external_blocker", blocker_kind="upstream_open", blocker_ref="acme/lib#7" + ) + world.step() + world.step() # the kill + world.step() # released: attempt 2 + world.step(advance=timedelta(seconds=5400)) # deadline with the process alive + return world.step() + + +def _escalations(world: CauseWorld) -> int: + return sum(len(actions_of(d, "escalate_operator")) for d in world.decisions) + + +def _assert_fallback_once( + world: CauseWorld, key: str, park: ModelLandingDecision +) -> None: + members = [p.key for p in three()] + assert len(actions_of(park, "escalate_operator")) == 1 + assert [s for s in dispatched(park) if is_cause_subject(s)] == [] + assert sorted(per_pr_dispatched(park)) == members + for action in actions_of(park, "dispatch_worker"): + assert action.brief is not None + assert action.brief.engine.value == "claude_sonnet" + assert action.brief.brief_class.value == "real_red" + assert world.cause(key).parked_until is not None + for _ in range(3): # parked, members working: no cause worker, no second MSG + later = world.step() + assert dispatched(later) == [] + assert actions_of(later, "escalate_operator") == [] + assert _escalations(world) == 1 + + +@pytest.mark.unit +def test_fallback_a_cause_parked_by_two_failed_attempts_releases_its_members() -> None: + world = CauseWorld(three()) + key = key_for() + park = _park_by_two_failed_attempts(world, key) + assert world.cause(key).attempts == 2 + _assert_fallback_once(world, key, park) + + +@pytest.mark.unit +def test_fallback_a_cause_parked_by_spawn_failures_releases_its_members() -> None: + world = CauseWorld(three()) + key = key_for() + park = _park_by_spawn_failures(world, key) + assert (world.cause(key).attempts, world.cause(key).spawn_failures) == (0, 3) + _assert_fallback_once(world, key, park) + + +@pytest.mark.unit +def test_fallback_positive_control_a_live_cause_lease_still_holds_its_members() -> None: + world = CauseWorld(three()) + key = key_for() + assert dispatched(world.step()) == [key] + for _ in range(4): + decision = world.step() + assert world.lease(key).revoked is False + assert per_pr_dispatched(decision) == [] + assert decision.actions == () + assert world.cause(key).parked_until is None + + +def _parked_world( + key: str, policy: dict[str, Any] | None = None, prs: list[Pr] | None = None +) -> CauseWorld: + """A fresh world carrying a parked cause and no worker: the park, then its members.""" + parked = CauseWorld(prs or three(), policy=policy) + _park_by_spawn_failures(parked, key) + parked.state = parked.state.model_copy(update={"leases": ()}) + fresh = CauseWorld(prs or three(), policy=policy) + fresh.state = ModelLandingControllerState.model_validate( + {"causes": [c.model_dump(mode="json") for c in parked.state.causes]} + ) + fresh.now = parked.now + return fresh + + +@pytest.mark.unit +def test_fallback_a_parked_cause_with_an_owner_still_holds_its_members() -> None: + key = key_for() + world = _parked_world(key) + world.extra["cause_owners"] = [{"lane": "cause-lane", "repo": REPO, "cause": key}] + decision = world.step() + assert decision.actions == () + assert world.cause(key).parked_until is not None + + +@pytest.mark.unit +def test_fallback_a_parked_cause_without_an_owner_releases_members_at_once() -> None: + key = key_for() + world = _parked_world(key) + decision = world.step() + assert sorted(per_pr_dispatched(decision)) == [p.key for p in three()] + assert [s for s in dispatched(decision) if is_cause_subject(s)] == [] + assert actions_of(decision, "escalate_operator") == [] + + +@pytest.mark.unit +def test_fallback_a_release_row_ends_the_park_and_the_cause_holds_members_again() -> ( + None +): + world = CauseWorld(three()) + key = key_for() + park = _park_by_spawn_failures(world, key) + workers = per_pr_dispatched(park) + assert len(workers) == 3 + for pr in workers: # the fallback workers died; their next rung would dispatch + world.exit(pr) + # the release row is newer than the park: the cause redispatches and holds + world.extra["cause_releases"] = [ + {"cause": key, "at": (world.now + timedelta(seconds=900)).isoformat()} + ] + decision = world.step(advance=timedelta(seconds=900)) + assert dispatched(decision) == [key] + assert per_pr_dispatched(decision) == [] + assert world.cause(key).parked_until is None + assert decision.actions[0].cause_brief is not None + assert decision.actions[0].cause_brief.attempt == 1 + + +@pytest.mark.unit +def test_fallback_without_a_release_row_the_dead_fallback_workers_climb_the_ladder() -> ( + None +): + """Control for the release test: the same exits, no row, members redispatch.""" + world = CauseWorld(three()) + key = key_for() + workers = per_pr_dispatched(_park_by_spawn_failures(world, key)) + assert len(workers) == 3 + for pr in workers: + world.exit(pr) + decision = world.step(advance=timedelta(seconds=900)) + assert [s for s in dispatched(decision) if is_cause_subject(s)] == [] + assert sorted(per_pr_dispatched(decision)) == workers + + +@pytest.mark.unit +def test_fallback_per_pr_caps_bound_the_dispatches_of_a_big_cluster() -> None: + members = [Pr(n) for n in range(1, 9)] + key = key_for() + world = _parked_world(key, prs=members) + decision = world.step() + assert len(per_pr_dispatched(decision)) == 6 # max_workers_per_repo + + +@pytest.mark.unit +def test_fallback_the_worker_pool_bounds_the_dispatches() -> None: + key = key_for() + world = _parked_world(key, policy={"max_workers": 2}) + assert len(per_pr_dispatched(world.step())) == 2 + + +@pytest.mark.unit +def test_fallback_the_load_pause_stops_the_dispatches() -> None: + key = key_for() + world = _parked_world(key) + world.extra["load1"] = 25.0 + assert dispatched(world.step()) == [] + world.extra["load1"] = 0.0 + assert len(per_pr_dispatched(world.step())) == 3 + + +@pytest.mark.unit +def test_fallback_a_fixer_hold_dispatches_nothing_for_a_parked_cause() -> None: + key = key_for() + world = _parked_world(key) + world.extra["fixer_hold"] = [REPO] + assert dispatched(world.step()) == [] + world.extra["fixer_hold"] = [] + assert sorted(per_pr_dispatched(world.step())) == [p.key for p in three()] + + +@pytest.mark.unit +def test_fallback_a_parked_pr_stays_parked() -> None: + """Per-PR parks (parked_head) are not part of the ruling.""" + prs = three() + key = key_for() + world = _parked_world(key, prs=prs) + world.state = ModelLandingControllerState.model_validate( + { + "causes": [c.model_dump(mode="json") for c in world.state.causes], + "records": [parked_record(prs[0])], + } + ) + decision = world.step() + assert sorted(per_pr_dispatched(decision)) == [prs[1].key, prs[2].key] diff --git a/uv.lock b/uv.lock index bee96c2ad6..3b2a2214f1 100644 --- a/uv.lock +++ b/uv.lock @@ -3143,7 +3143,7 @@ wheels = [ [[package]] name = "omnimarket" -version = "0.4.290" +version = "0.4.291" source = { editable = "." } dependencies = [ { name = "aiokafka" },