Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 55 additions & 0 deletions src/omnimarket/configs/task_class_contracts.v1.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,18 @@ task_model_overrides:
research: "Qwen3.6-35B-A3B"
task_classes:
code_generation:
score_source: deterministic_acceptance
acceptance_authority:
type: deterministic_acceptance
acceptance_version: delegation-deterministic-acceptance.v1
evidence_fields:
- acceptance_version
- corpus_hash
- validator_or_artifact_hash
- acceptance_command
- actual_score
- pass
- failure_cases
required_capabilities:
- code_completion
- context_window_large
Expand Down Expand Up @@ -201,6 +213,18 @@ task_classes:
- cheap_cloud
- claude
test:
score_source: deterministic_acceptance
acceptance_authority:
type: deterministic_acceptance
acceptance_version: delegation-deterministic-acceptance.v1
evidence_fields:
- acceptance_version
- corpus_hash
- validator_or_artifact_hash
- acceptance_command
- actual_score
- pass
- failure_cases
required_capabilities:
- code_completion
pricing_ceiling_per_1k_tokens: 0.015
Expand All @@ -227,6 +251,37 @@ task_classes:
- local
- cheap_cloud
- claude
validator_generation:
score_source: deterministic_acceptance
acceptance_authority:
type: deterministic_acceptance
acceptance_version: node_generation_consumer.corpus_acceptance.v1
evidence_fields:
- acceptance_version
- corpus_hash
- validator_or_artifact_hash
- acceptance_command
- actual_score
- pass
- failure_cases
required_capabilities:
- code_completion
pricing_ceiling_per_1k_tokens: 0.015
latency_sla_p99_ms: 30000
cloud_routing_policy: allowed
definition_of_done:
deterministic:
- compiles_without_errors
- final_artifact_only
- passes_existing_tests
heuristic:
- no_refusal
escalation_policy:
max_escalations: 2
tier_order:
- local
- cheap_cloud
- claude
reasoning:
required_capabilities:
- reasoning
Expand Down
43 changes: 42 additions & 1 deletion src/omnimarket/models/delegation/wire/model_quality_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,13 @@ class ModelQualityGateInput(BaseModel):
class ModelQualityGateResult(BaseModel):
"""Gate output: pass/fail verdict, score, failure reasons, and fallback flag."""

model_config = ConfigDict(frozen=True, extra="forbid", from_attributes=True)
model_config = ConfigDict(
frozen=True,
extra="forbid",
from_attributes=True,
populate_by_name=True,
serialize_by_alias=True,
)

correlation_id: UUID = Field(
...,
Expand All @@ -94,6 +100,41 @@ class ModelQualityGateResult(BaseModel):
default=False,
description="Whether fallback to Claude is recommended.",
)
score_source: str = Field(
default="",
description="Authority that produced actual_score when applicable.",
)
acceptance_version: str = Field(
default="",
description="Deterministic acceptance evaluator version when applicable.",
)
corpus_hash: str = Field(
default="",
description="Stable hash of the deterministic acceptance corpus/check set.",
)
validator_or_artifact_hash: str = Field(
default="",
description="Stable hash of the validator or delegated artifact evaluated.",
)
acceptance_command: str = Field(
default="",
description="Replay command or command identity for deterministic acceptance.",
)
actual_score: float | None = Field(
default=None,
ge=0.0,
le=1.0,
description="Authority score produced by the acceptance source.",
)
pass_: bool | None = Field(
default=None,
alias="pass",
description="Authority pass/fail verdict produced by the acceptance source.",
)
failure_cases: tuple[str, ...] = Field(
default=(),
description="Deterministic acceptance failure cases.",
)


__all__: list[str] = [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@
from __future__ import annotations

import ast
import hashlib
import json
import re
from collections.abc import Callable

Expand Down Expand Up @@ -72,6 +74,12 @@
"TASK_MISMATCH",
)

_ACCEPTANCE_VERSION = "delegation-deterministic-acceptance.v1"
_DETERMINISTIC_SCORE_SOURCE = "deterministic_acceptance"
_VERIFIABLE_TASK_TYPES: frozenset[str] = frozenset(
{"code_generation", "test", "validator_generation"}
)


def _recommends_fallback(failure_reasons: tuple[str, ...] | list[str]) -> bool:
"""Return whether any failure reason carries an escalation-worthy verdict.
Expand Down Expand Up @@ -689,6 +697,73 @@ def _run_contract_checks(
return det_failures, heuristic_failures


def _stable_hash(value: object) -> str:
"""Return a stable sha256 hash for acceptance replay identity."""
encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode()
return hashlib.sha256(encoded).hexdigest()


def _artifact_hash(content: str) -> str:
"""Return the stable hash of the evaluated delegated artifact."""
return hashlib.sha256(content.encode()).hexdigest()


def _deterministic_acceptance_score(
deterministic_total: int,
deterministic_failures: int,
) -> float:
"""Score deterministic acceptance by passed checks over total checks."""
if deterministic_total <= 0:
return 0.0
passed = max(0, deterministic_total - deterministic_failures)
return round(passed / deterministic_total, 3)


def _deterministic_acceptance_evidence(
*,
task_type: str,
content: str,
dod_deterministic: tuple[str, ...],
deterministic_failures: list[str],
) -> dict[str, object]:
"""Build deterministic acceptance evidence for verifiable task classes."""
deterministic_total = max(len(dod_deterministic), len(deterministic_failures))
actual_score = _deterministic_acceptance_score(
deterministic_total, len(deterministic_failures)
)
passed = deterministic_total > 0 and not deterministic_failures
corpus_identity = {
"acceptance_version": _ACCEPTANCE_VERSION,
"score_source": _DETERMINISTIC_SCORE_SOURCE,
"task_type": task_type,
"checks": dod_deterministic,
}
acceptance_command = (
"uv run python -m "
"omnimarket.nodes.node_delegation_quality_gate_reducer "
f"--score-source={_DETERMINISTIC_SCORE_SOURCE} "
f"--task-type={task_type}"
)
return {
"score_source": _DETERMINISTIC_SCORE_SOURCE,
"acceptance_version": _ACCEPTANCE_VERSION,
"corpus_hash": _stable_hash(corpus_identity),
"validator_or_artifact_hash": _artifact_hash(content),
"acceptance_command": acceptance_command,
"actual_score": actual_score,
"pass_": passed,
"failure_cases": tuple(deterministic_failures),
}


def _is_verifiable_deterministic_acceptance(
gate_input: ModelQualityGateInput,
dod_deterministic: tuple[str, ...],
) -> bool:
"""Return whether this contract path uses deterministic acceptance authority."""
return gate_input.task_type in _VERIFIABLE_TASK_TYPES and bool(dod_deterministic)


# Relative weighting of the two DoD bands when computing the graded quality
# score (OMN-12964). Deterministic checks gate harder, so a deterministic miss
# costs more than a heuristic miss, but neither band collapses the score to a
Expand Down Expand Up @@ -843,8 +918,8 @@ def _run_legacy_checks(
# same prefixes (see failure_reasons above), so WEAK_OUTPUT (length miss) and
# TASK_MISMATCH (missing markers) now escalate instead of terminating — the
# prior score-threshold gate (quality_score < 0.3) silently dropped them.
fallback_recommended = _recommends_fallback(failure_reasons)
fail_category: EnumQualityGateCategory = EnumQualityGateCategory.FAIL_HEURISTIC
fallback_recommended = not passed and _recommends_fallback(failure_reasons)
fail_category: EnumQualityGateCategory = "pass" if passed else "fail_heuristic"

return ModelQualityGateResult(
correlation_id=gate_input.correlation_id,
Expand Down Expand Up @@ -894,6 +969,19 @@ def delta(gate_input: ModelQualityGateInput) -> ModelQualityGateResult:
det_failures, heuristic_failures = _run_contract_checks(
content, dod_deterministic, dod_heuristic
)
deterministic_acceptance_authority = _is_verifiable_deterministic_acceptance(
gate_input, dod_deterministic
)
acceptance_evidence = (
_deterministic_acceptance_evidence(
task_type=gate_input.task_type,
content=content,
dod_deterministic=dod_deterministic,
deterministic_failures=det_failures,
)
if deterministic_acceptance_authority
else {}
)

all_failures = det_failures + heuristic_failures

Expand Down Expand Up @@ -923,6 +1011,18 @@ def delta(gate_input: ModelQualityGateInput) -> ModelQualityGateResult:
quality_score=quality_score,
failure_reasons=tuple(all_failures),
fallback_recommended=True,
**acceptance_evidence,
)

if deterministic_acceptance_authority:
return ModelQualityGateResult(
correlation_id=gate_input.correlation_id,
passed=True,
fail_category="pass",
quality_score=quality_score,
failure_reasons=(),
fallback_recommended=False,
**acceptance_evidence,
)

if heuristic_failures:
Expand All @@ -938,6 +1038,7 @@ def delta(gate_input: ModelQualityGateInput) -> ModelQualityGateResult:
quality_score=quality_score,
failure_reasons=tuple(heuristic_failures),
fallback_recommended=fallback_recommended,
**acceptance_evidence,
)

if not _has_adequacy_authority(dod_deterministic, dod_heuristic):
Expand All @@ -957,6 +1058,7 @@ def delta(gate_input: ModelQualityGateInput) -> ModelQualityGateResult:
quality_score=quality_score,
failure_reasons=(),
fallback_recommended=False,
**acceptance_evidence,
)


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,14 @@
# SPDX-License-Identifier: MIT

# Copyright (c) 2026 OmniNode Team
"""Compatibility imports for canonical quality gate result DTOs."""
"""Compatibility import for the OMN-13366 quality gate result DTO.

from omnibase_core.models.delegation.wire import (
``ModelQualityGateInput`` stays canonical in omnibase_core. The result carries
omnimarket P1 deterministic acceptance evidence fields until those fields are
promoted into the shared core wire DTO.
"""

from omnimarket.models.delegation.wire.model_quality_gate import (
EnumQualityGateCategory,
ModelQualityGateResult,
)
Expand Down
Loading
Loading