diff --git a/Makefile b/Makefile index f4e582fc..dc4b4763 100644 --- a/Makefile +++ b/Makefile @@ -58,6 +58,7 @@ eval-v1: --golden-set "$(GOLDEN_SET)" \ --lock "$(GOLDEN_LOCK)" \ --run "$(EVAL_RUN)" \ + $(if $(GOLDEN_LINEAGE_MAP),--lineage-map "$(GOLDEN_LINEAGE_MAP)",) \ --output .context-engine/eval/golden-v1-report.json \ --generated-at "$(GENERATED_AT)" @@ -66,6 +67,7 @@ eval-v1-execute: --golden-set "$(GOLDEN_SET)" \ --lock "$(GOLDEN_LOCK)" \ --judgments "$(EVAL_JUDGMENTS)" \ + $(if $(GOLDEN_LINEAGE_MAP),--lineage-map "$(GOLDEN_LINEAGE_MAP)",) \ --output .context-engine/eval/golden-v1-report.json \ --generated-at "$(GENERATED_AT)" diff --git a/applications/eval_v1.py b/applications/eval_v1.py index 9c9a1772..03a97af8 100644 --- a/applications/eval_v1.py +++ b/applications/eval_v1.py @@ -8,20 +8,40 @@ from collections.abc import Sequence from datetime import datetime from pathlib import Path +from typing import cast +from uuid import UUID +from engine.learning.comparison import ( + EvaluationComparisonUnavailable, + compare_release_evaluations, +) +from engine.learning.curation_candidate import ( + CurationCandidateUnavailable, + EvaluationCaseIntake, + build_curation_candidate, + curation_candidate_document, +) from engine.learning.eval_run import ( EvaluationRunUnavailable, + bind_evaluation_report_to_release, build_evaluation_report, load_evaluation_run, - record_lineage_check, +) +from engine.learning.feedback import ( + FeedbackBindingUnavailable, + FeedbackEvidence, + TriageCategory, + triage_feedback, ) from engine.learning.golden import ( GoldenSet, GoldenSetUnavailable, create_golden_lock, + load_golden_case, load_golden_set, relock_golden_set, ) +from engine.learning.golden_intake import admit_evaluation_case from engine.learning.golden_storage import ( durable_golden_root, require_durable_golden_path, @@ -40,6 +60,12 @@ require_resolved_lineage, ) from engine.learning.thresholds import DEFAULT_THRESHOLDS_PATH, load_thresholds +from engine.persistence import ( + DatabasePurpose, + PostgreSQLFeedbackInbox, + create_database_engine, + load_database_configuration, +) PUBLIC_SUBSET_MAINTAINER_SECRET_ENV = ( "CONTEXT_ENGINE_PUBLIC_SUBSET_MAINTAINER_SECRET" @@ -154,6 +180,24 @@ def _parser() -> argparse.ArgumentParser: execute.add_argument("--lineage-map", type=Path) execute.add_argument("--output", required=True, type=Path) execute.add_argument("--generated-at", required=True, type=_time) + + candidate = commands.add_parser("feedback-candidate") + candidate.add_argument("--organization-id", required=True, type=UUID) + candidate.add_argument("--feedback-ref", required=True) + candidate.add_argument("--category", required=True, choices=tuple(TriageCategory)) + candidate.add_argument("--case", required=True, type=Path) + candidate.add_argument("--output", required=True, type=Path) + candidate.add_argument("--proposed-at", required=True, type=_time) + + intake = commands.add_parser("feedback-intake") + intake.add_argument("--candidate", required=True, type=Path) + intake.add_argument("--golden-set", required=True, type=Path) + intake.add_argument("--lock", required=True, type=Path) + + compare = commands.add_parser("compare-releases") + compare.add_argument("--active-report", required=True, type=Path) + compare.add_argument("--candidate-report", required=True, type=Path) + compare.add_argument("--output", required=True, type=Path) return parser @@ -168,6 +212,82 @@ def _write_report(path: Path, report: dict[str, object]) -> None: print(f"golden v1 report written: digest={report['reportDigest']}", flush=True) +def _load_json(path: Path, name: str) -> object: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, ValueError): + raise ValueError(f"{name} is unavailable") from None + + +def _write_json(path: Path, document: dict[str, object]) -> None: + try: + _require_ignored_output(path) + path.parent.mkdir(parents=True, exist_ok=True) + _require_ignored_output(path) + path.write_text( + json.dumps(document, ensure_ascii=False, indent=2, sort_keys=True) + + "\n", + encoding="utf-8", + ) + except OSError: + raise ValueError("evaluation output is unavailable") from None + + +def _require_private_candidate_path(path: Path) -> None: + try: + _require_ignored_output(path) + except ValueError: + root = durable_golden_root() + require_durable_golden_path(path, root=root) + + +def _write_private_candidate(path: Path, document: dict[str, object]) -> None: + try: + _require_private_candidate_path(path) + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chmod(path.parent, 0o700) + flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path, flags, 0o600) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + descriptor = -1 + handle.write( + json.dumps( + document, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + + "\n" + ) + handle.flush() + os.fsync(handle.fileno()) + finally: + if descriptor >= 0: + os.close(descriptor) + except OSError: + raise ValueError("curation candidate output is unavailable") from None + + +def _captured_feedback( + organization_id: UUID, + feedback_ref: str, +) -> FeedbackEvidence: + engine = create_database_engine( + load_database_configuration(DatabasePurpose.LEARNING) + ) + try: + return PostgreSQLFeedbackInbox(engine).find_exact( + organization_id, + feedback_ref, + ) + finally: + engine.dispose() + + def _resolved_lineage_check( golden_set: GoldenSet, lineage_map_path: Path | None, @@ -188,16 +308,65 @@ def _record_lineage_check( ) -> dict[str, object]: if lineage_check is None: return report - return record_lineage_check(report, lineage_check) + return bind_evaluation_report_to_release(report, lineage_check) def main(argv: Sequence[str] | None = None) -> None: parser = _parser() args = parser.parse_args(argv) try: + if args.command == "feedback-candidate": + _require_private_candidate_path(args.case) + case_document = _load_json(args.case, "evaluation case intake") + candidate = build_curation_candidate( + triage_feedback( + _captured_feedback(args.organization_id, args.feedback_ref), + TriageCategory(args.category), + ), + EvaluationCaseIntake( + case=load_golden_case(case_document), + synthetic=False, + ), + proposed_at=args.proposed_at, + ) + _write_private_candidate( + args.output, + curation_candidate_document(candidate), + ) + print( + f"curation candidate written: digest={candidate.candidate_digest}", + flush=True, + ) + return + if args.command == "compare-releases": + comparison = compare_release_evaluations( + _load_json(args.active_report, "active evaluation report"), + _load_json(args.candidate_report, "candidate evaluation report"), + ) + _write_json(args.output, comparison) + slices = cast(list[object], comparison["slices"]) + print( + "release evaluation comparison written: " + f"slices={len(slices)}", + flush=True, + ) + return durable_root = durable_golden_root() require_durable_golden_path(args.golden_set, root=durable_root) require_durable_golden_path(args.lock, root=durable_root) + if args.command == "feedback-intake": + _require_private_candidate_path(args.candidate) + receipt = admit_evaluation_case( + args.candidate, + golden_path=args.golden_set, + lock_path=args.lock, + ) + print( + "golden v1 feedback case admitted: " + f"cases={receipt.case_count} digest={receipt.golden_digest}", + flush=True, + ) + return lineage_map_path = getattr(args, "lineage_map", None) if lineage_map_path is not None: require_durable_golden_path(lineage_map_path, root=durable_root) @@ -293,7 +462,10 @@ def main(argv: Sequence[str] | None = None) -> None: return except ( GoldenSetUnavailable, + CurationCandidateUnavailable, EvaluationRunUnavailable, + EvaluationComparisonUnavailable, + FeedbackBindingUnavailable, LineageMapUnavailable, StaleGoldenLineage, ValueError, diff --git a/engine/learning/__init__.py b/engine/learning/__init__.py index baa23503..70b1c6b9 100644 --- a/engine/learning/__init__.py +++ b/engine/learning/__init__.py @@ -1,4 +1,4 @@ -"""Public ContextLearning contracts and sole release owner.""" +"""Public ContextLearning release contracts and sole release owner.""" from engine.learning.contracts import ( ContentProfileRef, diff --git a/engine/learning/comparison.py b/engine/learning/comparison.py new file mode 100644 index 00000000..c17d4015 --- /dev/null +++ b/engine/learning/comparison.py @@ -0,0 +1,230 @@ +"""Authoritative candidate-versus-active Release slice comparison.""" + +from __future__ import annotations + +import json +from hashlib import sha256 +from re import fullmatch +from typing import cast + +COMPARISON_REPORT_VERSION = "context-engine-release-comparison-v1" +_OBSERVED_STATUSES = frozenset( + {"PASS", "FAIL", "INSUFFICIENT_DATA", "PENDING_PREREGISTRATION"} +) +_LAYERS = ("answer", "citation", "retrieval") +_SLICES = ("cross_doc", "single_doc", "temporal") +_SLICE_STATUSES = frozenset( + {"pass", "fail", "insufficient_data", "pending_preregistration"} +) + + +class EvaluationComparisonUnavailable(RuntimeError): + """Reports cannot support an authoritative comparison verdict.""" + + +def _object(value: object, name: str) -> dict[str, object]: + if type(value) is not dict: + raise EvaluationComparisonUnavailable(f"comparison {name} is unavailable") + return cast(dict[str, object], value) + + +def _text(value: object, name: str) -> str: + if type(value) is not str or not value or value.isspace(): + raise EvaluationComparisonUnavailable(f"comparison {name} is unavailable") + return value + + +def _digest(document: object) -> str: + return sha256( + json.dumps( + document, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + ).hexdigest() + + +def _require_authoritative(report: object, side: str) -> dict[str, object]: + document = _object(report, f"{side} report") + if document.get("reportVersion") != "context-engine-eval-report-v1": + raise EvaluationComparisonUnavailable(f"{side} report is not authoritative") + report_digest = document.get("reportDigest") + digest_input = { + key: value for key, value in document.items() if key != "reportDigest" + } + if ( + type(report_digest) is not str + or fullmatch(r"[0-9a-f]{64}", report_digest) is None + or report_digest != _digest(digest_input) + ): + raise EvaluationComparisonUnavailable(f"{side} report is not authoritative") + if document.get("status") not in _OBSERVED_STATUSES: + raise EvaluationComparisonUnavailable( + f"{side} report is not authoritative" + ) + security = _object(document.get("security"), f"{side} security") + security_state = security.get("observationState") + counts = tuple( + security.get(name) + for name in ( + "missingContextFallbackCount", + "unauthorizedEvidenceCount", + "wrongOrganizationEffectCount", + ) + ) + if ( + security_state not in {"observed_clean", "observed_violation"} + or any(type(value) is not int or value < 0 for value in counts) + or ( + security_state == "observed_clean" + and (security.get("status") != "pass" or counts != (0, 0, 0)) + ) + or ( + security_state == "observed_violation" + and ( + security.get("status") != "fail" + or not any(cast(int, value) > 0 for value in counts) + or document.get("status") != "FAIL" + ) + ) + ): + raise EvaluationComparisonUnavailable( + f"{side} report is not authoritative" + ) + if document.get("thresholdAuthority") != "tracked": + raise EvaluationComparisonUnavailable( + f"{side} report is not authoritative" + ) + run = _object(document.get("run"), f"{side} run") + _text(run.get("executedSeamRef"), f"{side} executed seam") + lineage = _object(document.get("lineageCheck"), f"{side} lineage") + if lineage.get("ran") is not True or lineage.get("staleCaseCount") != 0: + raise EvaluationComparisonUnavailable( + f"{side} lineage is not authoritative" + ) + release = _object(document.get("release"), f"{side} release") + _text(release.get("releaseRef"), f"{side} releaseRef") + return document + + +def _slice_rows( + report: dict[str, object], + side: str, +) -> dict[tuple[str, str], dict[str, object]]: + slices = _object(report.get("slices"), f"{side} slices") + rows: dict[tuple[str, str], dict[str, object]] = {} + for layer in _LAYERS: + values = slices.get(layer) + if type(values) is not list: + raise EvaluationComparisonUnavailable( + f"comparison {side} {layer} slices are unavailable" + ) + for value in cast(list[object], values): + row = _object(value, f"{side} {layer} slice") + slice_name = row.get("slice_name") + if slice_name not in _SLICES: + raise EvaluationComparisonUnavailable( + f"comparison {side} slice is unavailable" + ) + key = (layer, cast(str, slice_name)) + if key in rows: + raise EvaluationComparisonUnavailable( + f"comparison {side} slices are duplicated" + ) + if type(row.get("case_count")) is not int: + raise EvaluationComparisonUnavailable( + f"comparison {side} slice count is unavailable" + ) + case_count = cast(int, row["case_count"]) + score = row.get("score") + status = row.get("status") + if ( + case_count < 0 + or (score is not None and type(score) is not float) + or (type(score) is float and not 0.0 <= score <= 1.0) + or status not in _SLICE_STATUSES + or (case_count == 0 and score is not None) + or (status in {"pass", "fail"} and score is None) + ): + raise EvaluationComparisonUnavailable( + f"comparison {side} slice observation is unavailable" + ) + rows[key] = row + expected = frozenset( + (layer, slice_name) for layer in _LAYERS for slice_name in _SLICES + ) + if frozenset(rows) != expected: + raise EvaluationComparisonUnavailable( + f"comparison {side} slice set is unavailable" + ) + return rows + + +def compare_release_evaluations( + active_report: object, + candidate_report: object, +) -> dict[str, object]: + """Compare exact authoritative reports without activating either Release.""" + + active = _require_authoritative(active_report, "active") + candidate = _require_authoritative(candidate_report, "candidate") + active_golden = _object(active.get("goldenSet"), "active golden set") + candidate_golden = _object(candidate.get("goldenSet"), "candidate golden set") + golden_digest = _text(active_golden.get("digest"), "active golden digest") + if candidate_golden.get("digest") != golden_digest: + raise EvaluationComparisonUnavailable( + "comparison reports use different golden sets" + ) + active_rows = _slice_rows(active, "active") + candidate_rows = _slice_rows(candidate, "candidate") + if frozenset(active_rows) != frozenset(candidate_rows): + raise EvaluationComparisonUnavailable("comparison slice sets differ") + comparisons: list[dict[str, object]] = [] + for layer, slice_name in sorted(active_rows): + active_row = active_rows[(layer, slice_name)] + candidate_row = candidate_rows[(layer, slice_name)] + if active_row["case_count"] != candidate_row["case_count"]: + raise EvaluationComparisonUnavailable( + "comparison reports use different slice populations" + ) + active_score = active_row["score"] + candidate_score = candidate_row["score"] + delta = ( + None + if active_score is None or candidate_score is None + else round(cast(float, candidate_score) - cast(float, active_score), 12) + ) + comparisons.append( + { + "activeCaseCount": active_row["case_count"], + "activeScore": active_score, + "activeStatus": active_row["status"], + "candidateCaseCount": candidate_row["case_count"], + "candidateScore": candidate_score, + "candidateStatus": candidate_row["status"], + "delta": delta, + "layer": layer, + "slice": slice_name, + } + ) + active_release = _object(active["release"], "active release") + candidate_release = _object(candidate["release"], "candidate release") + pending = "PENDING_PREREGISTRATION" in { + active["status"], + candidate["status"], + } + comparison: dict[str, object] = { + "activeReleaseRef": active_release["releaseRef"], + "activeReportDigest": active["reportDigest"], + "activeReportStatus": active["status"], + "candidateReleaseRef": candidate_release["releaseRef"], + "candidateReportDigest": candidate["reportDigest"], + "candidateReportStatus": candidate["status"], + "goldenSetDigest": golden_digest, + "schemaVersion": COMPARISON_REPORT_VERSION, + "slices": comparisons, + "status": "PENDING_PREREGISTRATION" if pending else "compared", + } + comparison["reportDigest"] = _digest(comparison) + return comparison diff --git a/engine/learning/curation_candidate.py b/engine/learning/curation_candidate.py new file mode 100644 index 00000000..7aa161bd --- /dev/null +++ b/engine/learning/curation_candidate.py @@ -0,0 +1,336 @@ +"""Governed evaluation-case candidates produced from triaged feedback.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from hashlib import sha256 +from re import fullmatch +from typing import Final, cast +from uuid import UUID + +from engine.learning.feedback import ( + FeedbackBinding, + FeedbackCitation, + TriageCategory, + TriagedFeedback, +) +from engine.learning.golden import EvidenceLineage, GoldenCase, load_golden_case + +CURATION_CANDIDATE_VERSION: Final = "context-engine-curation-candidate-v1" +_PLACEHOLDER_PREFIXES: Final = ( + "synthetic-", + "synthetic/", + "placeholder-", + "placeholder/", +) + + +class CurationCandidateUnavailable(RuntimeError): + """A candidate cannot cross the exact-lineage or privacy boundary.""" + + +def _instant(value: object) -> datetime: + if ( + type(value) is not datetime + or value.tzinfo is None + or value.utcoffset() != timedelta(0) + ): + raise ValueError("curation candidate time must be aware UTC") + return value + + +def _timestamp(value: datetime) -> str: + return _instant(value).astimezone(UTC).isoformat(timespec="microseconds").replace( + "+00:00", "Z" + ) + + +def _digest(document: object) -> str: + return sha256( + json.dumps( + document, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + ).hexdigest() + + +def _placeholder(field_name: str, value: object) -> None: + if type(value) is not str or not value.startswith(_PLACEHOLDER_PREFIXES): + raise CurationCandidateUnavailable( + f"evaluation case {field_name} requires placeholder content" + ) + + +def _require_synthetic_case(case: GoldenCase) -> None: + _placeholder("caseRef", case.case_ref) + _placeholder("query", case.query) + _placeholder("expectedAnswer", case.expected_answer) + _placeholder("topicCluster", case.topic_cluster) + for expectation in case.expected_evidence: + _placeholder("path", expectation.path) + for field_name, value in expectation.lineage.document().items(): + _placeholder(field_name, value) + for claim in case.required_claims: + _placeholder("claimRef", claim.claim_ref) + _placeholder("claim", claim.claim) + for lineage in claim.expected_evidence: + for field_name, value in lineage.document().items(): + _placeholder(field_name, value) + for negative in case.hard_negative_evidence: + _placeholder("path", negative.path) + _placeholder("topicCluster", negative.topic_cluster) + for field_name, value in negative.lineage.document().items(): + _placeholder(field_name, value) + + +@dataclass(frozen=True, slots=True) +class EvaluationCaseIntake: + """A private or explicit-placeholder v1 case proposed for governed intake.""" + + case: GoldenCase = field(repr=False) + synthetic: bool + + def __post_init__(self) -> None: + if type(self.case) is not GoldenCase: + raise TypeError("evaluation intake requires GoldenCase") + if type(self.synthetic) is not bool: + raise TypeError("evaluation case synthetic marker must be bool") + if self.synthetic: + _require_synthetic_case(self.case) + + +@dataclass(frozen=True, slots=True) +class CurationCandidate: + """Immutable proposal; it has no ReleaseManifest publication operation.""" + + feedback_ref: str + category: TriageCategory + feedback_binding: FeedbackBinding = field(repr=False) + evaluation_case: GoldenCase = field(repr=False) + proposed_at: datetime + candidate_ref: str = field(init=False) + candidate_digest: str = field(init=False, repr=False) + + def __post_init__(self) -> None: + if type(self.category) is not TriageCategory: + raise TypeError("curation candidate category is unavailable") + if type(self.feedback_binding) is not FeedbackBinding: + raise TypeError("curation candidate feedback binding is unavailable") + _instant(self.proposed_at) + document = curation_candidate_document(self, include_identity=False) + digest = _digest(document) + object.__setattr__(self, "candidate_digest", digest) + object.__setattr__(self, "candidate_ref", f"cur_{digest}") + + @property + def base_release_ref(self) -> str: + return self.feedback_binding.release_ref + + @property + def base_release_generation(self) -> int: + return self.feedback_binding.release_generation + + +def curation_candidate_document( + candidate: CurationCandidate, + *, + include_identity: bool = True, +) -> dict[str, object]: + """Render the closed candidate document with no denied object details.""" + + if type(candidate) is not CurationCandidate: + raise TypeError("candidate must be CurationCandidate") + document: dict[str, object] = { + "baseReleaseGeneration": candidate.base_release_generation, + "baseReleaseRef": candidate.base_release_ref, + "category": candidate.category.value, + "evaluationCase": candidate.evaluation_case.document(), + "feedbackBinding": { + "citations": [ + { + "evidenceRef": citation.evidence_ref, + **citation.lineage.document(), + } + for citation in candidate.feedback_binding.citations + ], + "organizationId": str(candidate.feedback_binding.organization_id), + "packageDigest": candidate.feedback_binding.package_digest, + "packageRef": candidate.feedback_binding.package_ref, + "releaseGeneration": candidate.feedback_binding.release_generation, + "releaseRef": candidate.feedback_binding.release_ref, + "runRef": candidate.feedback_binding.run_ref, + }, + "feedbackRef": candidate.feedback_ref, + "proposedAt": _timestamp(candidate.proposed_at), + "schemaVersion": CURATION_CANDIDATE_VERSION, + } + if include_identity: + document["candidateDigest"] = candidate.candidate_digest + document["candidateRef"] = candidate.candidate_ref + return document + + +def curation_candidate_case(document: object) -> GoldenCase: + """Verify one immutable candidate document and return its proposed case.""" + + expected_fields = frozenset( + { + "baseReleaseGeneration", + "baseReleaseRef", + "candidateDigest", + "candidateRef", + "category", + "evaluationCase", + "feedbackBinding", + "feedbackRef", + "proposedAt", + "schemaVersion", + } + ) + if type(document) is not dict or frozenset(document) != expected_fields: + raise CurationCandidateUnavailable("curation candidate is malformed") + candidate_document = cast(dict[str, object], document) + if candidate_document["schemaVersion"] != CURATION_CANDIDATE_VERSION: + raise CurationCandidateUnavailable("curation candidate version is unavailable") + candidate_digest = candidate_document["candidateDigest"] + candidate_ref = candidate_document["candidateRef"] + digest_input = { + key: value + for key, value in candidate_document.items() + if key not in {"candidateDigest", "candidateRef"} + } + if ( + type(candidate_digest) is not str + or candidate_digest != _digest(digest_input) + or candidate_ref != f"cur_{candidate_digest}" + ): + raise CurationCandidateUnavailable("curation candidate identity is unavailable") + binding = candidate_document["feedbackBinding"] + binding_fields = frozenset( + { + "citations", + "organizationId", + "packageDigest", + "packageRef", + "releaseGeneration", + "releaseRef", + "runRef", + } + ) + if type(binding) is not dict or frozenset(binding) != binding_fields: + raise CurationCandidateUnavailable("curation candidate binding is unavailable") + binding_document = cast(dict[str, object], binding) + citations = binding_document["citations"] + if type(citations) is not list: + raise CurationCandidateUnavailable( + "curation candidate citations are unavailable" + ) + projected: list[FeedbackCitation] = [] + citation_fields = frozenset( + { + "evidenceRef", + "fragmentRef", + "resourceRef", + "revisionRef", + "sourceRef", + } + ) + try: + for value in cast(list[object], citations): + if type(value) is not dict or frozenset(value) != citation_fields: + raise ValueError + citation = cast(dict[str, object], value) + projected.append( + FeedbackCitation( + evidence_ref=cast(str, citation["evidenceRef"]), + lineage=EvidenceLineage( + source_ref=cast(str, citation["sourceRef"]), + resource_ref=cast(str, citation["resourceRef"]), + revision_ref=cast(str, citation["revisionRef"]), + fragment_ref=cast(str, citation["fragmentRef"]), + ), + ) + ) + feedback_binding = FeedbackBinding( + organization_id=UUID(cast(str, binding_document["organizationId"])), + run_ref=cast(str, binding_document["runRef"]), + package_ref=cast(str, binding_document["packageRef"]), + package_digest=cast(str, binding_document["packageDigest"]), + release_ref=cast(str, binding_document["releaseRef"]), + release_generation=cast(int, binding_document["releaseGeneration"]), + citations=tuple(projected), + ) + feedback_ref = candidate_document["feedbackRef"] + if ( + type(feedback_ref) is not str + or fullmatch(r"fb_[0-9a-f]{64}", feedback_ref) is None + ): + raise ValueError + TriageCategory(cast(str, candidate_document["category"])) + proposed_at = candidate_document["proposedAt"] + if type(proposed_at) is not str: + raise ValueError + parsed_at = datetime.fromisoformat(proposed_at.replace("Z", "+00:00")) + if _timestamp(parsed_at) != proposed_at: + raise ValueError + except (AttributeError, TypeError, ValueError, RuntimeError): + raise CurationCandidateUnavailable( + "curation candidate fields are unavailable" + ) from None + if ( + feedback_binding.release_ref != candidate_document["baseReleaseRef"] + or feedback_binding.release_generation + != candidate_document["baseReleaseGeneration"] + ): + raise CurationCandidateUnavailable("curation candidate binding is inconsistent") + case = load_golden_case(candidate_document["evaluationCase"]) + _require_case_binding(case, feedback_binding) + return case + + +def _require_case_binding(case: GoldenCase, binding: FeedbackBinding) -> None: + expected_lineage = frozenset( + expectation.lineage for expectation in case.expected_evidence + ) + claim_lineage = frozenset( + lineage + for claim in case.required_claims + for lineage in claim.expected_evidence + ) + bound = frozenset(citation.lineage for citation in binding.citations) + if case.answerability == "answerable" and ( + not expected_lineage or not case.required_claims + ): + raise CurationCandidateUnavailable( + "answerable evaluation case requires expected citation lineage" + ) + if not expected_lineage.issubset(bound) or not claim_lineage.issubset(bound): + raise CurationCandidateUnavailable( + "evaluation case citation lineage is outside the feedback binding" + ) + + +def build_curation_candidate( + feedback: TriagedFeedback, + intake: EvaluationCaseIntake, + *, + proposed_at: datetime, +) -> CurationCandidate: + """Propose one case without changing golden locks or a Release pointer.""" + + if type(feedback) is not TriagedFeedback: + raise TypeError("curation requires TriagedFeedback") + if type(intake) is not EvaluationCaseIntake: + raise TypeError("curation requires EvaluationCaseIntake") + _require_case_binding(intake.case, feedback.binding) + return CurationCandidate( + feedback_ref=feedback.feedback_ref, + category=feedback.category, + feedback_binding=feedback.binding, + evaluation_case=intake.case, + proposed_at=proposed_at, + ) diff --git a/engine/learning/eval_run.py b/engine/learning/eval_run.py index e97bcb28..4894a074 100644 --- a/engine/learning/eval_run.py +++ b/engine/learning/eval_run.py @@ -313,6 +313,19 @@ def record_lineage_check( return updated +def bind_evaluation_report_to_release( + report: dict[str, object], + lineage_check: LineageResolutionReport, +) -> dict[str, object]: + """Bind an evaluated corpus to the exact Release its lineage map names.""" + + updated = record_lineage_check(report, lineage_check) + updated["release"] = {"releaseRef": lineage_check.release_ref} + updated.pop("reportDigest", None) + updated["reportDigest"] = _report_digest(updated) + return updated + + def build_evaluation_report( golden_set: GoldenSet, run: EvaluationRun, diff --git a/engine/learning/feedback.py b/engine/learning/feedback.py new file mode 100644 index 00000000..a977e251 --- /dev/null +++ b/engine/learning/feedback.py @@ -0,0 +1,351 @@ +"""Exact authorized lineage for feedback triage inputs.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from enum import StrEnum +from pathlib import Path +from re import fullmatch +from typing import TYPE_CHECKING, Literal, cast +from uuid import UUID + +from engine.learning.golden import EvidenceLineage + +if TYPE_CHECKING: + from engine.persistence.context_runs import ContextRunView + +_MAX_SIGNED_BIGINT = (1 << 63) - 1 + + +class FeedbackBindingUnavailable(RuntimeError): + """Captured feedback cannot be bound to one exact delivered Package.""" + + +class TriageCategory(StrEnum): + """Closed operator diagnosis assigned after exact lineage binding.""" + + SOURCE = "source" + VISIBILITY = "visibility" + RETRIEVAL = "retrieval" + ASSEMBLY = "assembly" + EVALUATION = "evaluation" + + +def _opaque(field_name: str, value: object, pattern: str) -> str: + if type(value) is not str or fullmatch(pattern, value) is None: + raise FeedbackBindingUnavailable(f"feedback {field_name} is unavailable") + return value + + +def _instant(value: object) -> datetime: + if ( + type(value) is not datetime + or value.tzinfo is None + or value.utcoffset() != timedelta(0) + ): + raise ValueError("feedback recorded_at must be aware UTC") + return value + + +@dataclass(frozen=True, slots=True) +class FeedbackCitation: + """One exact delivered Evidence identity and content-free lineage.""" + + evidence_ref: str + lineage: EvidenceLineage + + def __post_init__(self) -> None: + _opaque("citation evidence_ref", self.evidence_ref, r"ev_[0-9a-f]{64}") + if type(self.lineage) is not EvidenceLineage: + raise TypeError("feedback citation requires EvidenceLineage") + + +@dataclass(frozen=True, slots=True) +class FeedbackBinding: + """Exact run, Package, Release generation, and citation lineage.""" + + organization_id: UUID = field(repr=False) + run_ref: str + package_ref: str + package_digest: str = field(repr=False) + release_ref: str + release_generation: int + citations: tuple[FeedbackCitation, ...] = field(repr=False) + + def __post_init__(self) -> None: + if type(self.organization_id) is not UUID: + raise TypeError("feedback Organization must be UUID") + _opaque("run_ref", self.run_ref, r"run_[0-9a-f]{32}") + _opaque("package_ref", self.package_ref, r"pkg_[0-9a-f]{32}") + _opaque("package_digest", self.package_digest, r"[0-9a-f]{64}") + _opaque("release_ref", self.release_ref, r"rel_[0-9a-f]{64}") + if ( + type(self.release_generation) is not int + or not 1 <= self.release_generation <= _MAX_SIGNED_BIGINT + ): + raise FeedbackBindingUnavailable( + "feedback Release generation is unavailable" + ) + if ( + type(self.citations) is not tuple + or not self.citations + or any(type(value) is not FeedbackCitation for value in self.citations) + or len(self.citations) != len(set(self.citations)) + or len({value.evidence_ref for value in self.citations}) + != len(self.citations) + or len({value.lineage for value in self.citations}) + != len(self.citations) + ): + raise FeedbackBindingUnavailable( + "feedback citation lineage is unavailable" + ) + + +@dataclass(frozen=True, slots=True) +class FeedbackEvidence: + """Captured feedback plus its exact authorized-only delivery binding.""" + + feedback_ref: str + binding: FeedbackBinding = field(repr=False) + rating: Literal["helpful", "not_helpful"] + recorded_at: datetime = field(repr=False) + note: str | None = field(default=None, repr=False) + + def __post_init__(self) -> None: + _opaque("feedback_ref", self.feedback_ref, r"fb_[0-9a-f]{64}") + if type(self.binding) is not FeedbackBinding: + raise TypeError("feedback requires an exact binding") + if self.rating not in {"helpful", "not_helpful"}: + raise ValueError("feedback rating is unavailable") + if self.note is not None and ( + type(self.note) is not str + or not self.note + or self.note.isspace() + or len(self.note) > 1_000 + ): + raise ValueError("feedback note is unavailable") + _instant(self.recorded_at) + + +@dataclass(frozen=True, slots=True) +class TriagedFeedback: + """One accepted triage classification with unchanged exact lineage.""" + + feedback_ref: str + binding: FeedbackBinding = field(repr=False) + rating: Literal["helpful", "not_helpful"] + note: str | None = field(repr=False) + recorded_at: datetime = field(repr=False) + category: TriageCategory + + def __post_init__(self) -> None: + _opaque("feedback_ref", self.feedback_ref, r"fb_[0-9a-f]{64}") + if type(self.binding) is not FeedbackBinding: + raise TypeError("triaged feedback requires an exact binding") + if self.rating not in {"helpful", "not_helpful"}: + raise ValueError("triaged feedback rating is unavailable") + if self.note is not None and ( + type(self.note) is not str + or not self.note + or self.note.isspace() + or len(self.note) > 1_000 + ): + raise ValueError("triaged feedback note is unavailable") + _instant(self.recorded_at) + if type(self.category) is not TriageCategory: + raise FeedbackBindingUnavailable( + "feedback triage category is unavailable" + ) + + +def _text(value: object, name: str) -> str: + if type(value) is not str or not value or value.isspace(): + raise FeedbackBindingUnavailable(f"feedback {name} is unavailable") + return value + + +def _timestamp(value: object) -> datetime: + if type(value) is not str: + raise FeedbackBindingUnavailable("feedback recordedAt is unavailable") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + return _instant(parsed) + except (ValueError, FeedbackBindingUnavailable): + raise FeedbackBindingUnavailable( + "feedback recordedAt is unavailable" + ) from None + + +def feedback_evidence_from_document(value: object) -> FeedbackEvidence: + """Validate one trusted persistence projection as a closed whole.""" + + fields = frozenset( + { + "citations", + "feedbackRef", + "note", + "organizationId", + "packageDigest", + "packageRef", + "rating", + "recordedAt", + "releaseGeneration", + "releaseRef", + "runRef", + "schemaVersion", + } + ) + if type(value) is not dict or frozenset(value) != fields: + raise FeedbackBindingUnavailable("feedback evidence is malformed") + document = value + if document["schemaVersion"] != "context-engine-feedback-evidence-v1": + raise FeedbackBindingUnavailable("feedback evidence version is unavailable") + citations = document["citations"] + if type(citations) is not list: + raise FeedbackBindingUnavailable("feedback citations are unavailable") + projected: list[FeedbackCitation] = [] + for citation in citations: + citation_fields = frozenset( + { + "evidenceRef", + "fragmentRef", + "resourceRef", + "revisionRef", + "sourceRef", + } + ) + if type(citation) is not dict or frozenset(citation) != citation_fields: + raise FeedbackBindingUnavailable("feedback citation is malformed") + projected.append( + FeedbackCitation( + evidence_ref=_text(citation["evidenceRef"], "citation evidenceRef"), + lineage=EvidenceLineage( + source_ref=_text(citation["sourceRef"], "citation sourceRef"), + resource_ref=_text( + citation["resourceRef"], "citation resourceRef" + ), + revision_ref=_text( + citation["revisionRef"], "citation revisionRef" + ), + fragment_ref=_text( + citation["fragmentRef"], "citation fragmentRef" + ), + ), + ) + ) + try: + organization_id = UUID(_text(document["organizationId"], "organizationId")) + except ValueError: + raise FeedbackBindingUnavailable( + "feedback organizationId is unavailable" + ) from None + rating = document["rating"] + if rating not in {"helpful", "not_helpful"}: + raise FeedbackBindingUnavailable("feedback rating is unavailable") + note = document["note"] + if note is not None and type(note) is not str: + raise FeedbackBindingUnavailable("feedback note is unavailable") + return FeedbackEvidence( + feedback_ref=_text(document["feedbackRef"], "feedbackRef"), + binding=FeedbackBinding( + organization_id=organization_id, + run_ref=_text(document["runRef"], "runRef"), + package_ref=_text(document["packageRef"], "packageRef"), + package_digest=_text(document["packageDigest"], "packageDigest"), + release_ref=_text(document["releaseRef"], "releaseRef"), + release_generation=cast(int, document["releaseGeneration"]), + citations=tuple(projected), + ), + rating=cast(Literal["helpful", "not_helpful"], rating), + recorded_at=_timestamp(document["recordedAt"]), + note=note, + ) + + +def load_feedback_evidence(path: Path) -> FeedbackEvidence: + """Load one closed captured-feedback projection or refuse it as a whole.""" + + if not isinstance(path, Path): + raise TypeError("feedback evidence path must be Path") + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, ValueError): + raise FeedbackBindingUnavailable("feedback evidence is unavailable") from None + return feedback_evidence_from_document(value) + + +def bind_captured_feedback( + *, + feedback_ref: str, + rating: Literal["helpful", "not_helpful"], + recorded_at: datetime, + run: ContextRunView, + note: str | None = None, +) -> FeedbackEvidence: + """Bind captured evidence to one authorized current ContextRun projection.""" + + from engine.persistence.context_runs import ContextRunView + + if type(run) is not ContextRunView: + raise TypeError("feedback binding requires ContextRunView") + if run.outcome.value != "delivered_authorized": + raise FeedbackBindingUnavailable( + "feedback citations are unavailable for an empty ContextRun" + ) + if ( + run.package_ref is None + or run.release_ref is None + or run.release_generation is None + or run.authorized_citation_lineage is None + ): + raise FeedbackBindingUnavailable( + "feedback exact binding is unavailable for a legacy ContextRun" + ) + return FeedbackEvidence( + feedback_ref=feedback_ref, + binding=FeedbackBinding( + organization_id=run.organization_id, + run_ref=run.run_ref, + package_ref=run.package_ref, + package_digest=run.package_digest, + release_ref=run.release_ref, + release_generation=run.release_generation, + citations=tuple( + FeedbackCitation( + evidence_ref=value["evidenceRef"], + lineage=EvidenceLineage( + source_ref=value["sourceRef"], + resource_ref=value["resourceRef"], + revision_ref=value["revisionRef"], + fragment_ref=value["fragmentRef"], + ), + ) + for value in run.authorized_citation_lineage + ), + ), + rating=rating, + recorded_at=recorded_at, + note=note, + ) + + +def triage_feedback( + feedback: FeedbackEvidence, + category: TriageCategory, +) -> TriagedFeedback: + """Classify one exactly bound item or refuse the whole item.""" + + if type(feedback) is not FeedbackEvidence: + raise TypeError("triage requires FeedbackEvidence") + if type(category) is not TriageCategory: + raise FeedbackBindingUnavailable("feedback triage category is unavailable") + return TriagedFeedback( + feedback_ref=feedback.feedback_ref, + binding=feedback.binding, + rating=feedback.rating, + note=feedback.note, + recorded_at=feedback.recorded_at, + category=category, + ) diff --git a/engine/learning/golden.py b/engine/learning/golden.py index 6bd63eaa..1710366d 100644 --- a/engine/learning/golden.py +++ b/engine/learning/golden.py @@ -379,6 +379,20 @@ def _case(value: object) -> GoldenCase: ) +def load_golden_case(document: object) -> GoldenCase: + """Load one strict v1 case for governed intake without relaxing the schema.""" + + wrapper = { + "entries": [document], + "name": "synthetic-feedback-intake", + "schemaVersion": GOLDEN_SET_SCHEMA_VERSION, + "synthetic": True, + } + validated = _validate_schema(wrapper, DEFAULT_GOLDEN_SCHEMA_PATH) + entries = cast(list[object], validated["entries"]) + return _case(entries[0]) + + def validate_composition(golden_set: GoldenSet) -> None: """Enforce every counted dev/pilot composition floor as a hard failure.""" diff --git a/engine/learning/golden_intake.py b/engine/learning/golden_intake.py new file mode 100644 index 00000000..23a7f3ba --- /dev/null +++ b/engine/learning/golden_intake.py @@ -0,0 +1,127 @@ +"""Governed case admission through the existing golden pilot lock.""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +from engine.learning.curation_candidate import curation_candidate_case +from engine.learning.golden import ( + GoldenSetUnavailable, + load_golden_set, + validate_golden_document_schema, +) + + +@dataclass(frozen=True, slots=True) +class GoldenIntakeReceipt: + """Content-free receipt for one case admitted without changing the lock.""" + + case_ref: str + case_count: int + golden_digest: str + pilot_digest: str + + +def _load_candidate(path: Path) -> dict[str, object]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, ValueError): + raise GoldenSetUnavailable("curation candidate intake is unavailable") from None + if type(value) is not dict: + raise GoldenSetUnavailable("curation candidate intake is malformed") + return cast(dict[str, object], value) + + +def _write_private_staged(path: Path, document: dict[str, object]) -> None: + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path, flags, 0o600) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + descriptor = -1 + handle.write( + json.dumps( + document, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + + "\n" + ) + handle.flush() + os.fsync(handle.fileno()) + except BaseException: + path.unlink(missing_ok=True) + raise + finally: + if descriptor >= 0: + os.close(descriptor) + + +def admit_evaluation_case( + candidate_path: Path, + *, + golden_path: Path, + lock_path: Path, +) -> GoldenIntakeReceipt: + """Append one candidate dev case only after verifying the active pilot lock.""" + + if not all( + isinstance(path, Path) for path in (candidate_path, golden_path, lock_path) + ): + raise TypeError("golden intake paths must be Path") + locked = load_golden_set(golden_path, lock_path=lock_path) + case = curation_candidate_case(_load_candidate(candidate_path)) + candidate_document = locked.document() + entries = cast(list[object], candidate_document["entries"]) + entries.append(case.document()) + validate_golden_document_schema(candidate_document) + if case.partition != "dev": + raise GoldenSetUnavailable( + "feedback intake admits dev cases; " + "pilot admission requires explicit re-lock" + ) + case_refs = {case.case_ref for case in locked.cases} + if case.case_ref in case_refs: + raise GoldenSetUnavailable("evaluation intake caseRef already exists") + combined_document = locked.document() + combined_entries = cast(list[object], combined_document["entries"]) + combined_entries.append(case.document()) + staged = golden_path.with_name(f".{golden_path.name}.intake") + staged_created = False + try: + _write_private_staged(staged, combined_document) + staged_created = True + admitted = load_golden_set(staged, lock_path=lock_path) + if admitted.pilot_digest != locked.pilot_digest: + raise GoldenSetUnavailable("evaluation intake changed the locked pilot") + staged.replace(golden_path) + directory = os.open(golden_path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + except GoldenSetUnavailable: + raise + except OSError: + raise GoldenSetUnavailable("evaluation intake write is unavailable") from None + finally: + if staged_created: + try: + staged.unlink(missing_ok=True) + except OSError: + raise GoldenSetUnavailable( + "evaluation intake cleanup is unavailable" + ) from None + return GoldenIntakeReceipt( + case_ref=case.case_ref, + case_count=len(admitted.cases), + golden_digest=admitted.digest, + pilot_digest=admitted.pilot_digest, + ) diff --git a/engine/persistence/__init__.py b/engine/persistence/__init__.py index 284e3c56..c656e6da 100644 --- a/engine/persistence/__init__.py +++ b/engine/persistence/__init__.py @@ -49,6 +49,7 @@ PostgreSQLDeliveryEvidenceRetentionPort, ) from engine.persistence.egress import PostgreSQLEgressGrantRedemptionAuthority +from engine.persistence.feedback import PostgreSQLFeedbackInbox from engine.persistence.file_imports import ( FileImportInterrupted, FileImportLeaseRedemption, @@ -165,6 +166,7 @@ "FileImportUnavailable", "FilePublicationBoundary", "PostgreSQLFileImportWorker", + "PostgreSQLFeedbackInbox", "PublishedFileImport", "VerifiedContextRunOperatorIdentity", "ResourceAccessRevocation", diff --git a/engine/persistence/context_runs.py b/engine/persistence/context_runs.py index d1bec1e7..9dbf379b 100644 --- a/engine/persistence/context_runs.py +++ b/engine/persistence/context_runs.py @@ -266,10 +266,16 @@ class ContextRunView: query_digest_key_version: int = field(repr=False) query_digest: str = field(repr=False) outcome: ContextRunOutcome + package_ref: str | None package_digest_profile: str package_digest: str + release_ref: str | None + release_generation: int | None package_retention_mode: str authorized_evidence_refs: tuple[str, ...] + authorized_citation_lineage: tuple[dict[str, str], ...] | None = field( + repr=False + ) effective_max_tokens: int effective_max_provider_calls: int effective_max_cost_microunits: int @@ -473,10 +479,18 @@ def _view(row: RowMapping) -> ContextRunView: query_digest_key_version=values["query_digest_key_version"], query_digest=values["query_digest"], outcome=ContextRunOutcome(values["outcome"]), + package_ref=values["package_ref"], package_digest_profile=values["package_digest_profile"], package_digest=values["package_digest"], + release_ref=values["release_ref"], + release_generation=values["release_generation"], package_retention_mode=values["package_retention_mode"], authorized_evidence_refs=tuple(values["authorized_evidence_refs"]), + authorized_citation_lineage=( + tuple(dict(value) for value in values["authorized_citation_lineage"]) + if values["authorized_citation_lineage"] is not None + else None + ), effective_max_tokens=values["effective_max_tokens"], effective_max_provider_calls=values["effective_max_provider_calls"], effective_max_cost_microunits=values["effective_max_cost_microunits"], diff --git a/engine/persistence/feedback.py b/engine/persistence/feedback.py new file mode 100644 index 00000000..2fa62e2c --- /dev/null +++ b/engine/persistence/feedback.py @@ -0,0 +1,56 @@ +"""Learning-only projection of captured feedback with exact authorized lineage.""" + +from __future__ import annotations + +from uuid import UUID + +from sqlalchemy import Engine, text +from sqlalchemy.exc import SQLAlchemyError + +from engine.learning.feedback import ( + FeedbackBindingUnavailable, + FeedbackEvidence, + feedback_evidence_from_document, +) +from engine.persistence.role_guard import assert_learning_role + + +class PostgreSQLFeedbackInbox: + """Read one captured item through the narrow ContextLearning function.""" + + def __init__(self, engine: Engine) -> None: + if not isinstance(engine, Engine): + raise TypeError("PostgreSQLFeedbackInbox requires a SQLAlchemy Engine") + self._engine = engine + + def find_exact( + self, + organization_id: UUID, + feedback_ref: str, + ) -> FeedbackEvidence: + if type(organization_id) is not UUID: + raise TypeError("feedback inbox requires an Organization UUID") + try: + with self._engine.begin() as connection: + assert_learning_role(connection) + document = connection.execute( + text( + "SELECT context_learning_read_feedback_evidence(" + ":organization_id, :feedback_ref)" + ), + { + "organization_id": organization_id, + "feedback_ref": feedback_ref, + }, + ).scalar_one_or_none() + if document is None: + raise FeedbackBindingUnavailable( + "feedback exact binding is unavailable" + ) + return feedback_evidence_from_document(document) + except FeedbackBindingUnavailable: + raise + except (AssertionError, SQLAlchemyError, TypeError, ValueError): + raise FeedbackBindingUnavailable( + "feedback exact binding is unavailable" + ) from None diff --git a/engine/persistence/membership_context.py b/engine/persistence/membership_context.py index ea384d67..d673ee38 100644 --- a/engine/persistence/membership_context.py +++ b/engine/persistence/membership_context.py @@ -1090,8 +1090,9 @@ def persist( request_id, purpose, policy_snapshot_ref, policy_epoch, effective_scope_digest, query_digest_profile, query_digest_key_version, query_digest, outcome, - package_digest_profile, package_digest, - package_retention_mode, authorized_evidence_refs, + package_ref, package_digest_profile, package_digest, + release_ref, release_generation, package_retention_mode, + authorized_evidence_refs, authorized_citation_lineage, effective_max_tokens, effective_max_provider_calls, effective_max_cost_microunits, effective_max_elapsed_ms, usage_tokens, usage_provider_calls, @@ -1105,8 +1106,9 @@ def persist( :request_id, :purpose, :policy_snapshot_ref, :policy_epoch, :effective_scope_digest, :query_digest_profile, :query_digest_key_version, :query_digest, :outcome, - :package_digest_profile, :package_digest, - :package_retention_mode, :authorized_evidence_refs, + :package_ref, :package_digest_profile, :package_digest, + :release_ref, :release_generation, :package_retention_mode, + :authorized_evidence_refs, :authorized_citation_lineage, :effective_max_tokens, :effective_max_provider_calls, :effective_max_cost_microunits, :effective_max_elapsed_ms, :usage_tokens, :usage_provider_calls, @@ -1119,7 +1121,8 @@ def persist( bindparam( "authorized_evidence_refs", type_=postgresql.JSONB(), - ) + ), + bindparam("authorized_citation_lineage", type_=postgresql.JSONB()), ) self._connection.execute( insert_run, diff --git a/engine/persistence/schema_security_manifest.yaml b/engine/persistence/schema_security_manifest.yaml index d98a78b8..204f1eef 100644 --- a/engine/persistence/schema_security_manifest.yaml +++ b/engine/persistence/schema_security_manifest.yaml @@ -1,5 +1,5 @@ { - "manifestVersion": "40.0.0", + "manifestVersion": "41.0.0", "controlOperations": [ { "name": "register_file_source", @@ -2780,7 +2780,7 @@ "table": "context_run" } }, - "purpose": "Authorized-only durable lineage for one finalized ContextPackage delivery; package and query bodies are never retained", + "purpose": "Authorized-only durable lineage for one finalized ContextPackage delivery, including exact Package, generation-bound Release, and citation identity for governed feedback intake; package and query bodies are never retained", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ { @@ -2905,6 +2905,10 @@ { "name": "ck_context_run_timestamp_order", "expression": "finalized_at >= accepted_at AND package_as_of = finalized_at AND package_expires_at > package_as_of" + }, + { + "name": "ck_context_run_feedback_lineage_complete", + "expression": "Legacy rows carry no feedback lineage; current rows bind a closed Package ref, generation-bound Release ref, positive generation, and an exact five-field citation object for every authorized Evidence ref" } ], "rowLevelSecurity": { @@ -2943,6 +2947,14 @@ "context_engine_context_run_reader_definer" ], "using": "context_run.organization_id = NULLIF(current_setting('app.ui_feedback_organization_id', true), '')::uuid AND context_run.run_ref = current_setting('app.ui_feedback_run_ref', true) AND context_run.user_id = NULLIF(current_setting('app.ui_feedback_user_id', true), '')::uuid AND context_run.membership_id = NULLIF(current_setting('app.ui_feedback_membership_id', true), '')::uuid AND context_run.membership_version = NULLIF(current_setting('app.ui_feedback_membership_version', true), '')::bigint AND context_run.principal_ref = current_setting('app.ui_feedback_principal_ref', true) AND current_setting('app.ui_feedback_mode', true) = 'capture'" + }, + { + "name": "context_run_learning_feedback_definer_select", + "command": "SELECT", + "roles": [ + "context_engine_context_run_reader_definer" + ], + "using": "context_run.organization_id = NULLIF(current_setting('app.learning_feedback_organization_id', true), '')::uuid AND current_setting('app.learning_feedback_mode', true) = 'read' AND EXISTS (SELECT 1 FROM public.context_feedback AS feedback WHERE feedback.organization_id = context_run.organization_id AND feedback.run_ref = context_run.run_ref AND feedback.feedback_ref = current_setting('app.learning_feedback_ref', true))" } ] }, @@ -2956,6 +2968,9 @@ "context_engine_context_run_reader_definer": [ "SELECT" ], + "context_engine_learning": [ + "EXECUTE context_learning_read_feedback_evidence" + ], "context_engine_worker": [], "context_engine_control": [ "EXECUTE issue_context_run_operator_read_ticket", @@ -2992,7 +3007,7 @@ "table": "context_feedback" } }, - "purpose": "Evidence-only answer feedback bound to one exact same-Organization ContextRun and current Membership; it has no Release publication authority", + "purpose": "Evidence-only answer feedback bound to one exact same-Organization ContextRun and current Membership; Learning receives only an exact authorized Package, generation-bound Release, and citation projection, with no Release publication authority", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ { @@ -3098,6 +3113,14 @@ "context_engine_context_run_reader_definer" ], "using": "context_feedback.organization_id = NULLIF(current_setting('app.ui_feedback_organization_id', true), '')::uuid AND context_feedback.run_ref = current_setting('app.ui_feedback_run_ref', true) AND context_feedback.membership_id = NULLIF(current_setting('app.ui_feedback_membership_id', true), '')::uuid AND context_feedback.membership_version = NULLIF(current_setting('app.ui_feedback_membership_version', true), '')::bigint AND current_setting('app.ui_feedback_mode', true) = 'capture'" + }, + { + "name": "context_feedback_learning_definer_select", + "command": "SELECT", + "roles": [ + "context_engine_context_run_reader_definer" + ], + "using": "context_feedback.organization_id = NULLIF(current_setting('app.learning_feedback_organization_id', true), '')::uuid AND context_feedback.feedback_ref = current_setting('app.learning_feedback_ref', true) AND current_setting('app.learning_feedback_mode', true) = 'read'" } ] }, @@ -3114,7 +3137,9 @@ "SELECT", "INSERT" ], - "context_engine_learning": [], + "context_engine_learning": [ + "EXECUTE context_learning_read_feedback_evidence" + ], "context_engine_release_operator": [], "context_engine_runtime": [ "EXECUTE context_runtime_capture_context_feedback" diff --git a/engine/runtime/construction.py b/engine/runtime/construction.py index 68633947..b71b3cf6 100644 --- a/engine/runtime/construction.py +++ b/engine/runtime/construction.py @@ -1871,6 +1871,7 @@ def resolve( final_effective_scope=policy_receipt.effective_scope, effective_budget=decision.effective_budget, keyring=self._query_digest_keyring, + active_release=active_release, ) persist_context_run( persistence_session, diff --git a/engine/runtime/context_run.py b/engine/runtime/context_run.py index 8c52093b..306e3a11 100644 --- a/engine/runtime/context_run.py +++ b/engine/runtime/context_run.py @@ -5,6 +5,7 @@ from dataclasses import dataclass, field from datetime import datetime, timedelta from enum import StrEnum +from re import fullmatch from typing import TYPE_CHECKING, Final, NoReturn, Protocol, cast from uuid import UUID @@ -98,10 +99,14 @@ class ContextRunRecord: query_digest_key_version: int = field(repr=False) query_digest: str = field(repr=False) outcome: ContextRunOutcome + package_ref: str package_digest_profile: str package_digest: str + release_ref: str + release_generation: int package_retention_mode: str authorized_evidence_refs: tuple[str, ...] + authorized_citation_lineage: tuple[dict[str, str], ...] = field(repr=False) effective_max_tokens: int effective_max_provider_calls: int effective_max_cost_microunits: int @@ -149,6 +154,15 @@ def __post_init__(self) -> None: raise TypeError("ContextRun outcome must be ContextRunOutcome") if self.package_digest_profile != PACKAGE_DIGEST_PROFILE: raise ValueError("ContextRun package digest profile is not active") + if fullmatch(r"pkg_[0-9a-f]{32}", self.package_ref) is None: + raise ValueError("ContextRun package_ref must use the closed format") + if fullmatch(r"rel_[0-9a-f]{64}", self.release_ref) is None: + raise ValueError("ContextRun release_ref must use the closed format") + if ( + type(self.release_generation) is not int + or not 1 <= self.release_generation <= MAX_SIGNED_BIGINT + ): + raise ValueError("ContextRun release generation must be positive") if self.package_retention_mode != PACKAGE_RETENTION_MODE: raise ValueError("ContextRun Package retention must remain digest_only") if type(self.authorized_evidence_refs) is not tuple: @@ -167,6 +181,39 @@ def __post_init__(self) -> None: self.authorized_evidence_refs ): raise ValueError("ContextRun authorized Evidence refs must be unique") + if type(self.authorized_citation_lineage) is not tuple: + raise TypeError("ContextRun citation lineage must be a tuple") + expected_citation_fields = frozenset( + { + "evidenceRef", + "fragmentRef", + "resourceRef", + "revisionRef", + "sourceRef", + } + ) + if any( + type(value) is not dict + or frozenset(value) != expected_citation_fields + or value["evidenceRef"] not in self.authorized_evidence_refs + or any(type(item) is not str or not item for item in value.values()) + for value in self.authorized_citation_lineage + ): + raise ValueError("ContextRun citation lineage requires exact Evidence") + citation_evidence_refs = tuple( + value["evidenceRef"] for value in self.authorized_citation_lineage + ) + if ( + citation_evidence_refs != self.authorized_evidence_refs + or len(self.authorized_citation_lineage) + != len( + { + tuple(sorted(value.items())) + for value in self.authorized_citation_lineage + } + ) + ): + raise ValueError("ContextRun citation lineage must be exact and unique") if self.outcome is ContextRunOutcome.DELIVERED_EMPTY: if self.authorized_evidence_refs: raise ValueError("empty ContextRun cannot contain Evidence refs") @@ -391,10 +438,12 @@ def build_context_run_records( final_effective_scope: EffectiveScope, effective_budget: PackageBudget, keyring: QueryDigestKeyring, + active_release: object, ) -> tuple[ContextRunRecord, DecisionAuditRecord | None]: """Project one finalized Package into safe durable lineage.""" from engine.runtime.invocation import AuthenticatedInvocation + from engine.runtime.release_lineage import ActiveRuntimeRelease from engine.runtime.scope import ( OMITTED_REQUEST_NARROWING, EffectiveScope, @@ -408,9 +457,13 @@ def build_context_run_records( raise TypeError("ContextRun projection requires Acquire or OpenCitation") if type(package) is not ContextPackage: raise TypeError("ContextRun projection requires ContextPackage") - active_release = invocation.user_actor.active_runtime_release - if active_release is None: + if type(active_release) is not ActiveRuntimeRelease: + raise TypeError("ContextRun projection requires ActiveRuntimeRelease") + invocation_release = invocation.user_actor.active_runtime_release + if invocation_release is None: raise ValueError("ContextRun requires an active Runtime release") + if active_release != invocation_release: + raise ValueError("ContextRun requires the invocation's exact Runtime release") if type(final_effective_scope) is not EffectiveScope: raise TypeError("ContextRun projection requires final EffectiveScope") if type(effective_budget) is not PackageBudget: @@ -532,9 +585,22 @@ def build_context_run_records( query_digest=query_receipt.value, outcome=outcome, package_digest_profile=PACKAGE_DIGEST_PROFILE, + package_ref=package.package_id, package_digest=package.package_digest, + release_ref=active_release.manifest_ref, + release_generation=active_release.active_generation, package_retention_mode=PACKAGE_RETENTION_MODE, authorized_evidence_refs=tuple(item.evidence_ref for item in package.evidence), + authorized_citation_lineage=tuple( + { + "evidenceRef": item.evidence_ref, + "fragmentRef": item.fragment_ref, + "resourceRef": item.resource_ref, + "revisionRef": item.revision_ref, + "sourceRef": item.source_ref, + } + for item in package.evidence + ), effective_max_tokens=effective_budget.max_tokens, effective_max_provider_calls=effective_budget.max_provider_calls, effective_max_cost_microunits=effective_budget.max_cost_microunits, diff --git a/eval/README.md b/eval/README.md index 00eadda8..8637d31a 100644 --- a/eval/README.md +++ b/eval/README.md @@ -222,6 +222,44 @@ uv run context-engine-eval validate \ The shell variables above deliberately point to the operator's durable private storage; no worktree-local default is provided. +## Feedback triage and governed case intake + +Captured feedback becomes a curation proposal only after the Learning database +role resolves its Organization and feedback reference through the narrow inbox +function. That trusted projection binds the item to its exact ContextRun, +ContextPackage reference and digest, generation-bound Release reference, and +complete citation Evidence lineage. Missing or partial binding refuses the item; +the workflow never substitutes a DecisionAudit denial or denied object detail. +Triage uses only the closed `source`, `visibility`, `retrieval`, `assembly`, and +`evaluation` categories. + +`context-engine-eval feedback-candidate` accepts the feedback locator plus one +private v1 case from either the durable corpus root or an ignored +`.context-engine/` directory and writes only a mode-0600 `CurationCandidate` +under the same two private storage boundaries. Its terminal output contains a +digest, never an input or output path. Caller-authored feedback projections are +not accepted. A reviewed candidate's dev case enters the durable corpus with +`context-engine-eval feedback-intake`; the command verifies the candidate digest +and existing pilot lock, validates the new case, proves the locked pilot digest +is unchanged, then replaces the corpus atomically. A pilot case is refused here: +admitting or changing pilot cases still requires the explicit existing `relock` +ceremony. Any fixture tracked under `eval/golden/` remains subject to the stricter +placeholder-only privacy scan. + +Authoritative evaluation reports that ran with a lineage map now record the +exact `release.releaseRef` established by that map. `compare-releases` accepts +only tracked-threshold reports with executor-observed security and a resolved +lineage check over the same golden digest, then renders per-layer/per-slice +counts, scores, and deltas. `REFUSED` and `NON_AUTHORITATIVE` reports are rejected. +While thresholds are pending preregistration, the same slice observations may be +reported only with `PENDING_PREREGISTRATION` status and without a verdict. + +This workflow owns no scheduler, ReleaseManifest operation, active pointer, +rollback, promotion call, or release-operator grant surface. It produces only +candidates and reports; every current manifest continues to select +`curation_off`, and publication remains the explicit ContextLearning promotion +transaction described by ADR-0033. + ## Layers, floors, and thresholds Retrieval reports case hit plus macro- and micro-averaged Evidence recall. diff --git a/eval/golden/v1/synthetic-feedback-intake.json b/eval/golden/v1/synthetic-feedback-intake.json new file mode 100644 index 00000000..c3e856e3 --- /dev/null +++ b/eval/golden/v1/synthetic-feedback-intake.json @@ -0,0 +1,19 @@ +{ + "entries": [ + { + "answerability": "unanswerable", + "caseRef": "synthetic-feedback-evaluation-case", + "expectedAnswer": "synthetic-feedback-refusal-answer", + "expectedEvidence": [], + "hardNegativeEvidence": [], + "partition": "dev", + "query": "synthetic-feedback-evaluation-query", + "requiredClaims": [], + "slice": "single_doc", + "topicCluster": "synthetic-feedback-topic" + } + ], + "name": "synthetic-feedback-intake-fixture", + "schemaVersion": "context-engine-golden-set-v1", + "synthetic": true +} diff --git a/migrations/versions/20260731_0048_feedback_curation_lineage.py b/migrations/versions/20260731_0048_feedback_curation_lineage.py new file mode 100644 index 00000000..3a828df5 --- /dev/null +++ b/migrations/versions/20260731_0048_feedback_curation_lineage.py @@ -0,0 +1,265 @@ +"""Bind captured feedback to exact authorized delivery lineage. + +Revision ID: 20260731_0048 +Revises: 20260730_0047 +Create Date: 2026-07-31 +""" + +# ruff: noqa: E501 + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "20260731_0048" +down_revision: str | None = "20260730_0047" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_LEARNING = "context_engine_learning" +_DEFINER = "context_engine_context_run_reader_definer" +_OPERATOR = "context_engine_security_operator" +_MAX = (1 << 63) - 1 +_READ = "context_learning_read_feedback_evidence" +_READ_SIGNATURE = "(uuid,text)" +_OPERATOR_READ = "read_context_run_by_operator_ticket" +_OPERATOR_READ_SIGNATURE = "(text,uuid,text)" + + +def _replace_operator_reader(*, install: bool) -> None: + definition = op.get_bind().execute( + sa.text( + "SELECT pg_catalog.pg_get_functiondef(CAST(:procedure AS regprocedure))" + ), + {"procedure": f"public.{_OPERATOR_READ}{_OPERATOR_READ_SIGNATURE}"}, + ).scalar_one() + if not isinstance(definition, str): + raise RuntimeError("ContextRun reader definition is unavailable") + old_return = ( + "outcome text, package_digest_profile text, package_digest text, " + "package_retention_mode text, authorized_evidence_refs jsonb," + ) + new_return = ( + "outcome text, package_ref text, package_digest_profile text, " + "package_digest text, release_ref text, release_generation bigint, " + "package_retention_mode text, authorized_evidence_refs jsonb, " + "authorized_citation_lineage jsonb," + ) + old_select = ( + "run.outcome,\n run.package_digest_profile, " + "run.package_digest,\n run.package_retention_mode, " + "run.authorized_evidence_refs," + ) + new_select = ( + "run.outcome,\n run.package_ref, " + "run.package_digest_profile, run.package_digest,\n " + "run.release_ref, run.release_generation,\n " + "run.package_retention_mode, run.authorized_evidence_refs,\n " + "run.authorized_citation_lineage," + ) + searched_return, replacement_return = ( + (old_return, new_return) if install else (new_return, old_return) + ) + searched_select, replacement_select = ( + (old_select, new_select) if install else (new_select, old_select) + ) + if definition.count(searched_return) != 1 or definition.count(searched_select) != 1: + raise RuntimeError("ContextRun reader shape was not recognized") + replacement = definition.replace(searched_return, replacement_return).replace( + searched_select, replacement_select + ) + replacement = replacement.replace("CREATE OR REPLACE FUNCTION", "CREATE FUNCTION", 1) + op.execute(f"GRANT CREATE ON SCHEMA public TO {_DEFINER}") + op.execute(f"SET LOCAL ROLE {_DEFINER}") + op.execute(f"DROP FUNCTION public.{_OPERATOR_READ}{_OPERATOR_READ_SIGNATURE}") + op.execute(replacement) + op.execute( + f"REVOKE ALL ON FUNCTION public.{_OPERATOR_READ}" + f"{_OPERATOR_READ_SIGNATURE} FROM PUBLIC" + ) + op.execute( + f"GRANT EXECUTE ON FUNCTION public.{_OPERATOR_READ}" + f"{_OPERATOR_READ_SIGNATURE} TO {_OPERATOR}" + ) + op.execute("RESET ROLE") + op.execute(f"REVOKE CREATE ON SCHEMA public FROM {_DEFINER}") + + +def upgrade() -> None: + """Retain exact authorized lineage and expose one Learning-only inbox read.""" + + op.add_column("context_run", sa.Column("package_ref", sa.Text())) + op.add_column("context_run", sa.Column("release_ref", sa.Text())) + op.add_column("context_run", sa.Column("release_generation", sa.BigInteger())) + op.add_column( + "context_run", + sa.Column( + "authorized_citation_lineage", + postgresql.JSONB(astext_type=sa.Text()), + ), + ) + op.create_check_constraint( + "ck_context_run_feedback_lineage_complete", + "context_run", + "(package_ref IS NULL AND release_ref IS NULL " + "AND release_generation IS NULL " + "AND authorized_citation_lineage IS NULL) OR (" + "package_ref ~ '^pkg_[0-9a-f]{32}$' " + "AND release_ref ~ '^rel_[0-9a-f]{64}$' " + f"AND release_generation BETWEEN 1 AND {_MAX} " + "AND jsonb_typeof(authorized_citation_lineage) = 'array' " + "AND jsonb_array_length(authorized_citation_lineage) = " + "jsonb_array_length(authorized_evidence_refs) " + "AND jsonb_path_query_array(authorized_citation_lineage, " + "'$[*].evidenceRef') = authorized_evidence_refs " + "AND NOT jsonb_path_exists(authorized_citation_lineage, " + "'$[*] ? (@.type() != \"object\" " + "|| !exists(@.evidenceRef) || !exists(@.sourceRef) " + "|| !exists(@.resourceRef) || !exists(@.revisionRef) " + "|| !exists(@.fragmentRef) " + "|| !(@.evidenceRef like_regex \"^ev_[0-9a-f]{64}$\") " + "|| @.sourceRef.type() != \"string\" " + "|| @.resourceRef.type() != \"string\" " + "|| @.revisionRef.type() != \"string\" " + "|| @.fragmentRef.type() != \"string\" " + "|| !(@.sourceRef like_regex \".*[^\\\\s\\\\u00a0\\\\u1680" + "\\\\u2000-\\\\u200a\\\\u2028\\\\u2029\\\\u202f" + "\\\\u205f\\\\u3000].*\") " + "|| !(@.resourceRef like_regex \".*[^\\\\s\\\\u00a0\\\\u1680" + "\\\\u2000-\\\\u200a\\\\u2028\\\\u2029\\\\u202f" + "\\\\u205f\\\\u3000].*\") " + "|| !(@.revisionRef like_regex \".*[^\\\\s\\\\u00a0\\\\u1680" + "\\\\u2000-\\\\u200a\\\\u2028\\\\u2029\\\\u202f" + "\\\\u205f\\\\u3000].*\") " + "|| !(@.fragmentRef like_regex \".*[^\\\\s\\\\u00a0\\\\u1680" + "\\\\u2000-\\\\u200a\\\\u2028\\\\u2029\\\\u202f" + "\\\\u205f\\\\u3000].*\"))') " + "AND NOT jsonb_path_exists(authorized_citation_lineage, " + "'$[*].keyvalue() ? (@.key != \"evidenceRef\" " + "&& @.key != \"sourceRef\" && @.key != \"resourceRef\" " + "&& @.key != \"revisionRef\" && @.key != \"fragmentRef\")'))", + ) + _replace_operator_reader(install=True) + op.execute( + "CREATE POLICY context_feedback_learning_definer_select " + "ON context_feedback FOR SELECT TO " + f"{_DEFINER} USING (" + "context_feedback.organization_id = NULLIF(current_setting(" + "'app.learning_feedback_organization_id', true), '')::uuid " + "AND context_feedback.feedback_ref = current_setting(" + "'app.learning_feedback_ref', true) " + "AND current_setting('app.learning_feedback_mode', true) = 'read')" + ) + op.execute( + "CREATE POLICY context_run_learning_feedback_definer_select " + "ON context_run FOR SELECT TO " + f"{_DEFINER} USING (" + "context_run.organization_id = NULLIF(current_setting(" + "'app.learning_feedback_organization_id', true), '')::uuid " + "AND current_setting('app.learning_feedback_mode', true) = 'read' " + "AND EXISTS (SELECT 1 FROM public.context_feedback AS feedback " + "WHERE feedback.organization_id = context_run.organization_id " + "AND feedback.run_ref = context_run.run_ref " + "AND feedback.feedback_ref = current_setting(" + "'app.learning_feedback_ref', true)))" + ) + op.execute(f"GRANT CREATE ON SCHEMA public TO {_DEFINER}") + op.execute(f"SET LOCAL ROLE {_DEFINER}") + op.execute( + f""" + CREATE FUNCTION public.{_READ}( + requested_organization_id uuid, + requested_feedback_ref text + ) RETURNS jsonb LANGUAGE plpgsql STABLE SECURITY DEFINER + SET search_path = pg_catalog SET row_security = on AS $function$ + DECLARE projection jsonb; + BEGIN + IF SESSION_USER <> '{_LEARNING}' + OR requested_organization_id IS NULL + OR requested_feedback_ref !~ '^fb_[0-9a-f]{{64}}$' + THEN RETURN NULL; END IF; + PERFORM pg_catalog.set_config( + 'app.learning_feedback_mode', 'read', true + ); + PERFORM pg_catalog.set_config( + 'app.learning_feedback_organization_id', + requested_organization_id::text, true + ); + PERFORM pg_catalog.set_config( + 'app.learning_feedback_ref', requested_feedback_ref, true + ); + SELECT pg_catalog.jsonb_build_object( + 'citations', run.authorized_citation_lineage, + 'feedbackRef', feedback.feedback_ref, + 'note', feedback.note, + 'organizationId', feedback.organization_id::text, + 'packageDigest', run.package_digest, + 'packageRef', run.package_ref, + 'rating', feedback.rating, + 'recordedAt', pg_catalog.to_char( + feedback.recorded_at AT TIME ZONE 'UTC', + 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"' + ), + 'releaseGeneration', run.release_generation, + 'releaseRef', run.release_ref, + 'runRef', run.run_ref, + 'schemaVersion', 'context-engine-feedback-evidence-v1' + ) INTO projection + FROM public.context_feedback AS feedback + JOIN public.context_run AS run + ON run.organization_id = feedback.organization_id + AND run.run_ref = feedback.run_ref + WHERE feedback.organization_id = requested_organization_id + AND feedback.feedback_ref = requested_feedback_ref + AND run.outcome = 'delivered_authorized' + AND run.package_ref IS NOT NULL + AND run.release_ref IS NOT NULL + AND run.release_generation IS NOT NULL + AND run.authorized_citation_lineage IS NOT NULL + AND pg_catalog.jsonb_array_length( + run.authorized_citation_lineage) > 0; + RETURN projection; + END; $function$ + """ + ) + op.execute( + f"REVOKE ALL ON FUNCTION public.{_READ}{_READ_SIGNATURE} FROM PUBLIC" + ) + op.execute( + f"GRANT EXECUTE ON FUNCTION public.{_READ}{_READ_SIGNATURE} TO {_LEARNING}" + ) + op.execute("RESET ROLE") + op.execute(f"REVOKE CREATE ON SCHEMA public FROM {_DEFINER}") + + +def downgrade() -> None: + """Remove the inbox only when no exact feedback lineage would be lost.""" + + op.execute( + "DO $block$ BEGIN IF EXISTS (SELECT 1 FROM context_feedback) " + "OR EXISTS (SELECT 1 FROM context_run WHERE package_ref IS NOT NULL) " + "THEN RAISE EXCEPTION USING ERRCODE = '55000', " + "MESSAGE = 'cannot downgrade with feedback curation lineage'; " + "END IF; END; $block$" + ) + op.execute(f"SET LOCAL ROLE {_DEFINER}") + op.execute(f"DROP FUNCTION public.{_READ}{_READ_SIGNATURE}") + op.execute("RESET ROLE") + op.execute( + "DROP POLICY context_run_learning_feedback_definer_select ON context_run" + ) + op.execute( + "DROP POLICY context_feedback_learning_definer_select ON context_feedback" + ) + _replace_operator_reader(install=False) + op.drop_constraint( + "ck_context_run_feedback_lineage_complete", + "context_run", + type_="check", + ) + op.drop_column("context_run", "authorized_citation_lineage") + op.drop_column("context_run", "release_generation") + op.drop_column("context_run", "release_ref") + op.drop_column("context_run", "package_ref") diff --git a/tests/integration/test_context_run_schema.py b/tests/integration/test_context_run_schema.py index 8fd74b8c..b71492c4 100644 --- a/tests/integration/test_context_run_schema.py +++ b/tests/integration/test_context_run_schema.py @@ -305,7 +305,83 @@ def insert_context_run( policy_snapshot_ref: str = POLICY_SNAPSHOT_REF, policy_epoch: int = 1, finalized_at: datetime = ACCEPTED_AT, + package_ref: str | None = None, + release_ref: str | None = None, + release_generation: int | None = None, + authorized_citation_lineage: tuple[dict[str, str], ...] | None = None, ) -> None: + current_columns = set( + connection.execute( + text( + "SELECT column_name FROM information_schema.columns " + "WHERE table_schema = 'public' AND table_name = 'context_run'" + ) + ).scalars() + ) + feedback_lineage_columns = { + "package_ref", + "release_ref", + "release_generation", + "authorized_citation_lineage", + } + if feedback_lineage_columns.isdisjoint(current_columns): + connection.execute( + text( + """ + INSERT INTO context_run ( + organization_id, run_ref, decision_ref, + user_id, membership_id, membership_version, + principal_ref, agent_version_ref, + authenticated_application_ref, authentication_binding_ref, + request_id, purpose, policy_snapshot_ref, policy_epoch, + effective_scope_digest, query_digest_profile, + query_digest_key_version, query_digest, outcome, + package_digest_profile, package_digest, + package_retention_mode, authorized_evidence_refs, + effective_max_tokens, effective_max_provider_calls, + effective_max_cost_microunits, effective_max_elapsed_ms, + usage_tokens, usage_provider_calls, + usage_cost_microunits, usage_elapsed_ms, + accepted_at, finalized_at, package_as_of, package_expires_at + ) VALUES ( + :organization_id, :run_ref, :decision_ref, + :user_id, :membership_id, 1, + 'principal:issue-19', 'agent:issue-19', + 'application:issue-19', 'binding:issue-19', + 'request:issue-19', 'answer', :policy_snapshot_ref, :policy_epoch, + :effective_scope_digest, 'context-query-json-hmac-sha256-v1', + 1, :query_digest, :outcome, + 'context-package-canonical-json-v1', :package_digest, + 'digest_only', CAST(:authorized_evidence_refs AS jsonb), + 1000, 8, 100000, 5000, + 0, 0, 0, 0, + :accepted_at, :finalized_at, :package_as_of, + :package_expires_at + ) + """ + ), + { + "organization_id": identity.organization_id, + "run_ref": identity.run_ref, + "decision_ref": identity.decision_ref, + "user_id": identity.user_id, + "membership_id": identity.membership_id, + "effective_scope_digest": "a" * 64, + "query_digest": "b" * 64, + "package_digest": "c" * 64, + "authorized_evidence_refs": json.dumps(authorized_evidence_refs), + "outcome": outcome, + "policy_snapshot_ref": policy_snapshot_ref, + "policy_epoch": policy_epoch, + "accepted_at": ACCEPTED_AT, + "finalized_at": finalized_at, + "package_as_of": finalized_at, + "package_expires_at": finalized_at + timedelta(minutes=5), + }, + ) + return + if not feedback_lineage_columns.issubset(current_columns): + raise AssertionError("context_run feedback lineage schema is partial") connection.execute( text( """ @@ -317,8 +393,9 @@ def insert_context_run( request_id, purpose, policy_snapshot_ref, policy_epoch, effective_scope_digest, query_digest_profile, query_digest_key_version, query_digest, outcome, - package_digest_profile, package_digest, - package_retention_mode, authorized_evidence_refs, + package_ref, package_digest_profile, package_digest, + release_ref, release_generation, package_retention_mode, + authorized_evidence_refs, authorized_citation_lineage, effective_max_tokens, effective_max_provider_calls, effective_max_cost_microunits, effective_max_elapsed_ms, usage_tokens, usage_provider_calls, @@ -332,8 +409,10 @@ def insert_context_run( 'request:issue-19', 'answer', :policy_snapshot_ref, :policy_epoch, :effective_scope_digest, 'context-query-json-hmac-sha256-v1', 1, :query_digest, :outcome, - 'context-package-canonical-json-v1', :package_digest, - 'digest_only', CAST(:authorized_evidence_refs AS jsonb), + :package_ref, 'context-package-canonical-json-v1', :package_digest, + :release_ref, :release_generation, 'digest_only', + CAST(:authorized_evidence_refs AS jsonb), + CAST(:authorized_citation_lineage AS jsonb), 1000, 8, 100000, 5000, 0, 0, 0, 0, :accepted_at, :finalized_at, :package_as_of, @@ -351,6 +430,14 @@ def insert_context_run( "query_digest": "b" * 64, "package_digest": "c" * 64, "authorized_evidence_refs": json.dumps(authorized_evidence_refs), + "authorized_citation_lineage": ( + json.dumps(authorized_citation_lineage) + if authorized_citation_lineage is not None + else None + ), + "package_ref": package_ref, + "release_ref": release_ref, + "release_generation": release_generation, "outcome": outcome, "policy_snapshot_ref": policy_snapshot_ref, "policy_epoch": policy_epoch, diff --git a/tests/integration/test_feedback_has_no_publication_authority.py b/tests/integration/test_feedback_has_no_publication_authority.py index d2411dd8..be130b2a 100644 --- a/tests/integration/test_feedback_has_no_publication_authority.py +++ b/tests/integration/test_feedback_has_no_publication_authority.py @@ -54,7 +54,23 @@ def test_feedback_has_no_publication_authority( migration_engine = create_database_engine(migration_configuration) try: with migration_engine.begin() as connection: - insert_context_run(connection, lineage_identity) + citation = { + "evidenceRef": "ev_" + "6" * 64, + "fragmentRef": "synthetic-fragment-feedback", + "resourceRef": "synthetic-resource-feedback", + "revisionRef": "synthetic-revision-feedback", + "sourceRef": "synthetic-source-feedback", + } + insert_context_run( + connection, + lineage_identity, + outcome="delivered_authorized", + authorized_evidence_refs=(citation["evidenceRef"],), + package_ref="pkg_" + "2" * 32, + release_ref="rel_" + "4" * 64, + release_generation=7, + authorized_citation_lineage=(citation,), + ) release_count_before = connection.execute( text( "SELECT count(*) FROM active_release_manifest " @@ -128,6 +144,30 @@ def test_feedback_has_no_publication_authority( 'context_runtime_capture_context_feedback' '(uuid,text,text,uuid,uuid,bigint,text,text,text)', 'EXECUTE' + ), + has_function_privilege( + 'context_engine_learning', + 'context_learning_read_feedback_evidence' + '(uuid,text)', + 'EXECUTE' + ), + has_function_privilege( + 'context_engine_runtime', + 'context_learning_read_feedback_evidence' + '(uuid,text)', + 'EXECUTE' + ), + has_function_privilege( + 'context_engine_control', + 'context_learning_read_feedback_evidence' + '(uuid,text)', + 'EXECUTE' + ), + has_function_privilege( + 'context_engine_release_operator', + 'context_learning_read_feedback_evidence' + '(uuid,text)', + 'EXECUTE' ) """ ) @@ -139,6 +179,65 @@ def test_feedback_has_no_publication_authority( "Lineage was clear", ) assert release_count_after == release_count_before - assert privileges == (True, False, False) + assert privileges == (True, False, False, True, False, False, False) finally: migration_engine.dispose() + + +def test_feedback_workflow_roles_cannot_issue_or_mutate_release_authority( + migration_configuration: DatabaseConfiguration, +) -> None: + engine = create_database_engine(migration_configuration) + try: + with engine.connect() as connection: + privileges = { + (row.role_name, row.relation_name, row.privilege_name) + for row in connection.execute( + text( + """ + SELECT role_name, relation_name, privilege_name + FROM ( + VALUES + ('context_engine_learning', + 'release_operator_grant', 'INSERT'), + ('context_engine_learning', + 'release_operator_grant', 'UPDATE'), + ('context_engine_learning', + 'active_release_manifest', 'INSERT'), + ('context_engine_learning', + 'active_release_manifest', 'UPDATE'), + ('context_engine_runtime', + 'release_operator_grant', 'INSERT'), + ('context_engine_control', + 'release_operator_grant', 'INSERT') + ) AS requested( + role_name, relation_name, privilege_name + ) + WHERE has_table_privilege( + role_name, + 'public.' || relation_name, + privilege_name + ) + """ + ) + ) + } + promotion_execute = connection.execute( + text( + """ + SELECT count(*) + FROM information_schema.routine_privileges + WHERE routine_schema = 'public' + AND routine_name = 'context_learning_promote_release' + AND grantee IN ( + 'context_engine_runtime', 'context_engine_control' + ) + AND privilege_type = 'EXECUTE' + """ + ) + ).scalar_one() + finally: + engine.dispose() + + assert privileges == set() + assert promotion_execute == 0 diff --git a/tests/integration/test_feedback_triage_intake.py b/tests/integration/test_feedback_triage_intake.py new file mode 100644 index 00000000..d249d45e --- /dev/null +++ b/tests/integration/test_feedback_triage_intake.py @@ -0,0 +1,201 @@ +from __future__ import annotations + +from datetime import UTC, datetime + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import Engine, text +from sqlalchemy.exc import ProgrammingError + +from adapters.http.app import create_app +from adapters.http.ui_api import PostgreSQLUiApi +from engine.learning.feedback import FeedbackBindingUnavailable +from engine.persistence import ( + DatabaseConfiguration, + PostgreSQLFeedbackInbox, + PostgreSQLMembershipAuthority, + create_database_engine, +) +from tests.integration.test_context_run_schema import ( + LineageIdentity, + current_user_actor, + insert_context_run, +) +from tests.integration.test_context_run_schema import ( + lineage_identity as _lineage_identity, +) +from tests.integration.test_feedback_has_no_publication_authority import ( + TOKEN, + _Authenticator, +) +from tests.support.ui import authenticate_ui + +pytestmark = pytest.mark.integration +lineage_identity = _lineage_identity + + +def _capture( + identity: LineageIdentity, + guarded_runtime_engine: Engine, + migration_configuration: DatabaseConfiguration, +) -> str: + client = TestClient( + create_app( + authenticator=_Authenticator(identity), + ui_bearer_token=TOKEN, + ui_api=PostgreSQLUiApi( + PostgreSQLMembershipAuthority(guarded_runtime_engine), + None, + preview_key=b"f" * 32, + feedback_engine=guarded_runtime_engine, + clock=lambda: datetime.now(UTC), + ), + ) + ) + authenticate_ui(client, TOKEN) + response = client.post( + "/ui/feedback", + content=( + f"runRef={identity.run_ref}&rating=not_helpful&" + "note=synthetic-feedback-note" + ), + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + assert response.status_code == 200 + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as connection: + feedback_ref = connection.execute( + text( + "SELECT feedback_ref FROM context_feedback " + "WHERE organization_id = :organization_id" + ), + {"organization_id": identity.organization_id}, + ).scalar_one() + assert isinstance(feedback_ref, str) + return feedback_ref + finally: + migration_engine.dispose() + + +def test_captured_feedback_resolves_only_from_exact_authorized_run_lineage( + lineage_identity: LineageIdentity, + migration_configuration: DatabaseConfiguration, + guarded_runtime_engine: Engine, + guarded_learning_engine: Engine, +) -> None: + citation = { + "evidenceRef": "ev_" + "6" * 64, + "fragmentRef": "synthetic-fragment-feedback", + "resourceRef": "synthetic-resource-feedback", + "revisionRef": "synthetic-revision-feedback", + "sourceRef": "synthetic-source-feedback", + } + with current_user_actor(guarded_runtime_engine, lineage_identity) as connection: + insert_context_run( + connection, + lineage_identity, + outcome="delivered_authorized", + authorized_evidence_refs=(citation["evidenceRef"],), + package_ref="pkg_" + "2" * 32, + release_ref="rel_" + "4" * 64, + release_generation=7, + authorized_citation_lineage=(citation,), + ) + + feedback_ref = _capture( + lineage_identity, + guarded_runtime_engine, + migration_configuration, + ) + item = PostgreSQLFeedbackInbox(guarded_learning_engine).find_exact( + lineage_identity.organization_id, + feedback_ref, + ) + + assert item.feedback_ref == feedback_ref + assert item.binding.run_ref == lineage_identity.run_ref + assert item.binding.package_ref == "pkg_" + "2" * 32 + assert item.binding.release_generation == 7 + assert item.binding.citations[0].evidence_ref == citation["evidenceRef"] + assert not hasattr(item, "denied_details") + + +def test_learning_refuses_feedback_from_an_empty_or_legacy_unbindable_run( + lineage_identity: LineageIdentity, + migration_configuration: DatabaseConfiguration, + guarded_runtime_engine: Engine, + guarded_learning_engine: Engine, +) -> None: + with current_user_actor(guarded_runtime_engine, lineage_identity) as connection: + insert_context_run(connection, lineage_identity) + + client = TestClient( + create_app( + authenticator=_Authenticator(lineage_identity), + ui_bearer_token=TOKEN, + ui_api=PostgreSQLUiApi( + PostgreSQLMembershipAuthority(guarded_runtime_engine), + None, + preview_key=b"f" * 32, + feedback_engine=guarded_runtime_engine, + clock=lambda: datetime.now(UTC), + ), + ) + ) + authenticate_ui(client, TOKEN) + response = client.post( + "/ui/feedback", + content=( + f"runRef={lineage_identity.run_ref}&rating=helpful&" + "note=synthetic-empty-run-feedback" + ), + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + + assert response.status_code == 200 + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as connection: + feedback_ref = connection.execute( + text( + "SELECT feedback_ref FROM context_feedback " + "WHERE organization_id = :organization_id" + ), + {"organization_id": lineage_identity.organization_id}, + ).scalar_one() + finally: + migration_engine.dispose() + with pytest.raises(FeedbackBindingUnavailable): + PostgreSQLFeedbackInbox(guarded_learning_engine).find_exact( + lineage_identity.organization_id, + feedback_ref, + ) + + +def test_learning_inbox_cannot_read_a_different_organization( + lineage_identity: LineageIdentity, + guarded_learning_engine: Engine, +) -> None: + with pytest.raises(FeedbackBindingUnavailable): + PostgreSQLFeedbackInbox(guarded_learning_engine).find_exact( + lineage_identity.organization_id, + "fb_" + "0" * 64, + ) + + +def test_learning_and_other_processes_have_no_direct_feedback_table_read( + guarded_learning_engine: Engine, + guarded_runtime_engine: Engine, + guarded_control_engine: Engine, +) -> None: + for engine in ( + guarded_learning_engine, + guarded_runtime_engine, + guarded_control_engine, + ): + with ( + engine.begin() as connection, + pytest.raises(ProgrammingError, match="permission denied"), + ): + connection.execute(text("SELECT * FROM context_feedback")) diff --git a/tests/integration/test_runtime_non_enumeration_integration.py b/tests/integration/test_runtime_non_enumeration_integration.py index 3129e5e3..ad92aee4 100644 --- a/tests/integration/test_runtime_non_enumeration_integration.py +++ b/tests/integration/test_runtime_non_enumeration_integration.py @@ -95,10 +95,14 @@ "query_digest_key_version", "query_digest", "outcome", + "package_ref", "package_digest_profile", "package_digest", + "release_ref", + "release_generation", "package_retention_mode", "authorized_evidence_refs", + "authorized_citation_lineage", "effective_max_tokens", "effective_max_provider_calls", "effective_max_cost_microunits", diff --git a/tests/unit/test_candidate_release_comparison.py b/tests/unit/test_candidate_release_comparison.py new file mode 100644 index 00000000..b5697d83 --- /dev/null +++ b/tests/unit/test_candidate_release_comparison.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +import json +from hashlib import sha256 +from typing import cast + +import pytest + +from engine.learning.comparison import ( + EvaluationComparisonUnavailable, + compare_release_evaluations, +) + + +def _report(release_ref: str, *, status: str = "PASS") -> dict[str, object]: + def row(slice_name: str) -> dict[str, object]: + return { + "case_count": 10, + "score": 0.8, + "slice_name": slice_name, + "status": "pass", + "wilson_95_high": 0.95, + "wilson_95_low": 0.65, + } + + report: dict[str, object] = { + "goldenSet": {"digest": "a" * 64}, + "lineageCheck": { + "ran": True, + "staleCaseCount": 0, + "totalCaseCount": 10, + }, + "release": {"releaseRef": release_ref}, + "reportVersion": "context-engine-eval-report-v1", + "run": {"executedSeamRef": "dogfood-loopback-resolve-acquire-v1"}, + "security": { + "missingContextFallbackCount": 0, + "observationState": "observed_clean", + "status": "pass", + "unauthorizedEvidenceCount": 0, + "wrongOrganizationEffectCount": 0, + }, + "slices": { + layer: [ + row(slice_name) + for slice_name in ("cross_doc", "single_doc", "temporal") + ] + for layer in ("answer", "citation", "retrieval") + }, + "status": status, + "thresholdAuthority": "tracked", + } + report["reportDigest"] = _digest(report) + return report + + +def _digest(document: dict[str, object]) -> str: + return sha256( + json.dumps( + document, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + ).hexdigest() + + +def _rebind(report: dict[str, object]) -> None: + report.pop("reportDigest", None) + report["reportDigest"] = _digest(report) + + +def test_comparison_reports_candidate_delta_per_slice() -> None: + active = _report("rel-active-1") + candidate = _report("rel-candidate-2") + candidate["slices"]["retrieval"][1]["score"] = 0.9 # type: ignore[index] + _rebind(candidate) + + comparison = compare_release_evaluations(active, candidate) + slices = cast(list[dict[str, object]], comparison["slices"]) + + retrieval = next( + item + for item in slices + if item["layer"] == "retrieval" and item["slice"] == "single_doc" + ) + assert comparison["status"] == "compared" + assert comparison["activeReleaseRef"] == "rel-active-1" + assert comparison["candidateReleaseRef"] == "rel-candidate-2" + assert retrieval == { + "activeCaseCount": 10, + "activeScore": 0.8, + "activeStatus": "pass", + "candidateCaseCount": 10, + "candidateScore": 0.9, + "candidateStatus": "pass", + "delta": 0.1, + "layer": "retrieval", + "slice": "single_doc", + } + assert comparison["schemaVersion"] == "context-engine-release-comparison-v1" + digest = cast(str, comparison.pop("reportDigest")) + assert digest == _digest(comparison) + + +@pytest.mark.parametrize("status", ("REFUSED", "NON_AUTHORITATIVE")) +def test_non_authoritative_report_is_never_presented_as_a_verdict( + status: str, +) -> None: + with pytest.raises(EvaluationComparisonUnavailable, match="authoritative"): + compare_release_evaluations( + _report("rel-active-1", status=status), + _report("rel-candidate-2"), + ) + + +def test_security_failure_remains_authoritative_but_never_offset_by_scores() -> None: + active = _report("rel-active-1", status="FAIL") + active["security"] = { + "missingContextFallbackCount": 0, + "observationState": "observed_violation", + "status": "fail", + "unauthorizedEvidenceCount": 1, + "wrongOrganizationEffectCount": 0, + } + _rebind(active) + + comparison = compare_release_evaluations( + active, + _report("rel-candidate-2", status="PASS"), + ) + + assert comparison["activeReportStatus"] == "FAIL" + assert comparison["candidateReportStatus"] == "PASS" + + +def test_pending_reports_compare_slices_without_claiming_a_verdict() -> None: + comparison = compare_release_evaluations( + _report("rel-active-1", status="PENDING_PREREGISTRATION"), + _report("rel-candidate-2", status="PENDING_PREREGISTRATION"), + ) + + assert comparison["status"] == "PENDING_PREREGISTRATION" + assert "verdict" not in comparison + assert cast(list[object], comparison["slices"]) + + +def test_comparison_refuses_different_golden_set_or_missing_lineage_check() -> None: + active = _report("rel-active-1") + candidate = _report("rel-candidate-2") + candidate["goldenSet"] = {"digest": "d" * 64} + _rebind(candidate) + with pytest.raises(EvaluationComparisonUnavailable, match="golden"): + compare_release_evaluations(active, candidate) + + candidate = _report("rel-candidate-2") + candidate["lineageCheck"] = { + "ran": False, + "staleCaseCount": None, + "totalCaseCount": 10, + } + _rebind(candidate) + with pytest.raises(EvaluationComparisonUnavailable, match="lineage"): + compare_release_evaluations(active, candidate) + + +def test_comparison_refuses_tampered_digest_or_incomplete_slice_set() -> None: + candidate = _report("rel-candidate-2") + candidate["status"] = "FAIL" + with pytest.raises(EvaluationComparisonUnavailable, match="authoritative"): + compare_release_evaluations(_report("rel-active-1"), candidate) + + candidate = _report("rel-candidate-2") + cast(dict[str, object], candidate["slices"])["answer"] = [] + _rebind(candidate) + with pytest.raises(EvaluationComparisonUnavailable, match="slice set"): + compare_release_evaluations(_report("rel-active-1"), candidate) diff --git a/tests/unit/test_context_run.py b/tests/unit/test_context_run.py index 8e53710e..47f1702e 100644 --- a/tests/unit/test_context_run.py +++ b/tests/unit/test_context_run.py @@ -62,6 +62,7 @@ _observe_current_policy_epoch, _open_policy_epoch_authority_scope, ) +from engine.runtime.release_lineage import ActiveRuntimeRelease from engine.runtime.scope import EffectiveScope, ScopeSet, ScopeTarget from engine.runtime.scope_authority import ( _close_scope_authority_scope, @@ -186,6 +187,12 @@ def _trusted_invocation() -> Iterator[AuthenticatedInvocation]: _close_membership_authority_scope(membership_scope) +def _invocation_release(invocation: AuthenticatedInvocation) -> ActiveRuntimeRelease: + release = invocation.user_actor.active_runtime_release + assert release is not None + return release + + def _provenance() -> DecisionProvenanceReceipt: return DecisionProvenanceReceipt( decision_ref=DECISION_REF, @@ -314,9 +321,21 @@ def _run(**changes: object) -> ContextRunRecord: "query_digest": "2" * 64, "outcome": ContextRunOutcome.DELIVERED_AUTHORIZED, "package_digest_profile": PACKAGE_DIGEST_PROFILE, + "package_ref": ORGANIZATION_REF, "package_digest": "3" * 64, + "release_ref": active_runtime_release(ORGANIZATION_ID).manifest_ref, + "release_generation": 1, "package_retention_mode": PACKAGE_RETENTION_MODE, "authorized_evidence_refs": (EVIDENCE_REF,), + "authorized_citation_lineage": ( + { + "evidenceRef": EVIDENCE_REF, + "fragmentRef": "fragment-authorized", + "resourceRef": "resource-authorized", + "revisionRef": "revision-authorized", + "sourceRef": "source-authorized", + }, + ), "effective_max_tokens": 1_000, "effective_max_provider_calls": 8, "effective_max_cost_microunits": 25_000, @@ -338,6 +357,7 @@ def _empty_run() -> ContextRunRecord: return _run( outcome=ContextRunOutcome.DELIVERED_EMPTY, authorized_evidence_refs=(), + authorized_citation_lineage=(), usage_tokens=0, usage_provider_calls=0, usage_cost_microunits=0, @@ -383,6 +403,7 @@ def test_projection_builds_complete_authorized_and_empty_final_records() -> None final_effective_scope=FINAL_EFFECTIVE_SCOPE, effective_budget=EFFECTIVE_BUDGET, keyring=QUERY_KEYRING, + active_release=_invocation_release(invocation), ) empty, empty_audit = build_context_run_records( invocation=invocation, @@ -392,9 +413,25 @@ def test_projection_builds_complete_authorized_and_empty_final_records() -> None final_effective_scope=FINAL_EFFECTIVE_SCOPE, effective_budget=EFFECTIVE_BUDGET, keyring=QUERY_KEYRING, + active_release=_invocation_release(invocation), ) assert authorized.outcome is ContextRunOutcome.DELIVERED_AUTHORIZED + assert authorized.package_ref == _authorized_package().package_id + assert ( + authorized.release_ref + == active_runtime_release(ORGANIZATION_ID).manifest_ref + ) + assert authorized.release_generation == 1 + assert authorized.authorized_citation_lineage == ( + { + "evidenceRef": EVIDENCE_REF, + "fragmentRef": "fragment-authorized", + "resourceRef": "resource-authorized", + "revisionRef": "revision-authorized", + "sourceRef": "source-authorized", + }, + ) assert authorized.authorized_evidence_refs == (EVIDENCE_REF,) assert authorized.package_digest == _authorized_package().package_digest assert authorized.query_digest != request.need.query @@ -438,6 +475,7 @@ def test_projection_rejects_final_scope_outside_original_or_empty_scope() -> Non final_effective_scope=unrelated_scope, effective_budget=EFFECTIVE_BUDGET, keyring=QUERY_KEYRING, + active_release=_invocation_release(invocation), ) @@ -459,6 +497,7 @@ def test_projection_rejects_evidence_after_final_scope_veto() -> None: final_effective_scope=empty_scope, effective_budget=EFFECTIVE_BUDGET, keyring=QUERY_KEYRING, + active_release=_invocation_release(invocation), ) @@ -497,6 +536,7 @@ def test_projection_rejects_provenance_not_bound_to_invocation_or_package( final_effective_scope=FINAL_EFFECTIVE_SCOPE, effective_budget=EFFECTIVE_BUDGET, keyring=QUERY_KEYRING, + active_release=_invocation_release(invocation), ) @@ -544,6 +584,7 @@ def test_projection_rejects_evidence_lineage_not_bound_to_provenance( final_effective_scope=FINAL_EFFECTIVE_SCOPE, effective_budget=EFFECTIVE_BUDGET, keyring=QUERY_KEYRING, + active_release=_invocation_release(invocation), ) @@ -568,6 +609,7 @@ def test_projection_rejects_package_purpose_or_time_not_bound_to_invocation() -> final_effective_scope=FINAL_EFFECTIVE_SCOPE, effective_budget=EFFECTIVE_BUDGET, keyring=QUERY_KEYRING, + active_release=_invocation_release(invocation), ) @@ -587,6 +629,7 @@ def test_projection_rejects_package_altered_after_digest_creation() -> None: final_effective_scope=FINAL_EFFECTIVE_SCOPE, effective_budget=EFFECTIVE_BUDGET, keyring=QUERY_KEYRING, + active_release=_invocation_release(invocation), ) diff --git a/tests/unit/test_context_run_operator_authority.py b/tests/unit/test_context_run_operator_authority.py index 67eb215b..8eeba7bb 100644 --- a/tests/unit/test_context_run_operator_authority.py +++ b/tests/unit/test_context_run_operator_authority.py @@ -176,10 +176,14 @@ def _safe_row() -> dict[str, object]: "query_digest_key_version": 1, "query_digest": "2" * 64, "outcome": "delivered_empty", + "package_ref": None, "package_digest_profile": "rfc8785-sha256-v1", "package_digest": "3" * 64, + "release_ref": None, + "release_generation": None, "package_retention_mode": "digest_only", "authorized_evidence_refs": [], + "authorized_citation_lineage": None, "effective_max_tokens": 1, "effective_max_provider_calls": 1, "effective_max_cost_microunits": 1, diff --git a/tests/unit/test_curation_workflow_authority.py b/tests/unit/test_curation_workflow_authority.py new file mode 100644 index 00000000..f3024fcd --- /dev/null +++ b/tests/unit/test_curation_workflow_authority.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +from engine.learning.contracts import CurationMode, CurationProfileRef + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +WORKFLOW_MODULES = ( + REPOSITORY_ROOT / "applications/eval_v1.py", + REPOSITORY_ROOT / "engine/learning/feedback.py", + REPOSITORY_ROOT / "engine/learning/curation_candidate.py", + REPOSITORY_ROOT / "engine/learning/golden_intake.py", + REPOSITORY_ROOT / "engine/learning/comparison.py", + REPOSITORY_ROOT / "engine/persistence/feedback.py", +) + +FORBIDDEN_AUTHORITY_SYMBOLS = frozenset( + { + "ContextLearning", + "PromotionAuthorizationRequest", + "PromotionReceipt", + "ReleaseManifest", + "ReleaseOperatorAuthority", + "TrustedPromotionCall", + "VerifiedReleaseOperatorIdentity", + "promote", + "promote_atomically", + "release_operator_grant", + "rollback", + } +) +FORBIDDEN_EFFECT_CALLS = frozenset( + { + "activate", + "authorize", + "execute", + "grant", + "promote", + "promote_atomically", + "rollback", + } +) + + +def _imported_symbols(tree: ast.AST) -> set[str]: + symbols: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import | ast.ImportFrom): + symbols.update(alias.name for alias in node.names) + return symbols + + +def _called_attributes(tree: ast.AST) -> set[str]: + return { + node.func.attr + for node in ast.walk(tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + } + + +def test_curation_workflow_has_a_closed_non_publication_authority_surface() -> None: + for path in WORKFLOW_MODULES: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + names = {node.id for node in ast.walk(tree) if isinstance(node, ast.Name)} + attributes = { + node.attr for node in ast.walk(tree) if isinstance(node, ast.Attribute) + } + assert not ( + FORBIDDEN_AUTHORITY_SYMBOLS + & (_imported_symbols(tree) | names | attributes) + ), path.name + + effects = _called_attributes( + ast.parse( + (REPOSITORY_ROOT / "engine/learning/golden_intake.py").read_text( + encoding="utf-8" + ) + ) + ) + assert not (effects & FORBIDDEN_EFFECT_CALLS) + assert effects & {"read_text", "replace", "write_text"} + + +def test_curation_workflow_does_not_export_release_authority() -> None: + import engine.learning.comparison as comparison + import engine.learning.curation_candidate as candidate + import engine.learning.feedback as feedback + import engine.learning.golden_intake as intake + + for module in (comparison, candidate, feedback, intake): + assert not any(hasattr(module, name) for name in FORBIDDEN_AUTHORITY_SYMBOLS) + + +def test_release_owner_package_does_not_export_curation_workflow() -> None: + import engine.learning as learning_package + + for name in ( + "CurationCandidate", + "EvaluationCaseIntake", + "FeedbackEvidence", + "TriageCategory", + "admit_evaluation_case", + "build_curation_candidate", + "compare_release_evaluations", + ): + assert not hasattr(learning_package, name) + + +def test_all_current_release_manifests_still_require_curation_off() -> None: + profile = CurationProfileRef.off( + profile_ref="curation-off-v0", + profile_digest="0" * 64, + ) + + assert profile.mode is CurationMode.OFF + assert profile.curation_snapshot_ref is None + + release_composition = ( + REPOSITORY_ROOT / "applications/release_promotion.py" + ).read_text(encoding="utf-8") + assert "CurationProfileRef.off(" in release_composition + assert "CurationProfileRef.on(" not in release_composition + + +def test_feedback_workflow_is_not_scheduled() -> None: + schedule = ( + REPOSITORY_ROOT / "deploy/daily-driver/scheduled-jobs.json" + ).read_text(encoding="utf-8") + + assert "feedback-candidate" not in schedule + assert "feedback-intake" not in schedule + assert "compare-releases" not in schedule diff --git a/tests/unit/test_eval_v1_cli.py b/tests/unit/test_eval_v1_cli.py index 5518b87e..a57c9571 100644 --- a/tests/unit/test_eval_v1_cli.py +++ b/tests/unit/test_eval_v1_cli.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import stat import subprocess import sys from dataclasses import replace @@ -10,6 +11,7 @@ import pytest +import applications.eval_v1 as eval_cli from applications.eval_executor import TRACKED_RUN_SEAM_REF from applications.eval_v1 import main from engine.learning.eval_report import ( @@ -22,6 +24,7 @@ build_evaluation_report, load_evaluation_run, ) +from engine.learning.feedback import feedback_evidence_from_document from engine.learning.golden import create_golden_lock, load_golden_set from engine.learning.golden_storage import GOLDEN_ROOT_ENV from engine.learning.thresholds import load_thresholds @@ -184,6 +187,7 @@ def test_cli_report_records_resolved_lineage_check_counts(tmp_path: Path) -> Non "staleCaseCount": 0, "totalCaseCount": len(entries), } + assert report["release"] == {"releaseRef": "synthetic-release-v1"} def test_cli_success_output_never_discloses_an_absolute_corpus_path( @@ -253,6 +257,92 @@ def test_importing_cli_does_not_eagerly_import_the_run_executor() -> None: assert completed.returncode == 0, completed.stderr +def test_feedback_candidate_cli_emits_only_a_path_free_candidate( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + feedback_path = tmp_path / "feedback.json" + case_path = tmp_path / ".context-engine/eval/case.json" + output_path = tmp_path / ".context-engine/eval/candidate.json" + case_path.parent.mkdir(parents=True) + feedback_path.write_text( + json.dumps( + { + "citations": [ + { + "evidenceRef": "ev_" + "6" * 64, + "fragmentRef": "synthetic-fragment-feedback", + "resourceRef": "synthetic-resource-feedback", + "revisionRef": "synthetic-revision-feedback", + "sourceRef": "synthetic-source-feedback", + } + ], + "feedbackRef": "fb_" + "5" * 64, + "note": "synthetic-feedback-note", + "organizationId": "00000000-0000-4000-8000-000000000152", + "packageDigest": "3" * 64, + "packageRef": "pkg_" + "2" * 32, + "rating": "not_helpful", + "recordedAt": "2026-07-31T00:00:00Z", + "releaseGeneration": 7, + "releaseRef": "rel_" + "4" * 64, + "runRef": "run_" + "1" * 32, + "schemaVersion": "context-engine-feedback-evidence-v1", + } + ), + encoding="utf-8", + ) + case = golden_case( + "synthetic-feedback-case", + answerability="unanswerable", + ) + case["hardNegativeEvidence"] = [] + case_path.write_text(json.dumps(case), encoding="utf-8") + captured = feedback_evidence_from_document( + json.loads(feedback_path.read_text(encoding="utf-8")) + ) + monkeypatch.setattr(eval_cli, "_captured_feedback", lambda *args: captured) + + main( + [ + "feedback-candidate", + "--organization-id", + "00000000-0000-4000-8000-000000000152", + "--feedback-ref", + "fb_" + "5" * 64, + "--category", + "retrieval", + "--case", + str(case_path), + "--output", + str(output_path), + "--proposed-at", + "2026-07-31T01:00:00Z", + ] + ) + + document = json.loads(output_path.read_text(encoding="utf-8")) + output = capsys.readouterr().out + assert document["schemaVersion"] == "context-engine-curation-candidate-v1" + assert document["category"] == "retrieval" + assert set(document) == { + "baseReleaseGeneration", + "baseReleaseRef", + "candidateDigest", + "candidateRef", + "category", + "evaluationCase", + "feedbackBinding", + "feedbackRef", + "proposedAt", + "schemaVersion", + } + assert str(tmp_path) not in output + assert "curation candidate written: digest=" in output + assert stat.S_IMODE(output_path.stat().st_mode) == 0o600 + + def test_cli_execute_reaches_a_non_refused_report_through_an_executed_run( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -299,6 +389,7 @@ def test_cli_execute_reaches_a_non_refused_report_through_an_executed_run( "staleCaseCount": 0, "totalCaseCount": len(entries), } + assert report["release"] == {"releaseRef": "synthetic-release-v1"} assert report["security"] == { "missingContextFallbackCount": 0, "observationState": "observed_clean", diff --git a/tests/unit/test_feedback_curation_candidate.py b/tests/unit/test_feedback_curation_candidate.py new file mode 100644 index 00000000..4f831796 --- /dev/null +++ b/tests/unit/test_feedback_curation_candidate.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from uuid import UUID + +import pytest + +from engine.learning.curation_candidate import ( + CurationCandidateUnavailable, + EvaluationCaseIntake, + build_curation_candidate, +) +from engine.learning.feedback import ( + FeedbackBinding, + FeedbackCitation, + FeedbackEvidence, + TriageCategory, + TriagedFeedback, + triage_feedback, +) +from engine.learning.golden import ( + EvidenceExpectation, + EvidenceLineage, + GoldenCase, + GoldenSetUnavailable, +) + + +def _lineage() -> EvidenceLineage: + return EvidenceLineage( + source_ref="synthetic-source-feedback", + resource_ref="synthetic-resource-feedback", + revision_ref="synthetic-revision-feedback", + fragment_ref="synthetic-fragment-feedback", + ) + + +def _triaged_feedback() -> TriagedFeedback: + return triage_feedback( + FeedbackEvidence( + feedback_ref="fb_" + "5" * 64, + binding=FeedbackBinding( + organization_id=UUID("00000000-0000-4000-8000-000000000152"), + run_ref="run_" + "1" * 32, + package_ref="pkg_" + "2" * 32, + package_digest="3" * 64, + release_ref="rel_" + "4" * 64, + release_generation=7, + citations=( + FeedbackCitation( + evidence_ref="ev_" + "6" * 64, + lineage=_lineage(), + ), + ), + ), + rating="not_helpful", + note="synthetic-feedback-note", + recorded_at=datetime(2026, 7, 31, tzinfo=UTC), + ), + TriageCategory.RETRIEVAL, + ) + + +def _case() -> GoldenCase: + return GoldenCase( + case_ref="synthetic-feedback-case", + query="synthetic-feedback-query", + expected_evidence=(), + expected_answer="synthetic-feedback-answer", + required_claims=(), + answerability="unanswerable", + slice_name="single_doc", + partition="dev", + topic_cluster="synthetic-feedback-topic", + hard_negative_evidence=(), + ) + + +def test_triaged_feedback_produces_only_an_immutable_curation_candidate() -> None: + candidate = build_curation_candidate( + _triaged_feedback(), + EvaluationCaseIntake(case=_case(), synthetic=True), + proposed_at=datetime(2026, 7, 31, 1, tzinfo=UTC), + ) + + assert candidate.feedback_ref == "fb_" + "5" * 64 + assert candidate.category is TriageCategory.RETRIEVAL + assert candidate.base_release_ref == "rel_" + "4" * 64 + assert candidate.base_release_generation == 7 + assert candidate.feedback_binding.run_ref == "run_" + "1" * 32 + assert candidate.feedback_binding.package_ref == "pkg_" + "2" * 32 + assert candidate.evaluation_case == _case() + assert candidate.candidate_ref.startswith("cur_") + assert len(candidate.candidate_digest) == 64 + + +def test_private_evaluation_case_intake_does_not_claim_to_be_synthetic() -> None: + private = GoldenCase( + case_ref="private-feedback-case", + query="Where is my real roadmap?", + expected_evidence=(), + expected_answer="It is in the private corpus.", + required_claims=(), + answerability="unanswerable", + slice_name="single_doc", + partition="dev", + topic_cluster="private-feedback-topic", + hard_negative_evidence=(), + ) + + assert EvaluationCaseIntake(case=private, synthetic=False).case is private + + personal = GoldenCase( + case_ref="synthetic-feedback-case", + query="Where is my real roadmap?", + expected_evidence=(), + expected_answer="synthetic-feedback-answer", + required_claims=(), + answerability="unanswerable", + slice_name="single_doc", + partition="dev", + topic_cluster="synthetic-feedback-topic", + hard_negative_evidence=(), + ) + with pytest.raises(CurationCandidateUnavailable, match="placeholder"): + EvaluationCaseIntake(case=personal, synthetic=True) + + +def test_evaluation_case_must_bind_feedback_citation_lineage() -> None: + foreign = EvidenceLineage( + source_ref="synthetic-source-foreign", + resource_ref="synthetic-resource-foreign", + revision_ref="synthetic-revision-foreign", + fragment_ref="synthetic-fragment-foreign", + ) + answerable = GoldenCase( + case_ref="synthetic-feedback-case", + query="synthetic-feedback-query", + expected_evidence=( + EvidenceExpectation( + path="synthetic/foreign.md", + lineage=foreign, + ), + ), + expected_answer="synthetic-feedback-answer", + required_claims=(), + answerability="answerable", + slice_name="single_doc", + partition="dev", + topic_cluster="synthetic-feedback-topic", + hard_negative_evidence=(), + ) + + with pytest.raises((CurationCandidateUnavailable, GoldenSetUnavailable)): + build_curation_candidate( + _triaged_feedback(), + EvaluationCaseIntake(case=answerable, synthetic=True), + proposed_at=datetime(2026, 7, 31, 1, tzinfo=UTC), + ) + + assert foreign not in { + citation.lineage for citation in _triaged_feedback().binding.citations + } diff --git a/tests/unit/test_feedback_golden_intake.py b/tests/unit/test_feedback_golden_intake.py new file mode 100644 index 00000000..4ebe69d9 --- /dev/null +++ b/tests/unit/test_feedback_golden_intake.py @@ -0,0 +1,218 @@ +from __future__ import annotations + +import json +import os +import stat +from datetime import UTC, datetime +from pathlib import Path + +import pytest + +from engine.learning.curation_candidate import ( + EvaluationCaseIntake, + build_curation_candidate, + curation_candidate_document, +) +from engine.learning.golden import ( + GoldenSetUnavailable, + create_golden_lock, + load_golden_case, + load_golden_set, +) +from engine.learning.golden_intake import admit_evaluation_case +from tests.support.golden import golden_case, valid_composed_entries, write_golden +from tests.unit.test_feedback_curation_candidate import _triaged_feedback + + +def _intake_case(*, partition: str = "dev") -> dict[str, object]: + case = golden_case( + "synthetic-feedback-intake", + partition=partition, + answerability="unanswerable", + ) + case["hardNegativeEvidence"] = [] + return case + + +def _locked_corpus(tmp_path: Path) -> tuple[Path, Path]: + golden_path = tmp_path / "golden.json" + lock_path = tmp_path / "golden.lock.json" + write_golden(golden_path, valid_composed_entries()) + golden_set = load_golden_set( + golden_path, + allow_unlocked_pilot_for_initial_lock=True, + ) + create_golden_lock( + golden_set, + lock_path, + authority="maintainer", + reason="synthetic-initial-lock", + recorded_at=datetime(2026, 7, 31, tzinfo=UTC), + ) + return golden_path, lock_path + + +def _write_candidate(path: Path, case: dict[str, object]) -> None: + candidate = build_curation_candidate( + _triaged_feedback(), + EvaluationCaseIntake( + case=load_golden_case(case), + synthetic=True, + ), + proposed_at=datetime(2026, 7, 31, 1, tzinfo=UTC), + ) + path.write_text( + json.dumps(curation_candidate_document(candidate)), + encoding="utf-8", + ) + + +def test_private_candidate_case_enters_only_the_private_durable_corpus( + tmp_path: Path, +) -> None: + golden_path, lock_path = _locked_corpus(tmp_path) + candidate_path = tmp_path / "private-candidate.json" + private_case = _intake_case() + private_case["caseRef"] = "private-feedback-intake" + private_case["query"] = "Where is my private evaluation input?" + private_case["expectedAnswer"] = "The private durable corpus owns it." + private_case["topicCluster"] = "private-feedback-topic" + candidate = build_curation_candidate( + _triaged_feedback(), + EvaluationCaseIntake( + case=load_golden_case(private_case), + synthetic=False, + ), + proposed_at=datetime(2026, 7, 31, 1, tzinfo=UTC), + ) + candidate_path.write_text( + json.dumps(curation_candidate_document(candidate)), + encoding="utf-8", + ) + + receipt = admit_evaluation_case( + candidate_path, + golden_path=golden_path, + lock_path=lock_path, + ) + + assert receipt.case_ref == "private-feedback-intake" + assert "private-feedback-intake" in golden_path.read_text(encoding="utf-8") + + +def test_intake_appends_a_schema_valid_dev_case_through_existing_lock( + tmp_path: Path, +) -> None: + golden_path, lock_path = _locked_corpus(tmp_path) + case_path = tmp_path / "intake.json" + _write_candidate(case_path, _intake_case()) + + outcome = admit_evaluation_case( + case_path, + golden_path=golden_path, + lock_path=lock_path, + ) + loaded = load_golden_set(golden_path, lock_path=lock_path) + + assert outcome.case_ref == "synthetic-feedback-intake" + assert outcome.case_count == 71 + assert len(loaded.cases) == 71 + assert loaded.pilot_digest == outcome.pilot_digest + assert loaded.cases[-1] == EvaluationCaseIntake( + case=loaded.cases[-1], + synthetic=True, + ).case + assert stat.S_IMODE(golden_path.stat().st_mode) == 0o600 + + +def test_intake_file_permissions_do_not_depend_on_process_umask( + tmp_path: Path, +) -> None: + golden_path, lock_path = _locked_corpus(tmp_path) + case_path = tmp_path / "intake.json" + _write_candidate(case_path, _intake_case()) + previous = os.umask(0o000) + try: + admit_evaluation_case( + case_path, + golden_path=golden_path, + lock_path=lock_path, + ) + finally: + os.umask(previous) + + assert stat.S_IMODE(golden_path.stat().st_mode) == 0o600 + + +def test_intake_refuses_unlocked_or_schema_invalid_case_without_writing( + tmp_path: Path, +) -> None: + golden_path, lock_path = _locked_corpus(tmp_path) + before = golden_path.read_bytes() + unlocked = tmp_path / "unlocked.json" + write_golden(unlocked, valid_composed_entries()) + case_path = tmp_path / "intake.json" + invalid = _intake_case() + invalid["slice"] = "unknown" + candidate = curation_candidate_document( + build_curation_candidate( + _triaged_feedback(), + EvaluationCaseIntake( + case=load_golden_case(_intake_case()), + synthetic=True, + ), + proposed_at=datetime(2026, 7, 31, 1, tzinfo=UTC), + ) + ) + candidate["evaluationCase"] = invalid + case_path.write_text(json.dumps(candidate), encoding="utf-8") + + with pytest.raises((GoldenSetUnavailable, RuntimeError)): + admit_evaluation_case( + case_path, + golden_path=golden_path, + lock_path=lock_path, + ) + assert golden_path.read_bytes() == before + + _write_candidate(case_path, _intake_case()) + with pytest.raises(GoldenSetUnavailable, match="lock"): + admit_evaluation_case( + case_path, + golden_path=unlocked, + lock_path=tmp_path / "missing.lock.json", + ) + + +def test_intake_refuses_a_pilot_case_because_relock_is_a_separate_ceremony( + tmp_path: Path, +) -> None: + golden_path, lock_path = _locked_corpus(tmp_path) + case_path = tmp_path / "intake.json" + _write_candidate(case_path, _intake_case(partition="pilot")) + + with pytest.raises(GoldenSetUnavailable, match="dev"): + admit_evaluation_case( + case_path, + golden_path=golden_path, + lock_path=lock_path, + ) + + +def test_intake_refuses_a_tampered_candidate_without_writing(tmp_path: Path) -> None: + golden_path, lock_path = _locked_corpus(tmp_path) + before = golden_path.read_bytes() + candidate_path = tmp_path / "candidate.json" + _write_candidate(candidate_path, _intake_case()) + document = json.loads(candidate_path.read_text(encoding="utf-8")) + document["feedbackBinding"]["releaseGeneration"] = 8 + candidate_path.write_text(json.dumps(document), encoding="utf-8") + + with pytest.raises(RuntimeError, match="identity"): + admit_evaluation_case( + candidate_path, + golden_path=golden_path, + lock_path=lock_path, + ) + + assert golden_path.read_bytes() == before diff --git a/tests/unit/test_feedback_inbox.py b/tests/unit/test_feedback_inbox.py new file mode 100644 index 00000000..f5a58a40 --- /dev/null +++ b/tests/unit/test_feedback_inbox.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from datetime import UTC, datetime +from typing import cast +from uuid import UUID + +import pytest +from sqlalchemy import Engine +from sqlalchemy.exc import OperationalError + +import engine.persistence.feedback as feedback_persistence +from engine.learning.feedback import FeedbackBindingUnavailable +from engine.persistence.feedback import PostgreSQLFeedbackInbox + +ORGANIZATION_ID = UUID("00000000-0000-4000-8000-000000000152") +FEEDBACK_REF = "fb_" + "5" * 64 + + +def _document() -> dict[str, object]: + return { + "citations": [ + { + "evidenceRef": "ev_" + "6" * 64, + "fragmentRef": "synthetic-fragment-feedback", + "resourceRef": "synthetic-resource-feedback", + "revisionRef": "synthetic-revision-feedback", + "sourceRef": "synthetic-source-feedback", + } + ], + "feedbackRef": FEEDBACK_REF, + "note": "synthetic-feedback-note", + "organizationId": str(ORGANIZATION_ID), + "packageDigest": "3" * 64, + "packageRef": "pkg_" + "2" * 32, + "rating": "not_helpful", + "recordedAt": datetime(2026, 7, 31, tzinfo=UTC).isoformat(), + "releaseGeneration": 7, + "releaseRef": "rel_" + "4" * 64, + "runRef": "run_" + "1" * 32, + "schemaVersion": "context-engine-feedback-evidence-v1", + } + + +class _Result: + def __init__(self, value: object) -> None: + self._value = value + + def scalar_one_or_none(self) -> object: + return self._value + + +class _Connection: + def __init__( + self, + result: object, + *, + error: Exception | None = None, + ) -> None: + self.result = result + self.error = error + self.calls: list[tuple[str, dict[str, object]]] = [] + + def execute( + self, + statement: object, + parameters: Mapping[str, object] | None = None, + ) -> _Result: + sql = str(statement) + values = dict(parameters or {}) + self.calls.append((sql, values)) + if self.error is not None: + raise self.error + return _Result(self.result) + + +class _Engine: + def __init__(self, connection: _Connection) -> None: + self.connection = connection + + @contextmanager + def begin(self) -> Iterator[_Connection]: + yield self.connection + + +@pytest.fixture +def accept_fake_engine(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(feedback_persistence, "Engine", _Engine) + + +def _inbox(connection: _Connection) -> PostgreSQLFeedbackInbox: + return PostgreSQLFeedbackInbox(cast(Engine, _Engine(connection))) + + +def test_inbox_requires_learning_role_and_calls_only_the_exact_read_function( + accept_fake_engine: None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + connection = _Connection(_document()) + guarded: list[object] = [] + monkeypatch.setattr( + feedback_persistence, + "assert_learning_role", + lambda value: guarded.append(value), + ) + + evidence = _inbox(connection).find_exact(ORGANIZATION_ID, FEEDBACK_REF) + + assert guarded == [connection] + assert len(connection.calls) == 1 + sql, parameters = connection.calls[0] + assert "context_learning_read_feedback_evidence" in sql + assert "promote" not in sql + assert "grant" not in sql + assert parameters == { + "organization_id": ORGANIZATION_ID, + "feedback_ref": FEEDBACK_REF, + } + assert evidence.binding.release_generation == 7 + rendered = repr(evidence) + assert "synthetic-feedback-note" not in rendered + assert "denied" not in rendered.casefold() + + +@pytest.mark.parametrize("projection", [None, {}, {"feedbackRef": FEEDBACK_REF}]) +def test_inbox_refuses_missing_or_malformed_projection( + projection: object, + accept_fake_engine: None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + feedback_persistence, + "assert_learning_role", + lambda connection: None, + ) + + with pytest.raises(FeedbackBindingUnavailable, match="unavailable|malformed"): + _inbox(_Connection(projection)).find_exact(ORGANIZATION_ID, FEEDBACK_REF) + + +@pytest.mark.parametrize( + "failure", + [ + AssertionError("wrong role"), + OperationalError("SELECT", {}, Exception("database refused")), + ], +) +def test_inbox_normalizes_role_and_database_failures( + failure: Exception, + accept_fake_engine: None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + if isinstance(failure, AssertionError): + monkeypatch.setattr( + feedback_persistence, + "assert_learning_role", + lambda connection: (_ for _ in ()).throw(failure), + ) + connection = _Connection(_document()) + else: + monkeypatch.setattr( + feedback_persistence, + "assert_learning_role", + lambda connection: None, + ) + connection = _Connection(None, error=failure) + + with pytest.raises( + FeedbackBindingUnavailable, + match="feedback exact binding is unavailable", + ): + _inbox(connection).find_exact(ORGANIZATION_ID, FEEDBACK_REF) diff --git a/tests/unit/test_feedback_triage.py b/tests/unit/test_feedback_triage.py new file mode 100644 index 00000000..a8294b74 --- /dev/null +++ b/tests/unit/test_feedback_triage.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import json +from dataclasses import replace +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, cast +from uuid import UUID + +import pytest + +from engine.learning.feedback import ( + FeedbackBinding, + FeedbackBindingUnavailable, + FeedbackCitation, + FeedbackEvidence, + TriageCategory, + load_feedback_evidence, + triage_feedback, +) +from engine.learning.golden import EvidenceLineage + + +def _binding() -> FeedbackBinding: + return FeedbackBinding( + organization_id=UUID("00000000-0000-4000-8000-000000000152"), + run_ref="run_" + "1" * 32, + package_ref="pkg_" + "2" * 32, + package_digest="3" * 64, + release_ref="rel_" + "4" * 64, + release_generation=7, + citations=( + FeedbackCitation( + evidence_ref="ev_" + "6" * 64, + lineage=EvidenceLineage( + source_ref="synthetic-source-feedback", + resource_ref="synthetic-resource-feedback", + revision_ref="synthetic-revision-feedback", + fragment_ref="synthetic-fragment-feedback", + ), + ), + ), + ) + + +def _feedback() -> FeedbackEvidence: + return FeedbackEvidence( + feedback_ref="fb_" + "5" * 64, + binding=_binding(), + rating="not_helpful", + note="synthetic-feedback-note", + recorded_at=datetime(2026, 7, 31, tzinfo=UTC), + ) + + +@pytest.mark.parametrize( + "category", + tuple(TriageCategory), +) +def test_triage_accepts_only_the_closed_categories( + category: TriageCategory, +) -> None: + item = triage_feedback(_feedback(), category) + + assert item.category is category + assert item.binding == _binding() + assert item.feedback_ref == "fb_" + "5" * 64 + + +def test_unknown_triage_category_is_refused() -> None: + with pytest.raises(FeedbackBindingUnavailable, match="category"): + triage_feedback(_feedback(), "ranking") # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("field", "replacement"), + ( + ("run_ref", ""), + ("package_ref", "pkg-not-closed"), + ("package_digest", "0" * 63), + ("release_ref", "rel-not-closed"), + ("release_generation", 0), + ("citations", ()), + ), +) +def test_partial_feedback_lineage_is_refused( + field: str, + replacement: object, +) -> None: + with pytest.raises((FeedbackBindingUnavailable, ValueError)): + replace(_binding(), **cast(Any, {field: replacement})) + + +def test_denied_details_cannot_enter_triage_fields() -> None: + with pytest.raises(TypeError): + FeedbackEvidence( # type: ignore[call-arg] + feedback_ref="fb_" + "5" * 64, + binding=_binding(), + rating="not_helpful", + note="synthetic-feedback-note", + recorded_at=datetime(2026, 7, 31, tzinfo=UTC), + denied_details="synthetic-denied-resource", + ) + + +def test_closed_feedback_projection_loads_exact_delivery_identity( + tmp_path: Path, +) -> None: + path = tmp_path / "feedback.json" + path.write_text( + json.dumps( + { + "citations": [ + { + "evidenceRef": "ev_" + "6" * 64, + "fragmentRef": "synthetic-fragment-feedback", + "resourceRef": "synthetic-resource-feedback", + "revisionRef": "synthetic-revision-feedback", + "sourceRef": "synthetic-source-feedback", + } + ], + "feedbackRef": "fb_" + "5" * 64, + "note": "synthetic-feedback-note", + "organizationId": "00000000-0000-4000-8000-000000000152", + "packageDigest": "3" * 64, + "packageRef": "pkg_" + "2" * 32, + "rating": "not_helpful", + "recordedAt": "2026-07-31T00:00:00Z", + "releaseGeneration": 7, + "releaseRef": "rel_" + "4" * 64, + "runRef": "run_" + "1" * 32, + "schemaVersion": "context-engine-feedback-evidence-v1", + } + ), + encoding="utf-8", + ) + + loaded = load_feedback_evidence(path) + + assert loaded == _feedback() + + +def test_feedback_projection_refuses_unknown_or_partial_fields(tmp_path: Path) -> None: + path = tmp_path / "feedback.json" + path.write_text(json.dumps({"feedbackRef": "fb_" + "5" * 64}), encoding="utf-8") + + with pytest.raises(FeedbackBindingUnavailable, match="malformed"): + load_feedback_evidence(path) diff --git a/tests/unit/test_golden_tracked_tree_is_synthetic.py b/tests/unit/test_golden_tracked_tree_is_synthetic.py index 63a6d679..e1149c10 100644 --- a/tests/unit/test_golden_tracked_tree_is_synthetic.py +++ b/tests/unit/test_golden_tracked_tree_is_synthetic.py @@ -33,6 +33,19 @@ def test_tracked_golden_tree_contains_only_placeholder_synthetic_cases() -> None assert_tracked_golden_tree_is_synthetic(REPOSITORY_ROOT / "eval/golden") +def test_feedback_intake_fixture_is_wholly_synthetic() -> None: + fixture = json.loads( + ( + REPOSITORY_ROOT + / "eval/golden/v1/synthetic-feedback-intake.json" + ).read_text(encoding="utf-8") + ) + + assert fixture["synthetic"] is True + assert fixture["entries"][0]["caseRef"].startswith("synthetic-") + assert fixture["entries"][0]["query"].startswith("synthetic-") + + def test_personal_or_non_placeholder_tracked_case_is_refused(tmp_path: Path) -> None: golden_root = tmp_path / "golden" golden_root.mkdir() diff --git a/tests/unit/test_schema_security_manifest.py b/tests/unit/test_schema_security_manifest.py index 9c386203..b2c67764 100644 --- a/tests/unit/test_schema_security_manifest.py +++ b/tests/unit/test_schema_security_manifest.py @@ -48,7 +48,7 @@ def test_manifest_classifies_the_exact_current_release_schema() -> None: document = manifest() tables = table_entries(document) - assert document["manifestVersion"] == "40.0.0" + assert document["manifestVersion"] == "41.0.0" assert set(tables) == { "active_release_manifest", "action_delivery_attempt", @@ -1122,6 +1122,7 @@ def test_issue_19_lineage_manifest_is_closed_and_role_separated() -> None: "ck_context_run_budget_usage_within_ceiling", "ck_context_run_outcome_evidence_consistency", "ck_context_run_timestamp_order", + "ck_context_run_feedback_lineage_complete", } expressions = " ".join( constraint["expression"] for constraint in run["checkConstraints"] @@ -1181,13 +1182,20 @@ def test_issue_19_lineage_manifest_is_closed_and_role_separated() -> None: "EXECUTE read_context_run_by_operator_ticket" ], "context_engine_context_run_reader_definer": ["SELECT"], + "context_engine_learning": [ + "EXECUTE context_learning_read_feedback_evidence" + ], "context_engine_worker": [], "context_engine_control": [ "EXECUTE issue_context_run_operator_read_ticket", "EXECUTE revoke_context_run_operator_read_ticket", ], } - assert audit["permittedOperations"] == run["permittedOperations"] + assert audit["permittedOperations"] == { + key: value + for key, value in run["permittedOperations"].items() + if key != "context_engine_learning" + } for entry in (run, audit): rls = entry["rowLevelSecurity"] assert rls["enabled"] is True