Skip to content

feat: feedback triage and governed curation candidates (#152) #199

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stone16 merged 3 commits into from
Jul 31, 2026
Merged

feat: feedback triage and governed curation candidates (#152) #199

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b459564
feat: govern feedback curation intake
stone16 Jul 31, 2026
237711d
fix: isolate curation workflow contracts
stone16 Jul 31, 2026
f13815c
test: preserve context run schema coverage
stone16 Jul 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ eval-v1:
--golden-set "$(GOLDEN_SET)" \
--lock "$(GOLDEN_LOCK)" \
--run "$(EVAL_RUN)" \
$(if $(GOLDEN_LINEAGE_MAP),--lineage-map "$(GOLDEN_LINEAGE_MAP)",) \
--output .context-engine/eval/golden-v1-report.json \
--generated-at "$(GENERATED_AT)"

Expand All @@ -66,6 +67,7 @@ eval-v1-execute:
--golden-set "$(GOLDEN_SET)" \
--lock "$(GOLDEN_LOCK)" \
--judgments "$(EVAL_JUDGMENTS)" \
$(if $(GOLDEN_LINEAGE_MAP),--lineage-map "$(GOLDEN_LINEAGE_MAP)",) \
--output .context-engine/eval/golden-v1-report.json \
--generated-at "$(GENERATED_AT)"

Expand Down
176 changes: 174 additions & 2 deletions applications/eval_v1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,40 @@
from collections.abc import Sequence
from datetime import datetime
from pathlib import Path
from typing import cast
from uuid import UUID

from engine.learning.comparison import (
EvaluationComparisonUnavailable,
compare_release_evaluations,
)
from engine.learning.curation_candidate import (
CurationCandidateUnavailable,
EvaluationCaseIntake,
build_curation_candidate,
curation_candidate_document,
)
from engine.learning.eval_run import (
EvaluationRunUnavailable,
bind_evaluation_report_to_release,
build_evaluation_report,
load_evaluation_run,
record_lineage_check,
)
from engine.learning.feedback import (
FeedbackBindingUnavailable,
FeedbackEvidence,
TriageCategory,
triage_feedback,
)
from engine.learning.golden import (
GoldenSet,
GoldenSetUnavailable,
create_golden_lock,
load_golden_case,
load_golden_set,
relock_golden_set,
)
from engine.learning.golden_intake import admit_evaluation_case
from engine.learning.golden_storage import (
durable_golden_root,
require_durable_golden_path,
Expand All @@ -40,6 +60,12 @@
require_resolved_lineage,
)
from engine.learning.thresholds import DEFAULT_THRESHOLDS_PATH, load_thresholds
from engine.persistence import (
DatabasePurpose,
PostgreSQLFeedbackInbox,
create_database_engine,
load_database_configuration,
)

PUBLIC_SUBSET_MAINTAINER_SECRET_ENV = (
"CONTEXT_ENGINE_PUBLIC_SUBSET_MAINTAINER_SECRET"
Expand Down Expand Up @@ -154,6 +180,24 @@ def _parser() -> argparse.ArgumentParser:
execute.add_argument("--lineage-map", type=Path)
execute.add_argument("--output", required=True, type=Path)
execute.add_argument("--generated-at", required=True, type=_time)

candidate = commands.add_parser("feedback-candidate")
candidate.add_argument("--organization-id", required=True, type=UUID)
candidate.add_argument("--feedback-ref", required=True)
candidate.add_argument("--category", required=True, choices=tuple(TriageCategory))
candidate.add_argument("--case", required=True, type=Path)
candidate.add_argument("--output", required=True, type=Path)
candidate.add_argument("--proposed-at", required=True, type=_time)

intake = commands.add_parser("feedback-intake")
intake.add_argument("--candidate", required=True, type=Path)
intake.add_argument("--golden-set", required=True, type=Path)
intake.add_argument("--lock", required=True, type=Path)

compare = commands.add_parser("compare-releases")
compare.add_argument("--active-report", required=True, type=Path)
compare.add_argument("--candidate-report", required=True, type=Path)
compare.add_argument("--output", required=True, type=Path)
return parser


Expand All @@ -168,6 +212,82 @@ def _write_report(path: Path, report: dict[str, object]) -> None:
print(f"golden v1 report written: digest={report['reportDigest']}", flush=True)


def _load_json(path: Path, name: str) -> object:
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, ValueError):
raise ValueError(f"{name} is unavailable") from None


def _write_json(path: Path, document: dict[str, object]) -> None:
try:
_require_ignored_output(path)
path.parent.mkdir(parents=True, exist_ok=True)
_require_ignored_output(path)
path.write_text(
json.dumps(document, ensure_ascii=False, indent=2, sort_keys=True)
+ "\n",
encoding="utf-8",
)
except OSError:
raise ValueError("evaluation output is unavailable") from None


def _require_private_candidate_path(path: Path) -> None:
try:
_require_ignored_output(path)
except ValueError:
root = durable_golden_root()
require_durable_golden_path(path, root=root)


def _write_private_candidate(path: Path, document: dict[str, object]) -> None:
try:
_require_private_candidate_path(path)
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
os.chmod(path.parent, 0o700)
flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
if hasattr(os, "O_NOFOLLOW"):
flags |= os.O_NOFOLLOW
descriptor = os.open(path, flags, 0o600)
try:
os.fchmod(descriptor, 0o600)
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
descriptor = -1
handle.write(
json.dumps(
document,
ensure_ascii=False,
indent=2,
sort_keys=True,
)
+ "\n"
)
handle.flush()
os.fsync(handle.fileno())
finally:
if descriptor >= 0:
os.close(descriptor)
except OSError:
raise ValueError("curation candidate output is unavailable") from None
Comment on lines +244 to +272

@coderabbitai coderabbitai Bot Jul 31, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔒 Security & Privacy | 🟡 Minor | ⚡ Quick win

Private-write path looks correct; one note on parent-directory chmod.

os.chmod(path.parent, 0o700) mutates permissions of a possibly pre-existing shared directory (e.g. the durable corpus root when --output sits directly under it), not just directories this call created. Consider restricting the chmod to newly created leaves.

🧰 Tools
🪛 ast-grep (0.45.0)

[info] 257-262: use jsonify instead of json.dumps for JSON output
Context: json.dumps(
document,
ensure_ascii=False,
indent=2,
sort_keys=True,
)
Note: [CWE-116] Improper Encoding or Escaping of Output.

(use-jsonify)

🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@applications/eval_v1.py` around lines 244 - 272, Update
_write_private_candidate so os.chmod(path.parent, 0o700) only applies when the
parent directory was newly created by this call, and does not alter permissions
on an existing shared directory. Preserve the existing secure mode for newly
created directories and the rest of the private-file write behavior.



def _captured_feedback(
organization_id: UUID,
feedback_ref: str,
) -> FeedbackEvidence:
engine = create_database_engine(
load_database_configuration(DatabasePurpose.LEARNING)
)
try:
return PostgreSQLFeedbackInbox(engine).find_exact(
organization_id,
feedback_ref,
)
finally:
engine.dispose()


def _resolved_lineage_check(
golden_set: GoldenSet,
lineage_map_path: Path | None,
Expand All @@ -188,16 +308,65 @@ def _record_lineage_check(
) -> dict[str, object]:
if lineage_check is None:
return report
return record_lineage_check(report, lineage_check)
return bind_evaluation_report_to_release(report, lineage_check)


def main(argv: Sequence[str] | None = None) -> None:
parser = _parser()
args = parser.parse_args(argv)
try:
if args.command == "feedback-candidate":
_require_private_candidate_path(args.case)
case_document = _load_json(args.case, "evaluation case intake")
candidate = build_curation_candidate(
triage_feedback(
_captured_feedback(args.organization_id, args.feedback_ref),
TriageCategory(args.category),
),
EvaluationCaseIntake(
case=load_golden_case(case_document),
synthetic=False,
),
proposed_at=args.proposed_at,
)
_write_private_candidate(
args.output,
curation_candidate_document(candidate),
)
print(
f"curation candidate written: digest={candidate.candidate_digest}",
flush=True,
)
return
if args.command == "compare-releases":
comparison = compare_release_evaluations(
_load_json(args.active_report, "active evaluation report"),
_load_json(args.candidate_report, "candidate evaluation report"),
)
_write_json(args.output, comparison)
slices = cast(list[object], comparison["slices"])
print(
"release evaluation comparison written: "
f"slices={len(slices)}",
flush=True,
)
return
durable_root = durable_golden_root()
require_durable_golden_path(args.golden_set, root=durable_root)
require_durable_golden_path(args.lock, root=durable_root)
if args.command == "feedback-intake":
_require_private_candidate_path(args.candidate)
receipt = admit_evaluation_case(
args.candidate,
golden_path=args.golden_set,
lock_path=args.lock,
)
print(
"golden v1 feedback case admitted: "
f"cases={receipt.case_count} digest={receipt.golden_digest}",
flush=True,
)
return
lineage_map_path = getattr(args, "lineage_map", None)
if lineage_map_path is not None:
require_durable_golden_path(lineage_map_path, root=durable_root)
Expand Down Expand Up @@ -293,7 +462,10 @@ def main(argv: Sequence[str] | None = None) -> None:
return
except (
GoldenSetUnavailable,
CurationCandidateUnavailable,
EvaluationRunUnavailable,
EvaluationComparisonUnavailable,
FeedbackBindingUnavailable,
LineageMapUnavailable,
StaleGoldenLineage,
ValueError,
Expand Down
2 changes: 1 addition & 1 deletion engine/learning/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""Public ContextLearning contracts and sole release owner."""
"""Public ContextLearning release contracts and sole release owner."""

from engine.learning.contracts import (
ContentProfileRef,
Expand Down
Loading
Loading