From 385cd744039339a70bcafcebb4ac4564faabf8f1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 5 Aug 2026 01:46:52 +0900 Subject: [PATCH 01/27] feat(essay): add accessible standalone score report renderer --- .../fast_mlsirm/scoring/essay/report_html.py | 315 ++++++++++++++++++ 1 file changed, 315 insertions(+) create mode 100644 python/fast_mlsirm/scoring/essay/report_html.py diff --git a/python/fast_mlsirm/scoring/essay/report_html.py b/python/fast_mlsirm/scoring/essay/report_html.py new file mode 100644 index 000000000..defe7cda2 --- /dev/null +++ b/python/fast_mlsirm/scoring/essay/report_html.py @@ -0,0 +1,315 @@ +"""Accessible standalone HTML rendering for governed essay score reports. + +The renderer emits a source-text-free, script-free audit artifact from one exact +:class:`~fast_mlsirm.scoring.essay.reporting.EssayScoreReport`. It performs no +scoring, aggregation, psychometric estimation, validity inference, or deployment +authorization. +""" + +from __future__ import annotations + +from html import escape +import json +from pathlib import Path +from typing import Any + +from .._validation import assessment_error +from .reporting import EssayScoreReport, build_essay_score_report + +_DEFAULT_TITLE = "Governed Automated-Essay Score Report" +_VALIDITY_NOTICE = ( + "Review routing is an audit signal only. Absence of a trigger is not " + "evidence of scoring validity, fairness, reliability, interchangeability, " + "or authorization for consequential deployment." +) + + +def _validated_report(report: EssayScoreReport) -> EssayScoreReport: + """Replay one report through governed factories before serialization.""" + if not isinstance(report, EssayScoreReport): + raise assessment_error( + "invalid_essay_score_report", + "$.report", + "report must be an EssayScoreReport", + ) + replayed = build_essay_score_report( + report_id=report.report_id, + request=report.essay_request, + result=report.scoring_result, + engine=report.engine_descriptor, + additional_review_trigger_ids=report.review_trigger_ids, + metadata=report.metadata, + ) + if replayed.report_fingerprint != report.report_fingerprint: + raise assessment_error( + "essay_score_report_replay_mismatch", + "$.report", + "report content does not match a freshly validated report", + ) + return replayed + + +def _content_security_policy() -> str: + """Return a restrictive meta-delivered policy for the standalone artifact.""" + return "; ".join( + ( + "default-src 'none'", + "style-src 'unsafe-inline'", + "img-src data:", + "object-src 'none'", + "base-uri 'none'", + "form-action 'none'", + ) + ) + + +def _display(value: object | None) -> str: + """Return one escaped exact display value with an explicit missing marker.""" + return "Not applicable" if value is None else escape(str(value)) + + +def _definition_rows(rows: tuple[tuple[str, object], ...]) -> str: + """Render exact key-value provenance as a semantic definition list.""" + items = [] + for label, value in rows: + items.extend((f"
{escape(label)}
", f"
{_display(value)}
")) + return "\n".join(("
", *items, "
")) + + +def _table( + *, + caption: str, + headers: tuple[str, ...], + rows: tuple[tuple[object | None, ...], ...], + empty_message: str, +) -> str: + """Render an accessible exact-value table or one explicit empty state.""" + if not rows: + return f'

{escape(empty_message)}

' + heading = "".join(f'{escape(header)}' for header in headers) + body = [] + for row in rows: + cells = "".join(f"{_display(value)}" for value in row) + body.append(f"{cells}") + return "\n".join( + ( + '
', + "", + f"", + f"{heading}", + f"{''.join(body)}", + "
{escape(caption)}
", + "
", + ) + ) + + +def _criterion_rows(report: EssayScoreReport) -> tuple[tuple[object | None, ...], ...]: + """Return criterion outcomes without averaging or interpreting scores.""" + return tuple( + ( + observation.criterion_id, + observation.status.value, + observation.score_category, + observation.reason_code, + len(observation.evidence_references), + observation.observation_fingerprint, + ) + for observation in report.scoring_result.observations + ) + + +def _evidence_rows(report: EssayScoreReport) -> tuple[tuple[object | None, ...], ...]: + """Return source-text-free evidence identities for every observation.""" + return tuple( + ( + observation.criterion_id, + evidence.source_id, + evidence.span_id, + evidence.evidence_role.value, + evidence.content_fingerprint, + evidence.evidence_fingerprint, + ) + for observation in report.scoring_result.observations + for evidence in observation.evidence_references + ) + + +def _trigger_section(report: EssayScoreReport) -> str: + """Render every transparent review trigger or an explicit empty state.""" + if not report.review_trigger_ids: + return '

No structural review trigger was emitted.

' + items = "".join( + f"
  • {escape(trigger_id)}
    • " + for trigger_id in report.review_trigger_ids + ) + return f'' + + +def _canonical_json(report: EssayScoreReport) -> str: + """Return escaped deterministic JSON for exact audit reconstruction.""" + serialized = json.dumps( + report.to_dict(), + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + return escape(serialized) + + +def _css() -> str: + """Return compact accessible styling without external resources.""" + return """ +:root { color-scheme: light dark; font-family: system-ui, sans-serif; } +* { box-sizing: border-box; } +body { margin: 0; background: Canvas; color: CanvasText; } +main { width: min(1120px, calc(100% - 32px)); margin: 0 auto 48px; } +.skip-link { position: absolute; left: 8px; top: -80px; padding: 10px; background: Canvas; color: CanvasText; z-index: 10; } +.skip-link:focus { top: 8px; } +.hero { padding: 48px 0 24px; } +h1 { margin: 0 0 8px; font-size: clamp(2rem, 5vw, 3.2rem); } +.subtitle { margin: 0; max-width: 78ch; } +section { margin-top: 20px; padding: 20px; border: 1px solid GrayText; border-radius: 10px; } +.review-required { border-inline-start: 8px solid #9c2f1f; } +.review-clear { border-inline-start: 8px solid #357a38; } +.notice { padding: 14px; border: 2px solid currentColor; font-weight: 650; } +.details-grid { display: grid; grid-template-columns: minmax(150px, 0.35fr) 1fr; gap: 8px 16px; } +.details-grid dt { font-weight: 700; } +.details-grid dd { margin: 0; overflow-wrap: anywhere; } +.table-scroll { overflow-x: auto; } +.table-scroll:focus-visible, pre:focus-visible { outline: 3px solid Highlight; outline-offset: 3px; } +table { width: 100%; border-collapse: collapse; } +caption { text-align: left; font-weight: 700; margin-bottom: 8px; } +th, td { padding: 10px; border: 1px solid GrayText; text-align: left; vertical-align: top; overflow-wrap: anywhere; } +code, pre { font-family: ui-monospace, monospace; } +pre { max-height: 32rem; overflow: auto; padding: 16px; border: 1px solid GrayText; white-space: pre-wrap; overflow-wrap: anywhere; } +.empty-state { font-style: italic; } +@media (max-width: 640px) { .details-grid { grid-template-columns: 1fr; } .details-grid dd { margin-bottom: 8px; } } +""".strip() + + +def _render_html(report: EssayScoreReport, title: str) -> str: + """Assemble one complete script-free report document.""" + engine = report.engine_descriptor + request = report.essay_request.scoring_request + review_class = "review-required" if report.human_review_required else "review-clear" + review_label = "Human review required" if report.human_review_required else "No structural trigger" + provenance = _definition_rows( + ( + ("Report ID", report.report_id), + ("Report handle", report.report_handle), + ("Report fingerprint", report.report_fingerprint), + ("Schema version", report.schema_version), + ("Request fingerprint", request.request_fingerprint), + ("Result fingerprint", report.scoring_result.result_fingerprint), + ("Assessment fingerprint", request.assessment_fingerprint), + ("Rubric fingerprint", request.rubric_fingerprint), + ("Task revision fingerprint", request.task_revision_fingerprint), + ("Engine ID", engine.engine_id), + ("Engine family", engine.engine_family_id), + ("Engine version", engine.engine_version), + ("Engine fingerprint", engine.engine_fingerprint), + ) + ) + criteria = _table( + caption="Criterion-level scoring outcomes", + headers=( + "Criterion", + "Status", + "Score category", + "Reason code", + "Evidence count", + "Observation fingerprint", + ), + rows=_criterion_rows(report), + empty_message="No criterion observations are available.", + ) + evidence = _table( + caption="Source-text-free evidence references", + headers=( + "Criterion", + "Source ID", + "Span ID", + "Role", + "Content fingerprint", + "Evidence fingerprint", + ), + rows=_evidence_rows(report), + empty_message="No evidence references are attached to this report.", + ) + return "\n".join( + ( + "", + '', + "", + '', + '', + '', + f"{escape(title)}", + f"", + "", + "", + '', + '
    ', + '
    ', + f"

    {escape(title)}

    ", + '

    Exact governed scoring provenance and transparent review routing without source text.

    ', + "
    ", + f'
    ', + '

    Review routing

    ', + f"

    {escape(review_label)}

    ", + f'

    {escape(_VALIDITY_NOTICE)}

    ', + _trigger_section(report), + "
    ", + '
    ', + '

    Exact provenance

    ', + provenance, + "
    ", + '
    ', + '

    Criterion outcomes

    ', + criteria, + "
    ", + '
    ', + '

    Evidence references

    ', + evidence, + "
    ", + '
    ', + '

    Canonical JSON

    ', + '

    The complete deterministic report payload is available below for audit reconstruction.

    ', + '
    ',
+            _canonical_json(report),
+            "
    ", + "
    ", + "
    ", + "", + "", + ) + ) + + +def render_essay_score_report_html( + report: EssayScoreReport, + output_path: str | Path, + *, + title: str | None = None, +) -> Path: + """Write one replay-verified, accessible standalone HTML audit report. + + The artifact contains exact criterion, evidence, engine, assessment, rubric, + request, result, and report provenance but no prompt, essay, or source text. + Its review state is not a validity or deployment decision. + """ + validated = _validated_report(report) + output = Path(output_path) + if output.suffix.lower() != ".html": + raise ValueError("essay score report output path must end with .html") + resolved_title = _DEFAULT_TITLE if title is None else title + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(_render_html(validated, resolved_title), encoding="utf-8") + return output + + +__all__ = ["render_essay_score_report_html"] From 0ca0e549ec3913c5236f4b96346675ce9b2f72bb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 5 Aug 2026 01:47:12 +0900 Subject: [PATCH 02/27] feat(essay): export standalone report renderer --- python/fast_mlsirm/scoring/essay/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/fast_mlsirm/scoring/essay/__init__.py b/python/fast_mlsirm/scoring/essay/__init__.py index 952b9d1b6..0f1deebbe 100644 --- a/python/fast_mlsirm/scoring/essay/__init__.py +++ b/python/fast_mlsirm/scoring/essay/__init__.py @@ -21,6 +21,9 @@ from .contracts import build_essay_scoring_request as build_essay_scoring_request from .contracts import build_essay_submission as build_essay_submission from .contracts import score_essay_request as score_essay_request +from .report_html import ( + render_essay_score_report_html as render_essay_score_report_html, +) from .reporting import EssayScoreReport as EssayScoreReport from .reporting import ( MAX_ESSAY_REPORT_REVIEW_TRIGGERS as MAX_ESSAY_REPORT_REVIEW_TRIGGERS, @@ -45,5 +48,6 @@ "build_essay_score_report", "build_essay_scoring_request", "build_essay_submission", + "render_essay_score_report_html", "score_essay_request", ] From 725741b78ccb793d08859b8fcfbb3e4fd6a20280 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 5 Aug 2026 01:48:25 +0900 Subject: [PATCH 03/27] test(essay): cover standalone score report renderer --- tests/test_scoring_essay_report_html.py | 165 ++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 tests/test_scoring_essay_report_html.py diff --git a/tests/test_scoring_essay_report_html.py b/tests/test_scoring_essay_report_html.py new file mode 100644 index 000000000..fd0bebe42 --- /dev/null +++ b/tests/test_scoring_essay_report_html.py @@ -0,0 +1,165 @@ +"""Tests for accessible standalone governed essay score reports.""" + +from __future__ import annotations + +from pathlib import Path +import runpy + +import pytest + +import fast_mlsirm.scoring.essay.report_html as report_html +from fast_mlsirm.scoring import AssessmentSpecError, ObservationStatus +from fast_mlsirm.scoring.essay import ( + EssayReviewFlag, + build_essay_score_report, + render_essay_score_report_html, +) + +_REPORT_FIXTURES = runpy.run_path( + str(Path(__file__).with_name("test_scoring_essay_reporting.py")) +) +essay_request = _REPORT_FIXTURES["essay_request"] +result_bundle = _REPORT_FIXTURES["result_bundle"] + + +def clean_report(): + """Return one deterministic report without structural review triggers.""" + request = essay_request() + _engine, descriptor, result = result_bundle(request) + return build_essay_score_report( + report_id="essay_score_report", + request=request, + result=result, + engine=descriptor, + metadata={"workflow_stage": "pilot_review"}, + ) + + +def review_report(): + """Return one deterministic report with mandatory and added triggers.""" + request = essay_request( + review_flags=(EssayReviewFlag.OFF_TOPIC_RESPONSE,) + ) + _engine, descriptor, result = result_bundle( + request, + claim_evidence=(), + alignment_status=ObservationStatus.ABSTAINED, + alignment_reason="insufficient_evidence", + ) + return build_essay_score_report( + report_id="review_required_report", + request=request, + result=result, + engine=descriptor, + additional_review_trigger_ids=("scorer_disagreement",), + ) + + +def test_html_renderer_surface_is_explicit_and_documented() -> None: + """The renderer module exposes one documented reporting operation.""" + assert report_html.__all__ == ["render_essay_score_report_html"] + assert render_essay_score_report_html.__doc__ + + +def test_clean_report_renders_deterministic_accessible_exact_values( + tmp_path: Path, +) -> None: + """A clean report remains exact, script-free, accessible, and deterministic.""" + report = clean_report() + first_path = tmp_path / "nested_report" / "first.html" + second_path = tmp_path / "second.html" + + returned = render_essay_score_report_html(report, first_path) + render_essay_score_report_html(report, second_path) + first = first_path.read_text(encoding="utf-8") + second = second_path.read_text(encoding="utf-8") + + assert returned == first_path + assert first == second + assert "" in first + assert '