diff --git a/backend/api/ai_hub.py b/backend/api/ai_hub.py index 2b9f22644..d34b4bf8f 100644 --- a/backend/api/ai_hub.py +++ b/backend/api/ai_hub.py @@ -16,6 +16,7 @@ WorkflowDefinition, ) from db.session import get_db +from services.ai_grounding_eval import load_grounding_metrics from services.llm_provider_readiness import ( is_llm_provider_configured, llm_provider_model_label, @@ -505,6 +506,32 @@ async def get_ai_hub_surface( len(audit_events), ) + grounding = await load_grounding_metrics( + db, + user_id=auth_context.user_id, + organization_id=auth_context.organization_id, + workspace_id=auth_context.workspace_id, + ) + evaluation_metrics = _evaluation_metrics( + len(prompt_cards), + len(providers), + active_provider_count, + len(audit_events), + ) + # Real AI-quality signal alongside the configuration-count metrics: how much + # extracted structure is bound to source evidence, and how often humans corrected it. + evaluation_metrics.append( + AiHubEvaluationMetric( + metric_key="extraction_grounding", + metric_label="추출 근거 결속률", + score_value=grounding.as_score(), + trend_text=( + f"{grounding.grounded_objects}/{grounding.total_objects} objects cite source" + f" · {grounding.correction_count} human corrections" + ), + ) + ) + return AiHubSurfaceResponse( summary_cards=_summary_cards( len(prompt_cards), @@ -517,11 +544,6 @@ async def get_ai_hub_surface( prompt_cards=prompt_cards, workflow_cards=workflow_cards, agent_cards=agent_cards, - evaluation_metrics=_evaluation_metrics( - len(prompt_cards), - len(providers), - active_provider_count, - len(audit_events), - ), + evaluation_metrics=evaluation_metrics, run_events=run_events, ) diff --git a/backend/services/ai_grounding_eval.py b/backend/services/ai_grounding_eval.py new file mode 100644 index 000000000..2a9283b7c --- /dev/null +++ b/backend/services/ai_grounding_eval.py @@ -0,0 +1,126 @@ +"""Grounding-quality metrics over stored project-graph extractions. + +Turns the AI Hub evaluation surface from configuration counts (how many prompts +/ providers are set up) into a real, owner-scoped measurement of extraction +*grounding*: how much extracted structure is bound to source evidence, how +confident it is, and how often humans had to correct it. This is the +measurement side of the evidence-based moat — you cannot claim or improve an AI +advantage you do not measure. +""" + +from dataclasses import dataclass + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from db.models import ProjectGraphCorrectionRecord, ProjectGraphObjectRecord + +LOW_CONFIDENCE_THRESHOLD = 0.5 + + +@dataclass(frozen=True) +class GroundingMetrics: + total_objects: int + grounded_objects: int + grounding_rate: float + mean_confidence: float + low_confidence_objects: int + correction_count: int + correction_rate: float + + def as_score(self) -> int: + """0-100 grounding score for the evaluation surface.""" + return round(self.grounding_rate * 100) + + +def compute_grounding_metrics( + *, + confidences: list[float], + citation_counts: list[int], + correction_count: int, +) -> GroundingMetrics: + """Derive grounding metrics from per-object confidence and citation counts. + + Pure: no DB, so the arithmetic is unit-testable in isolation. ``confidences`` + and ``citation_counts`` are per-object and expected to be the same length; + an object is *grounded* when it cites at least one source segment. + """ + total = len(confidences) + grounded = sum(1 for count in citation_counts if count > 0) + low_confidence = sum(1 for value in confidences if value < LOW_CONFIDENCE_THRESHOLD) + mean_confidence = (sum(confidences) / total) if total else 0.0 + return GroundingMetrics( + total_objects=total, + grounded_objects=grounded, + grounding_rate=(grounded / total) if total else 0.0, + mean_confidence=mean_confidence, + low_confidence_objects=low_confidence, + correction_count=correction_count, + correction_rate=(correction_count / total) if total else 0.0, + ) + + +def _owner_predicates( + model, user_id: str, organization_id: str | None, workspace_id: str +): + org_predicate = ( + model.organization_id.is_(None) + if organization_id is None + else model.organization_id == organization_id + ) + return ( + model.user_id == user_id, + org_predicate, + model.workspace_id == workspace_id, + ) + + +async def load_grounding_metrics( + session: AsyncSession, + *, + user_id: str, + organization_id: str | None, + workspace_id: str, +) -> GroundingMetrics: + """Owner-scoped fetch + compute over stored project-graph extraction objects. + + Scoped to user + organization + workspace to match the AI Hub surface, so a + user's other workspaces do not bleed into one workspace's grounding number. + """ + rows = ( + await session.execute( + select( + ProjectGraphObjectRecord.confidence, + ProjectGraphObjectRecord.source_segment_uids, + ).where( + *_owner_predicates( + ProjectGraphObjectRecord, user_id, organization_id, workspace_id + ) + ) + ) + ).all() + # ponytail: loads per-object rows (bounded per owner at eval cadence); switch + # to a SQL JSON-length aggregate if object volume outgrows a single fetch. + confidences = [float(row.confidence) for row in rows] + citation_counts = [len(row.source_segment_uids or []) for row in rows] + + correction_count = ( + await session.execute( + select( + func.count(ProjectGraphCorrectionRecord.project_graph_correction_id) + ).where( + *_owner_predicates( + ProjectGraphCorrectionRecord, + user_id, + organization_id, + workspace_id, + ) + ) + ) + ).scalar_one() + + return compute_grounding_metrics( + confidences=confidences, + citation_counts=citation_counts, + correction_count=int(correction_count or 0), + ) diff --git a/backend/tests/test_ai_grounding_eval.py b/backend/tests/test_ai_grounding_eval.py new file mode 100644 index 000000000..b2ea7dee5 --- /dev/null +++ b/backend/tests/test_ai_grounding_eval.py @@ -0,0 +1,39 @@ +from services.ai_grounding_eval import compute_grounding_metrics + + +def test_compute_grounding_metrics_typical(): + metrics = compute_grounding_metrics( + confidences=[0.9, 0.4, 0.8, 0.3], + citation_counts=[2, 0, 1, 1], + correction_count=1, + ) + assert metrics.total_objects == 4 + assert metrics.grounded_objects == 3 # citation_counts > 0 + assert metrics.grounding_rate == 0.75 + assert metrics.low_confidence_objects == 2 # 0.4 and 0.3 are below 0.5 + assert abs(metrics.mean_confidence - 0.6) < 1e-9 + assert metrics.correction_count == 1 + assert metrics.correction_rate == 0.25 + assert metrics.as_score() == 75 + + +def test_compute_grounding_metrics_all_grounded(): + metrics = compute_grounding_metrics( + confidences=[0.7, 0.9], + citation_counts=[1, 3], + correction_count=0, + ) + assert metrics.grounding_rate == 1.0 + assert metrics.low_confidence_objects == 0 + assert metrics.as_score() == 100 + + +def test_compute_grounding_metrics_empty_is_safe(): + metrics = compute_grounding_metrics( + confidences=[], citation_counts=[], correction_count=0 + ) + assert metrics.total_objects == 0 + assert metrics.grounding_rate == 0.0 + assert metrics.mean_confidence == 0.0 + assert metrics.correction_rate == 0.0 + assert metrics.as_score() == 0 diff --git a/backend/tests/test_ai_hub_api.py b/backend/tests/test_ai_hub_api.py index 056aee399..a68cc15bc 100644 --- a/backend/tests/test_ai_hub_api.py +++ b/backend/tests/test_ai_hub_api.py @@ -48,6 +48,12 @@ def __init__(self, rows): def scalars(self): return MockScalars(self._rows) + def all(self): + return self._rows + + def scalar_one(self): + return self._rows[0] if self._rows else 0 + class MockSession: def __init__(self):