feat(ai-hub): real extraction-grounding metric on the evaluation surface

backend/api/ai_hub.py

@@ -16,6 +16,7 @@
     WorkflowDefinition,
 )
 from db.session import get_db
+from services.ai_grounding_eval import load_grounding_metrics
 from services.llm_provider_readiness import (
     is_llm_provider_configured,
     llm_provider_model_label,
@@ -505,6 +506,32 @@ async def get_ai_hub_surface(
         len(audit_events),
     )
 
+    grounding = await load_grounding_metrics(
+        db,
+        user_id=auth_context.user_id,
+        organization_id=auth_context.organization_id,
+        workspace_id=auth_context.workspace_id,
+    )
+    evaluation_metrics = _evaluation_metrics(
+        len(prompt_cards),
+        len(providers),
+        active_provider_count,
+        len(audit_events),
+    )
+    # Real AI-quality signal alongside the configuration-count metrics: how much
+    # extracted structure is bound to source evidence, and how often humans corrected it.
+    evaluation_metrics.append(
+        AiHubEvaluationMetric(
+            metric_key="extraction_grounding",
+            metric_label="추출 근거 결속률",
+            score_value=grounding.as_score(),
+            trend_text=(
+                f"{grounding.grounded_objects}/{grounding.total_objects} objects cite source"
+                f" · {grounding.correction_count} human corrections"
+            ),
+        )
+    )
+
     return AiHubSurfaceResponse(
         summary_cards=_summary_cards(
             len(prompt_cards),
@@ -517,11 +544,6 @@ async def get_ai_hub_surface(
         prompt_cards=prompt_cards,
         workflow_cards=workflow_cards,
         agent_cards=agent_cards,
-        evaluation_metrics=_evaluation_metrics(
-            len(prompt_cards),
-            len(providers),
-            active_provider_count,
-            len(audit_events),
-        ),
+        evaluation_metrics=evaluation_metrics,
         run_events=run_events,
     )

backend/services/ai_grounding_eval.py

@@ -0,0 +1,126 @@
+"""Grounding-quality metrics over stored project-graph extractions.
+
+Turns the AI Hub evaluation surface from configuration counts (how many prompts
+/ providers are set up) into a real, owner-scoped measurement of extraction
+*grounding*: how much extracted structure is bound to source evidence, how
+confident it is, and how often humans had to correct it. This is the
+measurement side of the evidence-based moat — you cannot claim or improve an AI
+advantage you do not measure.
+"""
+
+from dataclasses import dataclass
+
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from db.models import ProjectGraphCorrectionRecord, ProjectGraphObjectRecord
+
+LOW_CONFIDENCE_THRESHOLD = 0.5
+
+
+@dataclass(frozen=True)
+class GroundingMetrics:
+    total_objects: int
+    grounded_objects: int
+    grounding_rate: float
+    mean_confidence: float
+    low_confidence_objects: int
+    correction_count: int
+    correction_rate: float
+
+    def as_score(self) -> int:
+        """0-100 grounding score for the evaluation surface."""
+        return round(self.grounding_rate * 100)
+
+
+def compute_grounding_metrics(
+    *,
+    confidences: list[float],
+    citation_counts: list[int],
+    correction_count: int,
+) -> GroundingMetrics:
+    """Derive grounding metrics from per-object confidence and citation counts.
+
+    Pure: no DB, so the arithmetic is unit-testable in isolation. ``confidences``
+    and ``citation_counts`` are per-object and expected to be the same length;
+    an object is *grounded* when it cites at least one source segment.
+    """
+    total = len(confidences)
+    grounded = sum(1 for count in citation_counts if count > 0)
+    low_confidence = sum(1 for value in confidences if value < LOW_CONFIDENCE_THRESHOLD)
+    mean_confidence = (sum(confidences) / total) if total else 0.0
+    return GroundingMetrics(
+        total_objects=total,
+        grounded_objects=grounded,
+        grounding_rate=(grounded / total) if total else 0.0,
+        mean_confidence=mean_confidence,
+        low_confidence_objects=low_confidence,
+        correction_count=correction_count,
+        correction_rate=(correction_count / total) if total else 0.0,
+    )
+
+
+def _owner_predicates(
+    model, user_id: str, organization_id: str | None, workspace_id: str
+):
+    org_predicate = (
+        model.organization_id.is_(None)
+        if organization_id is None
+        else model.organization_id == organization_id
+    )
+    return (
+        model.user_id == user_id,
+        org_predicate,
+        model.workspace_id == workspace_id,
+    )
+
+
+async def load_grounding_metrics(
+    session: AsyncSession,
+    *,
+    user_id: str,
+    organization_id: str | None,
+    workspace_id: str,
+) -> GroundingMetrics:
+    """Owner-scoped fetch + compute over stored project-graph extraction objects.
+
+    Scoped to user + organization + workspace to match the AI Hub surface, so a
+    user's other workspaces do not bleed into one workspace's grounding number.
+    """
+    rows = (
+        await session.execute(
+            select(
+                ProjectGraphObjectRecord.confidence,
+                ProjectGraphObjectRecord.source_segment_uids,
+            ).where(
+                *_owner_predicates(
+                    ProjectGraphObjectRecord, user_id, organization_id, workspace_id
+                )
+            )
+        )
+    ).all()
+    # ponytail: loads per-object rows (bounded per owner at eval cadence); switch
+    # to a SQL JSON-length aggregate if object volume outgrows a single fetch.
+    confidences = [float(row.confidence) for row in rows]
+    citation_counts = [len(row.source_segment_uids or []) for row in rows]
+
+    correction_count = (
+        await session.execute(
+            select(
+                func.count(ProjectGraphCorrectionRecord.project_graph_correction_id)
+            ).where(
+                *_owner_predicates(
+                    ProjectGraphCorrectionRecord,
+                    user_id,
+                    organization_id,
+                    workspace_id,
+                )
+            )
+        )
+    ).scalar_one()
+
+    return compute_grounding_metrics(
+        confidences=confidences,
+        citation_counts=citation_counts,
+        correction_count=int(correction_count or 0),
+    )

backend/tests/test_ai_grounding_eval.py

@@ -0,0 +1,39 @@
+from services.ai_grounding_eval import compute_grounding_metrics
+
+
+def test_compute_grounding_metrics_typical():
+    metrics = compute_grounding_metrics(
+        confidences=[0.9, 0.4, 0.8, 0.3],
+        citation_counts=[2, 0, 1, 1],
+        correction_count=1,
+    )
+    assert metrics.total_objects == 4
+    assert metrics.grounded_objects == 3  # citation_counts > 0
+    assert metrics.grounding_rate == 0.75
+    assert metrics.low_confidence_objects == 2  # 0.4 and 0.3 are below 0.5
+    assert abs(metrics.mean_confidence - 0.6) < 1e-9
+    assert metrics.correction_count == 1
+    assert metrics.correction_rate == 0.25
+    assert metrics.as_score() == 75
+
+
+def test_compute_grounding_metrics_all_grounded():
+    metrics = compute_grounding_metrics(
+        confidences=[0.7, 0.9],
+        citation_counts=[1, 3],
+        correction_count=0,
+    )
+    assert metrics.grounding_rate == 1.0
+    assert metrics.low_confidence_objects == 0
+    assert metrics.as_score() == 100
+
+
+def test_compute_grounding_metrics_empty_is_safe():
+    metrics = compute_grounding_metrics(
+        confidences=[], citation_counts=[], correction_count=0
+    )
+    assert metrics.total_objects == 0
+    assert metrics.grounding_rate == 0.0
+    assert metrics.mean_confidence == 0.0
+    assert metrics.correction_rate == 0.0
+    assert metrics.as_score() == 0

backend/tests/test_ai_hub_api.py

@@ -48,6 +48,12 @@ def __init__(self, rows):
     def scalars(self):
         return MockScalars(self._rows)
 
+    def all(self):
+        return self._rows
+
+    def scalar_one(self):
+        return self._rows[0] if self._rows else 0
+
 
 class MockSession:
     def __init__(self):