Skip to content
Merged
9 changes: 9 additions & 0 deletions services/intake/src/nmp/intake/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,3 +61,12 @@ class IntakeConfig(_BaseIntakeConfig):
le=MAX_ATIF_MAX_SUBAGENT_DEPTH,
description="Maximum number of trajectory levels accepted for recursive ATIF subagents.",
)
denormalization_interval_seconds: float = Field(
default=10.0,
Comment thread
shanaiabuggy marked this conversation as resolved.
Resolved
gt=0,
description=(
"How often the background worker drains the dirty set and denormalizes the distinct "
"agent/model name fields from ClickHouse onto Evaluation entities. Bounds how stale those "
"fields can be after ingest; ingest never blocks on it."
),
)
20 changes: 20 additions & 0 deletions services/intake/src/nmp/intake/entities/experiments.py
Original file line number Diff line number Diff line change
Expand Up @@ -204,3 +204,23 @@ def _coerce_metadata(cls, value: Any) -> Any:
"Pin state is workspace-shared: every user with workspace access sees the same pinned set."
),
)

# System-managed denormalized name fields. These mirror the distinct name sets that the ClickHouse
# rollup already derives from this evaluation's sessions; a background refresher writes them here
# (see nmp.intake.experiments.denormalizer) so the workspace-wide Evaluations list can filter by
# agent/model name against the entity store ($contains) instead of scanning the session table. Not
# accepted on the create/update body (the request schemas omit them); default empty until refreshed.
# Unlike computed rollups, these are raw observed strings with no formula, so they never need a
# backfill/recompute when aggregation logic changes — only when a new name is ingested.
agent_names: list[str] = Field(
default_factory=list,
description="System-managed: distinct agent names observed across this evaluation's ingested sessions.",
)
agent_versions: list[str] = Field(
default_factory=list,
description="System-managed: distinct agent versions observed across this evaluation's ingested sessions.",
)
model_names: list[str] = Field(
default_factory=list,
description="System-managed: distinct model names observed across this evaluation's ingested sessions.",
)
131 changes: 131 additions & 0 deletions services/intake/src/nmp/intake/experiments/denormalizer.py
Comment thread
shanaiabuggy marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Background worker that denormalizes name fields from ClickHouse onto Evaluation entities.

Ingest marks ``(workspace, evaluation_id)`` dirty — a cheap, non-blocking set add. A background loop
drains the dirty set on a fixed interval, recomputes each touched evaluation's rollup, and writes the
distinct ``agent_names``/``agent_versions``/``model_names`` sets onto the (system-managed) fields of
its Evaluation entity. That lets the workspace-wide Evaluations list filter by agent/model name against
the entity store (``$contains``) instead of scanning the ClickHouse session table on every request.

This is a deliberately narrowed descendant of the closed PR #424 rollup refresher: it denormalizes
only the name fields, not the computed metric rollups. Names are raw observed strings with no formula,
so a change to how any aggregate is computed never invalidates them — the staleness/backfill hazard
that made denormalizing metrics not worth it (ASE-319) does not apply here. Bursts coalesce: many
ingests for one evaluation within an interval collapse to a single recompute, and ingest latency is
never gated on the rollup query.
"""

from __future__ import annotations

import asyncio
import logging

from nmp.common.entities.client import EntityClient, EntityConflictError, EntityNotFoundError
from nmp.intake.entities.experiments import Experiment
from nmp.intake.repository.evaluation_rollup import EvaluationRollup, EvaluationRollupRepository

logger = logging.getLogger(__name__)


class EvaluationDenormalizer:
"""Coalesces dirty evaluation ids and refreshes their denormalized name fields on a fixed cadence."""

def __init__(
self,
*,
rollup_repository: EvaluationRollupRepository,
entity_client: EntityClient,
interval_seconds: float = 10.0,
) -> None:
self._rollup_repository = rollup_repository
self._entity_client = entity_client
self._interval_seconds = interval_seconds
self._dirty: set[tuple[str, str]] = set()
self._task: asyncio.Task[None] | None = None
self._stopping = asyncio.Event()

def mark_dirty(self, *, workspace: str, evaluation_id: str) -> None:
"""Queue an evaluation for refresh. Cheap and non-blocking; safe to call from the ingest path."""
self._dirty.add((workspace, evaluation_id))

def pending(self) -> set[tuple[str, str]]:
"""Return a copy of the currently-queued ``(workspace, evaluation_id)`` pairs (observability/tests)."""
return set(self._dirty)

def start(self) -> None:
if self._task is None:
self._stopping.clear()
self._task = asyncio.create_task(self._run())

async def stop(self) -> None:
# Signal the loop to exit and let it finish any in-flight flush — we never cancel mid-flush, so a
# detached batch can't be dropped before it's written. Then a final drain covers the cases the loop
# can't (it saw the stop flag before its first flush, or items were enqueued during the last flush).
self._stopping.set()
if self._task is not None:
await self._task
self._task = None
await self.flush()

async def _run(self) -> None:
while not self._stopping.is_set():
try:
# Interruptible sleep: wakes early when stop() sets the event so shutdown is prompt.
await asyncio.wait_for(self._stopping.wait(), timeout=self._interval_seconds)
except asyncio.TimeoutError:
pass # interval elapsed; time for a periodic flush
try:
await self.flush()
except Exception:
logger.exception("Evaluation denormalization cycle failed")

async def flush(self) -> None:
"""Drain the dirty set and write current name fields. Directly callable for deterministic tests."""
if not self._dirty:
return
batch = self._dirty
self._dirty = set()
by_workspace: dict[str, list[str]] = {}
for workspace, evaluation_id in batch:
by_workspace.setdefault(workspace, []).append(evaluation_id)
for workspace, evaluation_ids in by_workspace.items():
try:
await self._refresh_workspace(workspace, evaluation_ids)
except Exception:
# Re-queue the whole workspace batch for the next cycle (e.g. ClickHouse unavailable).
logger.exception("Failed to refresh evaluation names for workspace %s; re-queuing", workspace)
for evaluation_id in evaluation_ids:
self._dirty.add((workspace, evaluation_id))

async def _refresh_workspace(self, workspace: str, evaluation_ids: list[str]) -> None:
rollups = await self._rollup_repository.get_rollups(workspace=workspace, evaluation_ids=evaluation_ids)
for evaluation_id in evaluation_ids:
rollup = rollups.get(evaluation_id)
if rollup is None:
continue
await self._write_names(workspace, evaluation_id, rollup)

async def _write_names(self, workspace: str, evaluation_id: str, rollup: EvaluationRollup) -> None:
try:
evaluation = await self._entity_client.get(Experiment, name=evaluation_id, workspace=workspace)
except EntityNotFoundError:
# Deleted between ingest and refresh; nothing to update.
return
# Skip the write when nothing changed, so a burst of re-ingests that add no new names doesn't
# churn the entity store (and doesn't lose an optimistic-lock race for no reason).
if (
evaluation.agent_names == rollup.agent_names
and evaluation.agent_versions == rollup.agent_versions
and evaluation.model_names == rollup.model_names
):
return
evaluation.agent_names = rollup.agent_names
evaluation.agent_versions = rollup.agent_versions
evaluation.model_names = rollup.model_names
try:
await self._entity_client.update(evaluation)
except EntityConflictError:
# A concurrent user edit won the optimistic lock; re-queue for the next cycle.
self._dirty.add((workspace, evaluation_id))
21 changes: 21 additions & 0 deletions services/intake/src/nmp/intake/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
from nmp.common.service import RouterConfig, Service
from nmp.intake.api.v2.experiments import endpoints as experiments
from nmp.intake.config import IntakeConfig
from nmp.intake.experiments.denormalizer import EvaluationDenormalizer
from nmp.intake.repository.clickhouse.evaluation_rollup import ClickHouseEvaluationRollupRepository
from nmp.intake.repository.clickhouse.executor import ClickHouseExecutor
from nmp.intake.spans.api import annotations, evaluator_results, sessions, spans, traces
from nmp.intake.spans.clickhouse_client import ClickHouseSettings, ClickHouseSpanClient
from nmp.intake.spans.ingest import atif, chat_completions, otlp
Expand All @@ -26,6 +29,8 @@ def __init__(self):
super().__init__(name="intake", module_name="nmp.intake")
# The client is owned by the service lifecycle; it is absent before startup and after shutdown.
self.clickhouse_client: ClickHouseSpanClient | None = None
# Background worker that denormalizes agent/model name fields onto Evaluation entities.
self.denormalizer: EvaluationDenormalizer | None = None
self._ready = False

@property
Expand Down Expand Up @@ -82,12 +87,28 @@ async def on_startup(self) -> None:
"clickhouse_database": cfg.clickhouse_config.database,
},
)
# Start the background denormalizer. It needs a service-principal entity client (no request
# context) to write onto Evaluation entities; skip it if the entity client can't be built.
entity_client = self.dependency_provider.get_entity_client(as_service=self.name)
if entity_client is not None:
self.denormalizer = EvaluationDenormalizer(
rollup_repository=ClickHouseEvaluationRollupRepository(ClickHouseExecutor(self.clickhouse_client)),
entity_client=entity_client,
interval_seconds=cfg.denormalization_interval_seconds,
)
self.denormalizer.start()
else:
logger.warning("Entity client unavailable; evaluation denormalizer not started")
self._ready = True

async def on_shutdown(self) -> None:
"""Close the service-owned ClickHouse client."""

self._ready = False
# Stop the refresher first: its final flush still needs the ClickHouse client below.
if self.denormalizer is not None:
await self.denormalizer.stop()
self.denormalizer = None
if self.clickhouse_client is not None:
await self.clickhouse_client.close()
self.clickhouse_client = None
Expand Down
10 changes: 10 additions & 0 deletions services/intake/src/nmp/intake/spans/api/dependencies.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from fastapi import Depends, HTTPException, Request, status
from nemo_platform import AsyncNeMoPlatform
from nmp.common.service.dependencies import get_sdk_client
from nmp.intake.experiments.denormalizer import EvaluationDenormalizer
from nmp.intake.repository.annotations import AnnotationsRepository
from nmp.intake.repository.clickhouse.annotations import ClickHouseAnnotationsRepository
from nmp.intake.repository.clickhouse.evaluator_results import ClickHouseEvaluatorResultsRepository
Expand Down Expand Up @@ -105,3 +106,12 @@ def get_spans_service(


SpansServiceDep = Annotated[IntakeSpansService, Depends(get_spans_service)]


def get_denormalizer(request: Request) -> EvaluationDenormalizer | None:
"""Reach the service-owned denormalizer from the request (absent if startup didn't create one)."""
service = getattr(request.app.state, "intake_service", None) or getattr(request.app.state, "service", None)
return getattr(service, "denormalizer", None) if service is not None else None


DenormalizerDep = Annotated[EvaluationDenormalizer | None, Depends(get_denormalizer)]
6 changes: 5 additions & 1 deletion services/intake/src/nmp/intake/spans/ingest/atif.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from nmp.common.entities.client import EntityClient
from nmp.common.service.dependencies import get_entity_client
from nmp.intake.config import IntakeConfig
from nmp.intake.spans.api.dependencies import SpansServiceDep, require_workspace_access
from nmp.intake.spans.api.dependencies import DenormalizerDep, SpansServiceDep, require_workspace_access
from nmp.intake.spans.domain import TraceBatch
from nmp.intake.spans.ingest.atif_domain import (
AtifAgent,
Expand Down Expand Up @@ -109,6 +109,7 @@ async def ingest_atif(
request: Request,
service: SpansServiceDep,
entity_client: EntityClientDep,
denormalizer: DenormalizerDep,
) -> Response:
await validate_evaluation_context(
workspace=workspace,
Expand All @@ -135,4 +136,7 @@ async def ingest_atif(
max_subagent_depth=max_subagent_depth,
)
await service.ingest_batch(TraceBatch(spans=spans, evaluator_results=evaluator_results))
context = body.evaluation_context
if denormalizer is not None and context is not None and context.evaluation_id:
denormalizer.mark_dirty(workspace=workspace, evaluation_id=context.evaluation_id)
return Response(status_code=status.HTTP_201_CREATED)
Comment thread
shanaiabuggy marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from fastapi import APIRouter, Depends, status
from nmp.common.entities.client import EntityClient
from nmp.common.service.dependencies import get_entity_client
from nmp.intake.spans.api.dependencies import SpansServiceDep, require_workspace_access
from nmp.intake.spans.api.dependencies import DenormalizerDep, SpansServiceDep, require_workspace_access
from nmp.intake.spans.domain import IntakeSpan, SpanKind, SpanStatus, TraceBatch
from nmp.intake.spans.ingest.evaluation_context import EvaluationContext
from nmp.intake.spans.ingest.evaluation_context_validation import validate_evaluation_context
Expand Down Expand Up @@ -151,6 +151,7 @@ async def ingest_chat_completion(
body: ChatCompletionsIngestRequest,
service: SpansServiceDep,
entity_client: EntityClientDep,
denormalizer: DenormalizerDep,
) -> ChatCompletionsIngestResponse:
await validate_evaluation_context(
workspace=workspace,
Expand All @@ -160,6 +161,9 @@ async def ingest_chat_completion(
ingested_at = utc_now()
span = _chat_completion_to_span(workspace=workspace, body=body, ingested_at=ingested_at)
await service.ingest_batch(TraceBatch(spans=[span]))
context = body.evaluation_context
if denormalizer is not None and context is not None and context.evaluation_id:
denormalizer.mark_dirty(workspace=workspace, evaluation_id=context.evaluation_id)
return ChatCompletionsIngestResponse(
session_id=span.session_id,
span_id=span.external_span_id,
Expand Down
Loading