Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,8 @@ CREATE TABLE IF NOT EXISTS llm_call_metrics (
correlation_id UUID,

-- Session/run context (logical references, no FK - async event arrival)
session_id VARCHAR(255) NOT NULL,
-- session_id is nullable until the write path is fully integrated
session_id VARCHAR(255),
run_id VARCHAR(255),

-- Model identification
Expand All @@ -80,8 +81,8 @@ CREATE TABLE IF NOT EXISTS llm_call_metrics (
-- Cost (NULL for unknown models, NOT 0)
estimated_cost_usd NUMERIC(12, 6),

-- Performance
latency_ms INTEGER NOT NULL,
-- Performance (nullable: failed LLM calls may not have latency data)
latency_ms NUMERIC(10, 2),

-- Usage provenance
usage_source usage_source_type NOT NULL DEFAULT 'MISSING',
Expand All @@ -91,7 +92,7 @@ CREATE TABLE IF NOT EXISTS llm_call_metrics (
usage_raw JSONB,

-- Provenance columns
input_hash VARCHAR(64),
input_hash VARCHAR(71),
code_version VARCHAR(64),
contract_version VARCHAR(64),
source VARCHAR(255),
Expand All @@ -112,9 +113,9 @@ CREATE TABLE IF NOT EXISTS llm_call_metrics (
CONSTRAINT non_negative_estimated_cost_usd CHECK (
estimated_cost_usd IS NULL OR estimated_cost_usd >= 0
),
CONSTRAINT non_negative_latency_ms CHECK (latency_ms >= 0),
CONSTRAINT non_negative_latency_ms CHECK (latency_ms IS NULL OR latency_ms >= 0),
CONSTRAINT usage_raw_size_limit CHECK (
usage_raw IS NULL OR pg_column_size(usage_raw) <= 65536
usage_raw IS NULL OR octet_length(usage_raw::text) <= 65536
)
);

Expand Down Expand Up @@ -170,9 +171,10 @@ CREATE TABLE IF NOT EXISTS llm_cost_aggregates (
-- INDEXES: llm_call_metrics
-- ============================================================================

-- Session lookups (most common query pattern)
-- Session lookups (most common query pattern, partial: excludes NULLs)
CREATE INDEX IF NOT EXISTS idx_llm_call_metrics_session_id
ON llm_call_metrics (session_id);
ON llm_call_metrics (session_id)
WHERE session_id IS NOT NULL;

-- Run lookups (within-session drill-down)
CREATE INDEX IF NOT EXISTS idx_llm_call_metrics_run_id
Expand Down Expand Up @@ -253,7 +255,8 @@ COMMENT ON COLUMN llm_call_metrics.id IS
COMMENT ON COLUMN llm_call_metrics.correlation_id IS
'Request correlation ID for tracing across services';
COMMENT ON COLUMN llm_call_metrics.session_id IS
'Session identifier (logical reference, no FK - async event arrival)';
'Session identifier (logical reference, no FK - async event arrival). '
'Nullable until write path is fully integrated.';
COMMENT ON COLUMN llm_call_metrics.run_id IS
'Run identifier within a session (logical reference, no FK)';
COMMENT ON COLUMN llm_call_metrics.model_id IS
Expand All @@ -267,15 +270,16 @@ COMMENT ON COLUMN llm_call_metrics.total_tokens IS
COMMENT ON COLUMN llm_call_metrics.estimated_cost_usd IS
'Estimated cost in USD (NULL for unknown models, NOT 0)';
COMMENT ON COLUMN llm_call_metrics.latency_ms IS
'LLM call latency in milliseconds';
'LLM call latency in milliseconds (sub-millisecond precision, 2 decimal places). '
'NULL for failed calls where latency was not captured.';
COMMENT ON COLUMN llm_call_metrics.usage_source IS
'Token usage data provenance: API (provider-reported), ESTIMATED (calculated), MISSING (unavailable)';
COMMENT ON COLUMN llm_call_metrics.usage_is_estimated IS
'Whether token usage values are estimated (TRUE) or API-reported (FALSE)';
COMMENT ON COLUMN llm_call_metrics.usage_raw IS
'Redacted provider response payload as JSONB (max 64KB)';
COMMENT ON COLUMN llm_call_metrics.input_hash IS
'SHA-256 hash of input for deduplication (first 64 hex chars)';
'SHA-256 hash of input for deduplication (sha256- prefix + 64 hex chars = 71 chars)';
COMMENT ON COLUMN llm_call_metrics.code_version IS
'Application code version that produced this record';
COMMENT ON COLUMN llm_call_metrics.contract_version IS
Expand Down
4 changes: 2 additions & 2 deletions docker/migrations/schema_fingerprint.sha256
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Schema migration fingerprint for omnibase_infra (auto-generated)
# Regenerate: python scripts/check_schema_fingerprint.py stamp
# Verify: python scripts/check_schema_fingerprint.py verify
sha256:56ef7e39b9d1af17bea9caa92336ce3b4acca18e30ac9c92cc539a42f1547135
generated_at: 2026-02-15T23:12:51Z
sha256:0db5f5c925c17240690fc1f82eeabeeb599bd12c2d5eedc40863a3e2a702ef13
generated_at: 2026-02-16T19:23:56Z
migration_file_count: 17
13 changes: 13 additions & 0 deletions src/omnibase_infra/nodes/effects/models/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,22 @@
- ModelRegistryRequest: Registry effect input request
- ModelRegistryResponse: Dual-backend registry operation response

Adapters:
- to_call_metrics: ModelLlmUsage -> ContractLlmCallMetrics
- to_usage_normalized: ModelLlmUsage -> ContractLlmUsageNormalized
- to_usage_raw: ModelLlmUsage -> ContractLlmUsageRaw

Note:
ModelBackendResult canonical location is omnibase_infra.models.model_backend_result.
Re-exported here for ergonomic access.
"""

from omnibase_infra.models.model_backend_result import ModelBackendResult
from omnibase_infra.nodes.effects.models.adapter_llm_usage_to_contract import (
to_call_metrics,
to_usage_normalized,
to_usage_raw,
)
from omnibase_infra.nodes.effects.models.model_effect_idempotency_config import (
ModelEffectIdempotencyConfig,
)
Expand Down Expand Up @@ -70,4 +80,7 @@
"ModelLlmUsage",
"ModelRegistryRequest",
"ModelRegistryResponse",
"to_call_metrics",
"to_usage_normalized",
"to_usage_raw",
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
# SPDX-License-Identifier: MIT
# Copyright (c) 2025 OmniNode Team
"""Adapter bridging ModelLlmUsage to SPI LLM cost tracking contracts.

This module provides pure functions that translate between the infra-layer
``ModelLlmUsage`` value object and the SPI measurement contracts:

- ``ContractLlmCallMetrics``
- ``ContractLlmUsageRaw``
- ``ContractLlmUsageNormalized``
- ``ContractEnumUsageSource``

All functions are stateless and produce frozen Pydantic models suitable for
downstream measurement pipeline ingestion.

Related:
- OMN-2318: Integrate SPI 0.9.0 LLM cost tracking contracts
- ModelLlmUsage: Source infra-layer usage model
- ContractLlmCallMetrics: Target SPI per-call metrics contract
"""

from __future__ import annotations

from omnibase_infra.nodes.effects.models.model_llm_usage import ModelLlmUsage
from omnibase_spi.contracts.measurement import (
ContractEnumUsageSource,
ContractLlmCallMetrics,
ContractLlmUsageNormalized,
ContractLlmUsageRaw,
)
Comment on lines +1 to +30

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Rename module to match required Python file naming patterns.

The current filename converter_llm_usage_to_contract.py does not match the allowed prefixes. Please rename it (and update imports) to an approved pattern (e.g., adapter_llm_usage_to_contract.py or model_llm_usage_to_contract.py, depending on intent).
As per coding guidelines, File naming must follow pattern: model_.py, adapter_.py, dispatcher_.py, enum_.py, mixin_.py, protocol_.py, service_.py, store_.py, validator_.py, registry_infra_.py.

Also applies to: 136-140

🤖 Prompt for AI Agents
In `@src/omnibase_infra/nodes/effects/models/converter_llm_usage_to_contract.py`
around lines 1 - 30, Rename the module file from
converter_llm_usage_to_contract.py to an approved pattern (e.g.,
adapter_llm_usage_to_contract.py or model_llm_usage_to_contract.py) and update
every import that references the old module name; specifically update imports
that reference symbols exported from the module such as ModelLlmUsage,
ContractLlmCallMetrics, ContractLlmUsageRaw, ContractLlmUsageNormalized and
ContractEnumUsageSource where the current import path points to
converter_llm_usage_to_contract, as well as any package __init__.py exports,
tests, and CI/deployment references; ensure the module-level docstring and
internal references remain unchanged and run tests to confirm no import errors.



def to_usage_raw(
usage: ModelLlmUsage,
provider: str = "",
) -> ContractLlmUsageRaw:
"""Convert raw provider usage data from ModelLlmUsage to ContractLlmUsageRaw.

When ``usage.raw_provider_usage`` is ``None``, returns an empty raw
container with the given provider identifier.

Args:
usage: The infra-layer usage model.
provider: Provider identifier (e.g. ``"openai"``, ``"ollama"``,
``"vllm"``).

Returns:
A frozen ``ContractLlmUsageRaw`` preserving the verbatim provider data.
"""
return ContractLlmUsageRaw(
provider=provider,
raw_data=usage.raw_provider_usage or {},
)


def to_usage_normalized(usage: ModelLlmUsage) -> ContractLlmUsageNormalized:
"""Convert ModelLlmUsage token counts to ContractLlmUsageNormalized.

Maps infra-layer field names (``tokens_input``, ``tokens_output``,
``tokens_total``) to the SPI canonical names (``prompt_tokens``,
``completion_tokens``, ``total_tokens``).

The ``source`` and ``usage_is_estimated`` fields are derived from
``usage.usage_source``:

- ``API`` -> ``source=API, usage_is_estimated=False``
- ``ESTIMATED`` -> ``source=ESTIMATED, usage_is_estimated=True``
- ``MISSING`` -> ``source=MISSING, usage_is_estimated=False``

Args:
usage: The infra-layer usage model.

Returns:
A frozen ``ContractLlmUsageNormalized`` with canonical token counts
and provenance.
"""
source = usage.usage_source
usage_is_estimated = source == ContractEnumUsageSource.ESTIMATED
total = usage.tokens_total or 0

return ContractLlmUsageNormalized(
prompt_tokens=usage.tokens_input,
completion_tokens=usage.tokens_output,
total_tokens=total,
source=source,
usage_is_estimated=usage_is_estimated,
)


def to_call_metrics(
usage: ModelLlmUsage,
model_id: str,
*,
provider: str = "",
latency_ms: float | None = None,
timestamp_iso: str = "",
reporting_source: str = "",
) -> ContractLlmCallMetrics:
"""Convert ModelLlmUsage to a full ContractLlmCallMetrics record.

Builds both the raw and normalized usage sub-contracts and assembles
them into the top-level per-call metrics contract.

Args:
usage: The infra-layer usage model.
model_id: Identifier of the LLM model (e.g. ``"gpt-4o"``,
``"qwen2.5-coder-14b"``). Required and must be non-empty.
provider: Provider identifier for the raw usage envelope.
latency_ms: End-to-end call latency in milliseconds.
timestamp_iso: ISO-8601 timestamp string of the call.
reporting_source: Provenance label for this metrics record
(e.g. ``"llm-inference-effect"``).

Returns:
A frozen ``ContractLlmCallMetrics`` with token counts, cost,
raw/normalized usage, and metadata.

Raises:
ValueError: If *model_id* is empty.
"""
if not model_id:
raise ValueError("model_id must be a non-empty string")

total = usage.tokens_total or 0
is_estimated = usage.usage_source == ContractEnumUsageSource.ESTIMATED

return ContractLlmCallMetrics(
model_id=model_id,
prompt_tokens=usage.tokens_input,
completion_tokens=usage.tokens_output,
total_tokens=total,
estimated_cost_usd=usage.cost_usd,
latency_ms=latency_ms,
usage_raw=to_usage_raw(usage, provider=provider),
usage_normalized=to_usage_normalized(usage),
usage_is_estimated=is_estimated,
timestamp_iso=timestamp_iso,
reporting_source=reporting_source,
)
Comment thread
coderabbitai[bot] marked this conversation as resolved.


__all__ = [
"to_call_metrics",
"to_usage_normalized",
"to_usage_raw",
]
40 changes: 35 additions & 5 deletions src/omnibase_infra/nodes/effects/models/model_llm_usage.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,23 @@
only the information that every major provider returns in its
``usage`` block: input tokens, output tokens, and a pre-computed total.

Cost tracking is deliberately left as ``None`` in v1 because unit
pricing varies across providers and is subject to change.
Since OMN-2318, ModelLlmUsage also carries usage provenance tracking
via ``usage_source`` (API, ESTIMATED, or MISSING) and preserves the
raw provider usage payload in ``raw_provider_usage`` for auditing.

Related:
- OMN-2103: Phase 3 shared LLM models
- OMN-2318: Integrate SPI 0.9.0 LLM cost tracking contracts
"""

from __future__ import annotations

from typing import Any

from pydantic import BaseModel, ConfigDict, Field, model_validator

from omnibase_spi.contracts.measurement import ContractEnumUsageSource


class ModelLlmUsage(BaseModel):
"""Token-usage summary returned by an LLM provider.
Expand All @@ -41,8 +47,15 @@ class ModelLlmUsage(BaseModel):
tokens_total: Total tokens consumed (input + output). Defaults to
``None`` which triggers auto-computation from
``tokens_input + tokens_output``.
cost_usd: Estimated cost in US dollars. Always ``None`` in v1;
reserved for future provider-specific billing integration.
cost_usd: Estimated cost in US dollars. ``None`` when cost has
not been computed.
usage_source: Provenance of the usage data. ``API`` when the
provider reported token counts directly, ``ESTIMATED`` when
counts were derived locally (e.g. via a tokenizer),
``MISSING`` when no usage data was available.
raw_provider_usage: Verbatim provider response ``usage`` block
preserved for auditing. ``None`` when the provider did not
return a usage block or when the handler chose not to capture it.

Example:
>>> usage = ModelLlmUsage(tokens_input=120, tokens_output=45)
Expand Down Expand Up @@ -71,7 +84,24 @@ class ModelLlmUsage(BaseModel):
cost_usd: float | None = Field(
default=None,
ge=0.0,
description="Cost in USD. Always None in v1.",
description="Estimated cost in USD. None when not computed.",
)
usage_source: ContractEnumUsageSource = Field(
default=ContractEnumUsageSource.MISSING,
description=(
"Provenance of the usage data: API (provider-reported), "
"ESTIMATED (locally computed), or MISSING (no data)."
),
)
# ONEX_EXCLUDE: any_type - Raw provider usage payload is an untyped dict
# because each LLM provider (OpenAI, Ollama, vLLM) returns a different
# wire format. The verbatim data is preserved for auditing, not processed.
raw_provider_usage: dict[str, Any] | None = Field(
default=None,
description=(
"Verbatim provider response usage block for auditing. "
"None when not captured."
),
)

@model_validator(mode="before")
Expand Down
Loading