diff --git a/docker/migrations/forward/031_create_llm_call_metrics_and_cost_aggregates.sql b/docker/migrations/forward/031_create_llm_call_metrics_and_cost_aggregates.sql index 93041f9d64..30962d72d8 100644 --- a/docker/migrations/forward/031_create_llm_call_metrics_and_cost_aggregates.sql +++ b/docker/migrations/forward/031_create_llm_call_metrics_and_cost_aggregates.sql @@ -66,7 +66,8 @@ CREATE TABLE IF NOT EXISTS llm_call_metrics ( correlation_id UUID, -- Session/run context (logical references, no FK - async event arrival) - session_id VARCHAR(255) NOT NULL, + -- session_id is nullable until the write path is fully integrated + session_id VARCHAR(255), run_id VARCHAR(255), -- Model identification @@ -80,8 +81,8 @@ CREATE TABLE IF NOT EXISTS llm_call_metrics ( -- Cost (NULL for unknown models, NOT 0) estimated_cost_usd NUMERIC(12, 6), - -- Performance - latency_ms INTEGER NOT NULL, + -- Performance (nullable: failed LLM calls may not have latency data) + latency_ms NUMERIC(10, 2), -- Usage provenance usage_source usage_source_type NOT NULL DEFAULT 'MISSING', @@ -91,7 +92,7 @@ CREATE TABLE IF NOT EXISTS llm_call_metrics ( usage_raw JSONB, -- Provenance columns - input_hash VARCHAR(64), + input_hash VARCHAR(71), code_version VARCHAR(64), contract_version VARCHAR(64), source VARCHAR(255), @@ -112,9 +113,9 @@ CREATE TABLE IF NOT EXISTS llm_call_metrics ( CONSTRAINT non_negative_estimated_cost_usd CHECK ( estimated_cost_usd IS NULL OR estimated_cost_usd >= 0 ), - CONSTRAINT non_negative_latency_ms CHECK (latency_ms >= 0), + CONSTRAINT non_negative_latency_ms CHECK (latency_ms IS NULL OR latency_ms >= 0), CONSTRAINT usage_raw_size_limit CHECK ( - usage_raw IS NULL OR pg_column_size(usage_raw) <= 65536 + usage_raw IS NULL OR octet_length(usage_raw::text) <= 65536 ) ); @@ -170,9 +171,10 @@ CREATE TABLE IF NOT EXISTS llm_cost_aggregates ( -- INDEXES: llm_call_metrics -- ============================================================================ --- Session lookups (most common query pattern) +-- Session lookups (most common query pattern, partial: excludes NULLs) CREATE INDEX IF NOT EXISTS idx_llm_call_metrics_session_id - ON llm_call_metrics (session_id); + ON llm_call_metrics (session_id) + WHERE session_id IS NOT NULL; -- Run lookups (within-session drill-down) CREATE INDEX IF NOT EXISTS idx_llm_call_metrics_run_id @@ -253,7 +255,8 @@ COMMENT ON COLUMN llm_call_metrics.id IS COMMENT ON COLUMN llm_call_metrics.correlation_id IS 'Request correlation ID for tracing across services'; COMMENT ON COLUMN llm_call_metrics.session_id IS - 'Session identifier (logical reference, no FK - async event arrival)'; + 'Session identifier (logical reference, no FK - async event arrival). ' + 'Nullable until write path is fully integrated.'; COMMENT ON COLUMN llm_call_metrics.run_id IS 'Run identifier within a session (logical reference, no FK)'; COMMENT ON COLUMN llm_call_metrics.model_id IS @@ -267,7 +270,8 @@ COMMENT ON COLUMN llm_call_metrics.total_tokens IS COMMENT ON COLUMN llm_call_metrics.estimated_cost_usd IS 'Estimated cost in USD (NULL for unknown models, NOT 0)'; COMMENT ON COLUMN llm_call_metrics.latency_ms IS - 'LLM call latency in milliseconds'; + 'LLM call latency in milliseconds (sub-millisecond precision, 2 decimal places). ' + 'NULL for failed calls where latency was not captured.'; COMMENT ON COLUMN llm_call_metrics.usage_source IS 'Token usage data provenance: API (provider-reported), ESTIMATED (calculated), MISSING (unavailable)'; COMMENT ON COLUMN llm_call_metrics.usage_is_estimated IS @@ -275,7 +279,7 @@ COMMENT ON COLUMN llm_call_metrics.usage_is_estimated IS COMMENT ON COLUMN llm_call_metrics.usage_raw IS 'Redacted provider response payload as JSONB (max 64KB)'; COMMENT ON COLUMN llm_call_metrics.input_hash IS - 'SHA-256 hash of input for deduplication (first 64 hex chars)'; + 'SHA-256 hash of input for deduplication (sha256- prefix + 64 hex chars = 71 chars)'; COMMENT ON COLUMN llm_call_metrics.code_version IS 'Application code version that produced this record'; COMMENT ON COLUMN llm_call_metrics.contract_version IS diff --git a/docker/migrations/schema_fingerprint.sha256 b/docker/migrations/schema_fingerprint.sha256 index 654735ca0d..2fe62c04c4 100644 --- a/docker/migrations/schema_fingerprint.sha256 +++ b/docker/migrations/schema_fingerprint.sha256 @@ -1,6 +1,6 @@ # Schema migration fingerprint for omnibase_infra (auto-generated) # Regenerate: python scripts/check_schema_fingerprint.py stamp # Verify: python scripts/check_schema_fingerprint.py verify -sha256:56ef7e39b9d1af17bea9caa92336ce3b4acca18e30ac9c92cc539a42f1547135 -generated_at: 2026-02-15T23:12:51Z +sha256:0db5f5c925c17240690fc1f82eeabeeb599bd12c2d5eedc40863a3e2a702ef13 +generated_at: 2026-02-16T19:23:56Z migration_file_count: 17 diff --git a/src/omnibase_infra/nodes/effects/models/__init__.py b/src/omnibase_infra/nodes/effects/models/__init__.py index 1508979a1b..e3e1c9598e 100644 --- a/src/omnibase_infra/nodes/effects/models/__init__.py +++ b/src/omnibase_infra/nodes/effects/models/__init__.py @@ -19,12 +19,22 @@ - ModelRegistryRequest: Registry effect input request - ModelRegistryResponse: Dual-backend registry operation response +Adapters: + - to_call_metrics: ModelLlmUsage -> ContractLlmCallMetrics + - to_usage_normalized: ModelLlmUsage -> ContractLlmUsageNormalized + - to_usage_raw: ModelLlmUsage -> ContractLlmUsageRaw + Note: ModelBackendResult canonical location is omnibase_infra.models.model_backend_result. Re-exported here for ergonomic access. """ from omnibase_infra.models.model_backend_result import ModelBackendResult +from omnibase_infra.nodes.effects.models.adapter_llm_usage_to_contract import ( + to_call_metrics, + to_usage_normalized, + to_usage_raw, +) from omnibase_infra.nodes.effects.models.model_effect_idempotency_config import ( ModelEffectIdempotencyConfig, ) @@ -70,4 +80,7 @@ "ModelLlmUsage", "ModelRegistryRequest", "ModelRegistryResponse", + "to_call_metrics", + "to_usage_normalized", + "to_usage_raw", ] diff --git a/src/omnibase_infra/nodes/effects/models/adapter_llm_usage_to_contract.py b/src/omnibase_infra/nodes/effects/models/adapter_llm_usage_to_contract.py new file mode 100644 index 0000000000..935603bd8b --- /dev/null +++ b/src/omnibase_infra/nodes/effects/models/adapter_llm_usage_to_contract.py @@ -0,0 +1,146 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2025 OmniNode Team +"""Adapter bridging ModelLlmUsage to SPI LLM cost tracking contracts. + +This module provides pure functions that translate between the infra-layer +``ModelLlmUsage`` value object and the SPI measurement contracts: + +- ``ContractLlmCallMetrics`` +- ``ContractLlmUsageRaw`` +- ``ContractLlmUsageNormalized`` +- ``ContractEnumUsageSource`` + +All functions are stateless and produce frozen Pydantic models suitable for +downstream measurement pipeline ingestion. + +Related: + - OMN-2318: Integrate SPI 0.9.0 LLM cost tracking contracts + - ModelLlmUsage: Source infra-layer usage model + - ContractLlmCallMetrics: Target SPI per-call metrics contract +""" + +from __future__ import annotations + +from omnibase_infra.nodes.effects.models.model_llm_usage import ModelLlmUsage +from omnibase_spi.contracts.measurement import ( + ContractEnumUsageSource, + ContractLlmCallMetrics, + ContractLlmUsageNormalized, + ContractLlmUsageRaw, +) + + +def to_usage_raw( + usage: ModelLlmUsage, + provider: str = "", +) -> ContractLlmUsageRaw: + """Convert raw provider usage data from ModelLlmUsage to ContractLlmUsageRaw. + + When ``usage.raw_provider_usage`` is ``None``, returns an empty raw + container with the given provider identifier. + + Args: + usage: The infra-layer usage model. + provider: Provider identifier (e.g. ``"openai"``, ``"ollama"``, + ``"vllm"``). + + Returns: + A frozen ``ContractLlmUsageRaw`` preserving the verbatim provider data. + """ + return ContractLlmUsageRaw( + provider=provider, + raw_data=usage.raw_provider_usage or {}, + ) + + +def to_usage_normalized(usage: ModelLlmUsage) -> ContractLlmUsageNormalized: + """Convert ModelLlmUsage token counts to ContractLlmUsageNormalized. + + Maps infra-layer field names (``tokens_input``, ``tokens_output``, + ``tokens_total``) to the SPI canonical names (``prompt_tokens``, + ``completion_tokens``, ``total_tokens``). + + The ``source`` and ``usage_is_estimated`` fields are derived from + ``usage.usage_source``: + + - ``API`` -> ``source=API, usage_is_estimated=False`` + - ``ESTIMATED`` -> ``source=ESTIMATED, usage_is_estimated=True`` + - ``MISSING`` -> ``source=MISSING, usage_is_estimated=False`` + + Args: + usage: The infra-layer usage model. + + Returns: + A frozen ``ContractLlmUsageNormalized`` with canonical token counts + and provenance. + """ + source = usage.usage_source + usage_is_estimated = source == ContractEnumUsageSource.ESTIMATED + total = usage.tokens_total or 0 + + return ContractLlmUsageNormalized( + prompt_tokens=usage.tokens_input, + completion_tokens=usage.tokens_output, + total_tokens=total, + source=source, + usage_is_estimated=usage_is_estimated, + ) + + +def to_call_metrics( + usage: ModelLlmUsage, + model_id: str, + *, + provider: str = "", + latency_ms: float | None = None, + timestamp_iso: str = "", + reporting_source: str = "", +) -> ContractLlmCallMetrics: + """Convert ModelLlmUsage to a full ContractLlmCallMetrics record. + + Builds both the raw and normalized usage sub-contracts and assembles + them into the top-level per-call metrics contract. + + Args: + usage: The infra-layer usage model. + model_id: Identifier of the LLM model (e.g. ``"gpt-4o"``, + ``"qwen2.5-coder-14b"``). Required and must be non-empty. + provider: Provider identifier for the raw usage envelope. + latency_ms: End-to-end call latency in milliseconds. + timestamp_iso: ISO-8601 timestamp string of the call. + reporting_source: Provenance label for this metrics record + (e.g. ``"llm-inference-effect"``). + + Returns: + A frozen ``ContractLlmCallMetrics`` with token counts, cost, + raw/normalized usage, and metadata. + + Raises: + ValueError: If *model_id* is empty. + """ + if not model_id: + raise ValueError("model_id must be a non-empty string") + + total = usage.tokens_total or 0 + is_estimated = usage.usage_source == ContractEnumUsageSource.ESTIMATED + + return ContractLlmCallMetrics( + model_id=model_id, + prompt_tokens=usage.tokens_input, + completion_tokens=usage.tokens_output, + total_tokens=total, + estimated_cost_usd=usage.cost_usd, + latency_ms=latency_ms, + usage_raw=to_usage_raw(usage, provider=provider), + usage_normalized=to_usage_normalized(usage), + usage_is_estimated=is_estimated, + timestamp_iso=timestamp_iso, + reporting_source=reporting_source, + ) + + +__all__ = [ + "to_call_metrics", + "to_usage_normalized", + "to_usage_raw", +] diff --git a/src/omnibase_infra/nodes/effects/models/model_llm_usage.py b/src/omnibase_infra/nodes/effects/models/model_llm_usage.py index 4fb6142cea..4f7b0a337c 100644 --- a/src/omnibase_infra/nodes/effects/models/model_llm_usage.py +++ b/src/omnibase_infra/nodes/effects/models/model_llm_usage.py @@ -11,17 +11,23 @@ only the information that every major provider returns in its ``usage`` block: input tokens, output tokens, and a pre-computed total. - Cost tracking is deliberately left as ``None`` in v1 because unit - pricing varies across providers and is subject to change. + Since OMN-2318, ModelLlmUsage also carries usage provenance tracking + via ``usage_source`` (API, ESTIMATED, or MISSING) and preserves the + raw provider usage payload in ``raw_provider_usage`` for auditing. Related: - OMN-2103: Phase 3 shared LLM models + - OMN-2318: Integrate SPI 0.9.0 LLM cost tracking contracts """ from __future__ import annotations +from typing import Any + from pydantic import BaseModel, ConfigDict, Field, model_validator +from omnibase_spi.contracts.measurement import ContractEnumUsageSource + class ModelLlmUsage(BaseModel): """Token-usage summary returned by an LLM provider. @@ -41,8 +47,15 @@ class ModelLlmUsage(BaseModel): tokens_total: Total tokens consumed (input + output). Defaults to ``None`` which triggers auto-computation from ``tokens_input + tokens_output``. - cost_usd: Estimated cost in US dollars. Always ``None`` in v1; - reserved for future provider-specific billing integration. + cost_usd: Estimated cost in US dollars. ``None`` when cost has + not been computed. + usage_source: Provenance of the usage data. ``API`` when the + provider reported token counts directly, ``ESTIMATED`` when + counts were derived locally (e.g. via a tokenizer), + ``MISSING`` when no usage data was available. + raw_provider_usage: Verbatim provider response ``usage`` block + preserved for auditing. ``None`` when the provider did not + return a usage block or when the handler chose not to capture it. Example: >>> usage = ModelLlmUsage(tokens_input=120, tokens_output=45) @@ -71,7 +84,24 @@ class ModelLlmUsage(BaseModel): cost_usd: float | None = Field( default=None, ge=0.0, - description="Cost in USD. Always None in v1.", + description="Estimated cost in USD. None when not computed.", + ) + usage_source: ContractEnumUsageSource = Field( + default=ContractEnumUsageSource.MISSING, + description=( + "Provenance of the usage data: API (provider-reported), " + "ESTIMATED (locally computed), or MISSING (no data)." + ), + ) + # ONEX_EXCLUDE: any_type - Raw provider usage payload is an untyped dict + # because each LLM provider (OpenAI, Ollama, vLLM) returns a different + # wire format. The verbatim data is preserved for auditing, not processed. + raw_provider_usage: dict[str, Any] | None = Field( + default=None, + description=( + "Verbatim provider response usage block for auditing. " + "None when not captured." + ), ) @model_validator(mode="before") diff --git a/src/omnibase_infra/nodes/node_llm_embedding_effect/handlers/handler_embedding_ollama.py b/src/omnibase_infra/nodes/node_llm_embedding_effect/handlers/handler_embedding_ollama.py index 4b1123a220..0932ffdd82 100644 --- a/src/omnibase_infra/nodes/node_llm_embedding_effect/handlers/handler_embedding_ollama.py +++ b/src/omnibase_infra/nodes/node_llm_embedding_effect/handlers/handler_embedding_ollama.py @@ -53,6 +53,7 @@ from omnibase_infra.nodes.node_llm_embedding_effect.models.model_llm_embedding_response import ( ModelLlmEmbeddingResponse, ) +from omnibase_spi.contracts.measurement import ContractEnumUsageSource logger = logging.getLogger(__name__) @@ -158,6 +159,17 @@ async def execute( context=ctx, ) from exc usage = _parse_ollama_usage(data) + if not embeddings: + ctx = ModelInfraErrorContext.with_correlation( + correlation_id=request.correlation_id, + transport_type=EnumInfraTransportType.HTTP, + operation="parse_ollama_embeddings", + target_name=self._llm_target_name, + ) + raise InfraProtocolError( + "Ollama embedding response returned no embeddings", + context=ctx, + ) dimensions = len(embeddings[0].vector) return ModelLlmEmbeddingResponse( @@ -217,19 +229,36 @@ def _parse_ollama_usage(data: dict[str, JsonType]) -> ModelLlmUsage: Ollama reports ``prompt_eval_count`` as the token count for input. Output tokens are not applicable for embeddings. + When ``prompt_eval_count`` is present and numeric, ``usage_source`` + is set to ``API``. Otherwise ``MISSING``. + Args: data: Parsed JSON response body. Returns: - ModelLlmUsage with token counts. Zeros if fields are absent. + ModelLlmUsage with token counts and provenance. Zeros if fields + are absent. """ prompt_eval_count = data.get("prompt_eval_count", 0) + has_usage = isinstance(prompt_eval_count, int) and prompt_eval_count > 0 if not isinstance(prompt_eval_count, int): prompt_eval_count = 0 + raw_usage_data: dict[str, object] = { + "prompt_eval_count": data.get("prompt_eval_count"), + "total_duration": data.get("total_duration"), + "load_duration": data.get("load_duration"), + } + return ModelLlmUsage( tokens_input=prompt_eval_count, tokens_output=0, + usage_source=( + ContractEnumUsageSource.API + if has_usage + else ContractEnumUsageSource.MISSING + ), + raw_provider_usage=raw_usage_data, ) diff --git a/src/omnibase_infra/nodes/node_llm_embedding_effect/handlers/handler_embedding_openai_compatible.py b/src/omnibase_infra/nodes/node_llm_embedding_effect/handlers/handler_embedding_openai_compatible.py index 6d3e5c7107..47313284b2 100644 --- a/src/omnibase_infra/nodes/node_llm_embedding_effect/handlers/handler_embedding_openai_compatible.py +++ b/src/omnibase_infra/nodes/node_llm_embedding_effect/handlers/handler_embedding_openai_compatible.py @@ -40,6 +40,7 @@ from omnibase_infra.nodes.node_llm_embedding_effect.models.model_llm_embedding_response import ( ModelLlmEmbeddingResponse, ) +from omnibase_spi.contracts.measurement import ContractEnumUsageSource logger = logging.getLogger(__name__) @@ -146,6 +147,17 @@ async def execute( context=ctx, ) from exc usage = _parse_openai_usage(data) + if not embeddings: + ctx = ModelInfraErrorContext.with_correlation( + correlation_id=request.correlation_id, + transport_type=EnumInfraTransportType.HTTP, + operation="parse_openai_embeddings", + target_name=self._llm_target_name, + ) + raise InfraProtocolError( + "OpenAI embedding response returned no embeddings", + context=ctx, + ) dimensions = len(embeddings[0].vector) return ModelLlmEmbeddingResponse( @@ -205,23 +217,50 @@ def _parse_openai_embeddings(data: dict[str, JsonType]) -> list[ModelEmbedding]: def _parse_openai_usage(data: dict[str, JsonType]) -> ModelLlmUsage: """Parse usage metadata from an OpenAI-compatible response. + When the provider returns a valid usage dict, ``usage_source`` is set + to ``API`` and the raw dict is preserved. When absent, ``MISSING``. + Args: data: Parsed JSON response body. Returns: - ModelLlmUsage with token counts. Zeros if usage block is absent. + ModelLlmUsage with token counts and provenance. Zeros if usage + block is absent. """ usage_raw = data.get("usage") if not isinstance(usage_raw, dict): - return ModelLlmUsage() + return ModelLlmUsage( + usage_source=ContractEnumUsageSource.MISSING, + ) prompt_tokens = usage_raw.get("prompt_tokens", 0) if not isinstance(prompt_tokens, int): prompt_tokens = 0 + total_tokens = usage_raw.get("total_tokens", 0) + if not isinstance(total_tokens, int): + total_tokens = 0 + + # Fallback: for embeddings, if prompt_tokens is 0 but total_tokens is + # valid, use total_tokens. Embedding endpoints typically report only + # total_tokens (== prompt_tokens); dropping it would under-report usage. + if prompt_tokens == 0 and total_tokens > 0: + prompt_tokens = total_tokens + + # Only mark as API-reported when at least one token counter is positive. + # A usage dict with all-zero values is semantically equivalent to missing + # usage data (matches handler_embedding_ollama pattern). + has_usage = prompt_tokens > 0 or total_tokens > 0 + return ModelLlmUsage( tokens_input=prompt_tokens, tokens_output=0, + usage_source=( + ContractEnumUsageSource.API + if has_usage + else ContractEnumUsageSource.MISSING + ), + raw_provider_usage=dict(usage_raw), ) diff --git a/src/omnibase_infra/nodes/node_llm_inference_effect/handlers/handler_llm_ollama.py b/src/omnibase_infra/nodes/node_llm_inference_effect/handlers/handler_llm_ollama.py index 94ad560e3d..794bcbef1c 100644 --- a/src/omnibase_infra/nodes/node_llm_inference_effect/handlers/handler_llm_ollama.py +++ b/src/omnibase_infra/nodes/node_llm_inference_effect/handlers/handler_llm_ollama.py @@ -62,6 +62,7 @@ ModelLlmToolDefinition, ModelLlmUsage, ) +from omnibase_spi.contracts.measurement import ContractEnumUsageSource logger = logging.getLogger(__name__) @@ -412,13 +413,31 @@ async def handle( "Non-numeric usage value for eval_count, defaulting to 0: type=%s", type(tokens_output).__name__, ) + resolved_input = ( + round(tokens_input) if isinstance(tokens_input, (int, float)) else 0 + ) + resolved_output = ( + round(tokens_output) if isinstance(tokens_output, (int, float)) else 0 + ) + # Determine provenance: treat as API-reported only when at least + # one resolved counter is positive. + has_usage = resolved_input > 0 or resolved_output > 0 + raw_usage_data: dict[str, object] = { + "prompt_eval_count": raw_response.get("prompt_eval_count"), + "eval_count": raw_response.get("eval_count"), + "total_duration": raw_response.get("total_duration"), + "load_duration": raw_response.get("load_duration"), + "eval_duration": raw_response.get("eval_duration"), + } usage = ModelLlmUsage( - tokens_input=round(tokens_input) - if isinstance(tokens_input, (int, float)) - else 0, - tokens_output=round(tokens_output) - if isinstance(tokens_output, (int, float)) - else 0, + tokens_input=resolved_input, + tokens_output=resolved_output, + usage_source=( + ContractEnumUsageSource.API + if has_usage + else ContractEnumUsageSource.MISSING + ), + raw_provider_usage=raw_usage_data, ) # Enforce text XOR tool_calls BEFORE constructing response diff --git a/src/omnibase_infra/nodes/node_llm_inference_effect/handlers/handler_llm_openai_compatible.py b/src/omnibase_infra/nodes/node_llm_inference_effect/handlers/handler_llm_openai_compatible.py index bece850e68..8691c12343 100644 --- a/src/omnibase_infra/nodes/node_llm_inference_effect/handlers/handler_llm_openai_compatible.py +++ b/src/omnibase_infra/nodes/node_llm_inference_effect/handlers/handler_llm_openai_compatible.py @@ -72,7 +72,12 @@ import httpx from omnibase_core.types import JsonType -from omnibase_infra.enums import EnumLlmFinishReason, EnumLlmOperationType +from omnibase_infra.enums import ( + EnumHandlerType, + EnumHandlerTypeCategory, + EnumLlmFinishReason, + EnumLlmOperationType, +) from omnibase_infra.mixins.mixin_llm_http_transport import MixinLlmHttpTransport from omnibase_infra.models.model_backend_result import ModelBackendResult from omnibase_infra.nodes.effects.models.model_llm_function_call import ( @@ -95,6 +100,7 @@ from omnibase_infra.nodes.node_llm_inference_effect.services.service_llm_usage_normalizer import ( normalize_llm_usage, ) +from omnibase_spi.contracts.measurement import ContractEnumUsageSource from omnibase_spi.contracts.measurement.contract_llm_call_metrics import ( ContractLlmCallMetrics, ) @@ -153,9 +159,10 @@ class HandlerLlmOpenaiCompatible: This handler operates at the infrastructure layer with a typed request model (``ModelLlmInferenceRequest``) and returns a typed response model (``ModelLlmInferenceResponse``), bypassing - envelope-based dispatch entirely. This is a documented deviation - from the standard ONEX handler pattern because LLM inference - calls are direct infrastructure effects, not routed events. + envelope-based dispatch entirely. LLM inference calls are direct + infrastructure effects, not routed events. The ``handler_type`` + and ``handler_category`` properties are still provided for + introspection and classification consistency. Auth Strategy: When a request includes ``api_key``, the handler creates a @@ -204,6 +211,27 @@ def __init__( self._transport = transport self.last_call_metrics: ContractLlmCallMetrics | None = None + @property + def handler_type(self) -> EnumHandlerType: + """Architectural role classification. + + Returns: + ``EnumHandlerType.INFRA_HANDLER`` indicating this handler + provides infrastructure-level LLM transport, not domain + business logic. + """ + return EnumHandlerType.INFRA_HANDLER + + @property + def handler_category(self) -> EnumHandlerTypeCategory: + """Behavioral classification. + + Returns: + ``EnumHandlerTypeCategory.EFFECT`` indicating this handler + performs external I/O (HTTP calls to OpenAI-compatible APIs). + """ + return EnumHandlerTypeCategory.EFFECT + async def handle( self, request: ModelLlmInferenceRequest, @@ -233,6 +261,10 @@ async def handle( InfraUnavailableError: On 5xx or circuit breaker open. ValueError: If operation_type has no known URL path. """ + # Reset metrics from any previous call so that a failure in + # _build_usage_metrics does not leave stale metrics visible. + self.last_call_metrics = None + if correlation_id is None: correlation_id = uuid4() @@ -923,23 +955,53 @@ def _parse_usage(raw_usage: JsonType) -> ModelLlmUsage: Non-numeric string values (e.g. ``"abc"``) are treated as zero (or ``None`` for ``tokens_total``) rather than raising ``ValueError``. + When the provider returns a valid usage dict, ``usage_source`` is set + to ``API`` and the raw dict is preserved in ``raw_provider_usage``. + When usage data is absent or malformed, ``usage_source`` is ``MISSING``. + Args: raw_usage: The ``usage`` field from the response JSON, or None. Returns: - ModelLlmUsage with parsed or default token counts. + ModelLlmUsage with parsed or default token counts and provenance. """ if not isinstance(raw_usage, dict): - return ModelLlmUsage() + return ModelLlmUsage( + usage_source=ContractEnumUsageSource.MISSING, + ) - tokens_input = raw_usage.get("prompt_tokens", 0) - tokens_output = raw_usage.get("completion_tokens", 0) - tokens_total = raw_usage.get("total_tokens") + tokens_input = _safe_int(raw_usage.get("prompt_tokens", 0), 0) + tokens_output = _safe_int(raw_usage.get("completion_tokens", 0), 0) + tokens_total = _safe_int_or_none(raw_usage.get("total_tokens")) + + # If the provider total doesn't match the sum, let ModelLlmUsage + # auto-compute it. Some providers include cached/reasoning tokens + # in total_tokens which makes it exceed prompt + completion. + # The raw provider data is preserved in raw_provider_usage for auditing. + expected = tokens_input + tokens_output + if tokens_total is not None and tokens_total != expected: + tokens_total = None + + # Only mark as API-reported when at least one token counter is positive. + # A usage dict with all-zero values (e.g. some providers return + # {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}) + # is semantically equivalent to missing usage data. + has_usage = ( + tokens_input > 0 + or tokens_output > 0 + or (tokens_total is not None and tokens_total > 0) + ) return ModelLlmUsage( - tokens_input=_safe_int(tokens_input, 0), - tokens_output=_safe_int(tokens_output, 0), - tokens_total=_safe_int_or_none(tokens_total), + tokens_input=tokens_input, + tokens_output=tokens_output, + tokens_total=tokens_total, + usage_source=( + ContractEnumUsageSource.API + if has_usage + else ContractEnumUsageSource.MISSING + ), + raw_provider_usage=dict(raw_usage), ) diff --git a/src/omnibase_infra/topics/__init__.py b/src/omnibase_infra/topics/__init__.py index 01983d4640..e543b3fb8f 100644 --- a/src/omnibase_infra/topics/__init__.py +++ b/src/omnibase_infra/topics/__init__.py @@ -28,6 +28,7 @@ SUFFIX_FSM_STATE_TRANSITIONS, SUFFIX_INTELLIGENCE_CLAUDE_HOOK_EVENT, SUFFIX_INTELLIGENCE_INTENT_CLASSIFIED, + SUFFIX_INTELLIGENCE_LLM_CALL_COMPLETED, SUFFIX_INTELLIGENCE_PATTERN_DISCOVERED, SUFFIX_INTELLIGENCE_PATTERN_LEARNED, SUFFIX_INTELLIGENCE_PATTERN_LIFECYCLE_TRANSITION, @@ -64,6 +65,7 @@ "SUFFIX_INTELLIGENCE_SESSION_OUTCOME", "SUFFIX_INTELLIGENCE_PATTERN_LIFECYCLE_TRANSITION", "SUFFIX_INTELLIGENCE_INTENT_CLASSIFIED", + "SUFFIX_INTELLIGENCE_LLM_CALL_COMPLETED", "SUFFIX_INTELLIGENCE_PATTERN_LEARNED", "SUFFIX_INTELLIGENCE_PATTERN_STORED", "SUFFIX_INTELLIGENCE_PATTERN_PROMOTED", diff --git a/src/omnibase_infra/topics/platform_topic_suffixes.py b/src/omnibase_infra/topics/platform_topic_suffixes.py index 9e5faa90ae..4ec89e4b1b 100644 --- a/src/omnibase_infra/topics/platform_topic_suffixes.py +++ b/src/omnibase_infra/topics/platform_topic_suffixes.py @@ -187,6 +187,15 @@ ) """Topic for pattern lifecycle transition completion events.""" +SUFFIX_INTELLIGENCE_LLM_CALL_COMPLETED: str = ( + "onex.evt.omniintelligence.llm-call-completed.v1" +) +"""Topic for LLM call completed metrics events. + +Published by LLM inference handlers after each call. Contains per-call +token counts, cost, and latency for the cost aggregation pipeline. +""" + SUFFIX_INTELLIGENCE_PATTERN_DISCOVERED: str = "onex.evt.pattern.discovered.v1" """Topic for generic pattern discovery events.""" @@ -251,6 +260,7 @@ def _snapshot_kafka_config() -> dict[str, str]: suffix=SUFFIX_INTELLIGENCE_PATTERN_LIFECYCLE_TRANSITIONED, partitions=3 ), ModelTopicSpec(suffix=SUFFIX_INTELLIGENCE_PATTERN_DISCOVERED, partitions=3), + ModelTopicSpec(suffix=SUFFIX_INTELLIGENCE_LLM_CALL_COMPLETED, partitions=3), ) """Intelligence domain topic specs provisioned for PluginIntelligence.""" diff --git a/tests/unit/models/effects/test_adapter_llm_usage_to_contract.py b/tests/unit/models/effects/test_adapter_llm_usage_to_contract.py new file mode 100644 index 0000000000..b54949cad7 --- /dev/null +++ b/tests/unit/models/effects/test_adapter_llm_usage_to_contract.py @@ -0,0 +1,354 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2025 OmniNode Team +"""Tests for the ModelLlmUsage -> SPI contract adapter functions. + +Tests cover: +- to_usage_raw: ModelLlmUsage -> ContractLlmUsageRaw +- to_usage_normalized: ModelLlmUsage -> ContractLlmUsageNormalized +- to_call_metrics: ModelLlmUsage -> ContractLlmCallMetrics +- Provenance mapping (API/ESTIMATED/MISSING) +- Edge cases (empty usage, zero tokens, None raw data) + +OMN-2318: Integrate SPI 0.9.0 LLM cost tracking contracts. +""" + +from __future__ import annotations + +import pytest + +from omnibase_infra.nodes.effects.models.adapter_llm_usage_to_contract import ( + to_call_metrics, + to_usage_normalized, + to_usage_raw, +) +from omnibase_infra.nodes.effects.models.model_llm_usage import ModelLlmUsage +from omnibase_spi.contracts.measurement import ( + ContractEnumUsageSource, + ContractLlmCallMetrics, + ContractLlmUsageNormalized, + ContractLlmUsageRaw, +) + +# ============================================================================ +# to_usage_raw Tests +# ============================================================================ + + +class TestToUsageRaw: + """Tests for converting ModelLlmUsage to ContractLlmUsageRaw.""" + + def test_with_raw_data(self) -> None: + """Raw provider data is preserved in the contract.""" + raw_data = {"prompt_tokens": 100, "completion_tokens": 50} + usage = ModelLlmUsage( + tokens_input=100, + tokens_output=50, + raw_provider_usage=raw_data, + ) + + result = to_usage_raw(usage, provider="openai") + + assert isinstance(result, ContractLlmUsageRaw) + assert result.provider == "openai" + assert result.raw_data == raw_data + + def test_without_raw_data(self) -> None: + """None raw_provider_usage produces empty raw_data dict.""" + usage = ModelLlmUsage() + + result = to_usage_raw(usage, provider="vllm") + + assert result.provider == "vllm" + assert result.raw_data == {} + + def test_default_provider(self) -> None: + """Default provider is empty string.""" + usage = ModelLlmUsage() + + result = to_usage_raw(usage) + + assert result.provider == "" + + def test_frozen_output(self) -> None: + """Returned ContractLlmUsageRaw is frozen.""" + usage = ModelLlmUsage(raw_provider_usage={"tokens": 10}) + result = to_usage_raw(usage) + + with pytest.raises(Exception): + result.provider = "changed" # type: ignore[misc] + + +# ============================================================================ +# to_usage_normalized Tests +# ============================================================================ + + +class TestToUsageNormalized: + """Tests for converting ModelLlmUsage to ContractLlmUsageNormalized.""" + + def test_api_source_mapping(self) -> None: + """API source maps to source=API, usage_is_estimated=False.""" + usage = ModelLlmUsage( + tokens_input=100, + tokens_output=50, + usage_source=ContractEnumUsageSource.API, + ) + + result = to_usage_normalized(usage) + + assert isinstance(result, ContractLlmUsageNormalized) + assert result.prompt_tokens == 100 + assert result.completion_tokens == 50 + assert result.total_tokens == 150 + assert result.source == ContractEnumUsageSource.API + assert result.usage_is_estimated is False + + def test_estimated_source_mapping(self) -> None: + """ESTIMATED source maps to source=ESTIMATED, usage_is_estimated=True.""" + usage = ModelLlmUsage( + tokens_input=200, + tokens_output=80, + usage_source=ContractEnumUsageSource.ESTIMATED, + ) + + result = to_usage_normalized(usage) + + assert result.prompt_tokens == 200 + assert result.completion_tokens == 80 + assert result.total_tokens == 280 + assert result.source == ContractEnumUsageSource.ESTIMATED + assert result.usage_is_estimated is True + + def test_missing_source_mapping(self) -> None: + """MISSING source maps to source=MISSING, usage_is_estimated=False.""" + usage = ModelLlmUsage( + usage_source=ContractEnumUsageSource.MISSING, + ) + + result = to_usage_normalized(usage) + + assert result.prompt_tokens == 0 + assert result.completion_tokens == 0 + assert result.total_tokens == 0 + assert result.source == ContractEnumUsageSource.MISSING + assert result.usage_is_estimated is False + + def test_field_name_mapping(self) -> None: + """Infra field names (tokens_input/output) map to SPI names (prompt/completion).""" + usage = ModelLlmUsage(tokens_input=42, tokens_output=17) + + result = to_usage_normalized(usage) + + assert result.prompt_tokens == 42 + assert result.completion_tokens == 17 + assert result.total_tokens == 59 + + def test_frozen_output(self) -> None: + """Returned ContractLlmUsageNormalized is frozen.""" + usage = ModelLlmUsage(tokens_input=10, tokens_output=5) + result = to_usage_normalized(usage) + + with pytest.raises(Exception): + result.prompt_tokens = 999 # type: ignore[misc] + + +# ============================================================================ +# to_call_metrics Tests +# ============================================================================ + + +class TestToCallMetrics: + """Tests for converting ModelLlmUsage to ContractLlmCallMetrics.""" + + def test_to_call_metrics_empty_model_id_raises(self) -> None: + """to_call_metrics rejects empty model_id with ValueError.""" + usage = ModelLlmUsage(tokens_input=10, tokens_output=5) + + with pytest.raises(ValueError, match="model_id must be a non-empty string"): + to_call_metrics(usage, model_id="") + + def test_basic_conversion(self) -> None: + """Basic conversion with required fields.""" + usage = ModelLlmUsage( + tokens_input=100, + tokens_output=50, + usage_source=ContractEnumUsageSource.API, + raw_provider_usage={"prompt_tokens": 100, "completion_tokens": 50}, + ) + + result = to_call_metrics(usage, model_id="gpt-4o") + + assert isinstance(result, ContractLlmCallMetrics) + assert result.model_id == "gpt-4o" + assert result.prompt_tokens == 100 + assert result.completion_tokens == 50 + assert result.total_tokens == 150 + assert result.usage_is_estimated is False + + def test_with_optional_fields(self) -> None: + """Conversion with all optional fields populated.""" + usage = ModelLlmUsage( + tokens_input=200, + tokens_output=80, + cost_usd=0.05, + usage_source=ContractEnumUsageSource.API, + raw_provider_usage={"prompt_tokens": 200}, + ) + + result = to_call_metrics( + usage, + model_id="qwen2.5-coder-14b", + provider="vllm", + latency_ms=150.0, + timestamp_iso="2026-02-15T10:00:00Z", + reporting_source="llm-inference-effect", + ) + + assert result.model_id == "qwen2.5-coder-14b" + assert result.estimated_cost_usd == pytest.approx(0.05) + assert result.latency_ms == pytest.approx(150.0) + assert result.timestamp_iso == "2026-02-15T10:00:00Z" + assert result.reporting_source == "llm-inference-effect" + assert result.usage_raw is not None + assert result.usage_raw.provider == "vllm" + assert result.usage_normalized is not None + assert result.usage_normalized.prompt_tokens == 200 + + def test_estimated_usage(self) -> None: + """Estimated usage sets usage_is_estimated=True.""" + usage = ModelLlmUsage( + tokens_input=100, + tokens_output=50, + usage_source=ContractEnumUsageSource.ESTIMATED, + ) + + result = to_call_metrics(usage, model_id="test-model") + + assert result.usage_is_estimated is True + assert result.usage_normalized is not None + assert result.usage_normalized.usage_is_estimated is True + assert result.usage_normalized.source == ContractEnumUsageSource.ESTIMATED + + def test_missing_usage(self) -> None: + """Missing usage source produces zero tokens.""" + usage = ModelLlmUsage() + + result = to_call_metrics(usage, model_id="test-model") + + assert result.prompt_tokens == 0 + assert result.completion_tokens == 0 + assert result.total_tokens == 0 + assert result.usage_is_estimated is False + + def test_no_cost(self) -> None: + """None cost_usd passes through as None.""" + usage = ModelLlmUsage(tokens_input=10) + + result = to_call_metrics(usage, model_id="test-model") + + assert result.estimated_cost_usd is None + + def test_usage_raw_included(self) -> None: + """usage_raw sub-contract is populated from raw_provider_usage.""" + raw = {"prompt_tokens": 50, "completion_tokens": 20, "total_tokens": 70} + usage = ModelLlmUsage( + tokens_input=50, + tokens_output=20, + raw_provider_usage=raw, + ) + + result = to_call_metrics(usage, model_id="model", provider="openai") + + assert result.usage_raw is not None + assert result.usage_raw.raw_data == raw + assert result.usage_raw.provider == "openai" + + def test_usage_normalized_included(self) -> None: + """usage_normalized sub-contract is populated and consistent.""" + usage = ModelLlmUsage( + tokens_input=300, + tokens_output=100, + usage_source=ContractEnumUsageSource.API, + ) + + result = to_call_metrics(usage, model_id="model") + + assert result.usage_normalized is not None + assert result.usage_normalized.prompt_tokens == 300 + assert result.usage_normalized.completion_tokens == 100 + assert result.usage_normalized.total_tokens == 400 + assert result.usage_normalized.source == ContractEnumUsageSource.API + assert result.usage_normalized.usage_is_estimated is False + + def test_token_consistency_between_top_level_and_normalized(self) -> None: + """Top-level and normalized token counts are always consistent. + + This verifies that the ContractLlmCallMetrics validator does not + reject the output (it requires top-level == normalized). + """ + usage = ModelLlmUsage( + tokens_input=42, + tokens_output=17, + usage_source=ContractEnumUsageSource.API, + ) + + result = to_call_metrics(usage, model_id="model") + + assert result.prompt_tokens == result.usage_normalized.prompt_tokens # type: ignore[union-attr] + assert result.completion_tokens == result.usage_normalized.completion_tokens # type: ignore[union-attr] + assert result.total_tokens == result.usage_normalized.total_tokens # type: ignore[union-attr] + + def test_frozen_output(self) -> None: + """Returned ContractLlmCallMetrics is frozen.""" + usage = ModelLlmUsage(tokens_input=10, tokens_output=5) + result = to_call_metrics(usage, model_id="model") + + with pytest.raises(Exception): + result.model_id = "changed" # type: ignore[misc] + + def test_default_string_fields(self) -> None: + """Default optional string fields are empty strings.""" + usage = ModelLlmUsage(tokens_input=10) + result = to_call_metrics(usage, model_id="model") + + assert result.timestamp_iso == "" + assert result.reporting_source == "" + + +# ============================================================================ +# Provenance Round-Trip +# ============================================================================ + + +class TestProvenanceRoundTrip: + """Verify provenance is correctly threaded through the full conversion + pipeline: ModelLlmUsage -> ContractLlmCallMetrics.""" + + @pytest.mark.parametrize( + ("source", "expected_estimated"), + [ + (ContractEnumUsageSource.API, False), + (ContractEnumUsageSource.ESTIMATED, True), + (ContractEnumUsageSource.MISSING, False), + ], + ) + def test_provenance_propagation( + self, + source: ContractEnumUsageSource, + expected_estimated: bool, + ) -> None: + """Provenance source is consistently mapped through all layers.""" + usage = ModelLlmUsage( + tokens_input=50, + tokens_output=25, + usage_source=source, + ) + + metrics = to_call_metrics(usage, model_id="test") + + # Top-level flag + assert metrics.usage_is_estimated is expected_estimated + # Normalized sub-contract + assert metrics.usage_normalized is not None + assert metrics.usage_normalized.source == source + assert metrics.usage_normalized.usage_is_estimated is expected_estimated diff --git a/tests/unit/models/effects/test_effects_models_exports.py b/tests/unit/models/effects/test_effects_models_exports.py index 4a59d42a99..3e9bdb1fbd 100644 --- a/tests/unit/models/effects/test_effects_models_exports.py +++ b/tests/unit/models/effects/test_effects_models_exports.py @@ -5,14 +5,18 @@ Tests cover: - Individual importability of each new LLM model class -- __all__ completeness for the six new model names -- All __all__ entries resolve to classes (types) +- Individual importability of adapter functions (OMN-2318) +- __all__ completeness for model names and adapter functions +- All __all__ entries resolve to classes or callables OMN-2103: Phase 3 shared LLM models - export verification +OMN-2318: Integrate SPI 0.9.0 LLM cost tracking contracts """ from __future__ import annotations +import types + import pytest # ============================================================================ @@ -60,13 +64,35 @@ def test_model_llm_usage_importable(self) -> None: assert ModelLlmUsage is not None +class TestAdapterImports: + """Verify adapter functions are importable from effects models package.""" + + def test_to_call_metrics_importable(self) -> None: + """to_call_metrics is importable from effects models package.""" + from omnibase_infra.nodes.effects.models import to_call_metrics + + assert callable(to_call_metrics) + + def test_to_usage_normalized_importable(self) -> None: + """to_usage_normalized is importable from effects models package.""" + from omnibase_infra.nodes.effects.models import to_usage_normalized + + assert callable(to_usage_normalized) + + def test_to_usage_raw_importable(self) -> None: + """to_usage_raw is importable from effects models package.""" + from omnibase_infra.nodes.effects.models import to_usage_raw + + assert callable(to_usage_raw) + + # ============================================================================ # __all__ Completeness Tests # ============================================================================ class TestAllExports: - """Verify __all__ contains all new model names and all entries are classes.""" + """Verify __all__ contains all expected names and entries resolve correctly.""" _EXPECTED_NEW_MODELS: tuple[str, ...] = ( "ModelLlmFunctionCall", @@ -77,6 +103,12 @@ class TestAllExports: "ModelLlmUsage", ) + _EXPECTED_ADAPTER_FUNCTIONS: tuple[str, ...] = ( + "to_call_metrics", + "to_usage_normalized", + "to_usage_raw", + ) + def test_all_contains_all_new_models(self) -> None: """__all__ includes every new LLM model class name.""" import omnibase_infra.nodes.effects.models as effects_models @@ -86,12 +118,22 @@ def test_all_contains_all_new_models(self) -> None: for name in self._EXPECTED_NEW_MODELS: assert name in all_exports, f"{name} missing from __all__" - def test_all_exports_are_classes(self) -> None: - """Every name in __all__ resolves to a class (type).""" + def test_all_contains_adapter_functions(self) -> None: + """__all__ includes adapter function names (OMN-2318).""" + import omnibase_infra.nodes.effects.models as effects_models + + all_exports = set(effects_models.__all__) + + for name in self._EXPECTED_ADAPTER_FUNCTIONS: + assert name in all_exports, f"{name} missing from __all__" + + def test_all_exports_are_classes_or_callables(self) -> None: + """Every name in __all__ resolves to a class or callable.""" import omnibase_infra.nodes.effects.models as effects_models for name in effects_models.__all__: obj = getattr(effects_models, name) - assert isinstance(obj, type), ( - f"__all__ entry {name!r} is {type(obj).__name__}, expected a class" + assert isinstance(obj, (type, types.FunctionType)), ( + f"__all__ entry {name!r} is {type(obj).__name__}, " + f"expected a class or function" ) diff --git a/tests/unit/models/effects/test_model_llm_usage_provenance.py b/tests/unit/models/effects/test_model_llm_usage_provenance.py new file mode 100644 index 0000000000..e69153107e --- /dev/null +++ b/tests/unit/models/effects/test_model_llm_usage_provenance.py @@ -0,0 +1,309 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2025 OmniNode Team +"""Tests for ModelLlmUsage provenance tracking fields. + +Tests the ``usage_source`` and ``raw_provider_usage`` fields added in +OMN-2318 to bridge ModelLlmUsage with SPI LLM cost tracking contracts. + +Tests cover: +- Default values for new fields (backwards compatibility) +- Explicit usage_source construction for all enum members +- raw_provider_usage preservation and serialization +- Combination of new fields with existing token fields +- Frozen model enforcement on new fields +- Serialization round-trips with new fields +""" + +from __future__ import annotations + +import json + +import pytest +from pydantic import ValidationError + +from omnibase_infra.nodes.effects.models.model_llm_usage import ModelLlmUsage +from omnibase_spi.contracts.measurement import ContractEnumUsageSource + +# ============================================================================ +# Backwards Compatibility +# ============================================================================ + + +class TestBackwardsCompatibility: + """Verify that existing callers constructing ModelLlmUsage without + new fields continue to work identically.""" + + def test_default_construction_unchanged(self) -> None: + """Default ModelLlmUsage() has MISSING source and no raw data.""" + usage = ModelLlmUsage() + + assert usage.usage_source == ContractEnumUsageSource.MISSING + assert usage.raw_provider_usage is None + assert usage.tokens_input == 0 + assert usage.tokens_output == 0 + assert usage.tokens_total == 0 + assert usage.cost_usd is None + + def test_token_only_construction_unchanged(self) -> None: + """Construction with only token fields defaults new fields.""" + usage = ModelLlmUsage(tokens_input=100, tokens_output=50) + + assert usage.tokens_total == 150 + assert usage.usage_source == ContractEnumUsageSource.MISSING + assert usage.raw_provider_usage is None + + +# ============================================================================ +# Usage Source Field +# ============================================================================ + + +class TestUsageSource: + """Tests for the usage_source provenance field.""" + + def test_source_api(self) -> None: + """usage_source=API is accepted.""" + usage = ModelLlmUsage( + tokens_input=100, + tokens_output=50, + usage_source=ContractEnumUsageSource.API, + ) + + assert usage.usage_source == ContractEnumUsageSource.API + + def test_source_estimated(self) -> None: + """usage_source=ESTIMATED is accepted.""" + usage = ModelLlmUsage( + tokens_input=100, + tokens_output=50, + usage_source=ContractEnumUsageSource.ESTIMATED, + ) + + assert usage.usage_source == ContractEnumUsageSource.ESTIMATED + + def test_source_missing(self) -> None: + """usage_source=MISSING is accepted (and is the default).""" + usage = ModelLlmUsage( + usage_source=ContractEnumUsageSource.MISSING, + ) + + assert usage.usage_source == ContractEnumUsageSource.MISSING + + def test_source_from_string(self) -> None: + """usage_source accepts string values matching enum.""" + usage = ModelLlmUsage( + tokens_input=10, + usage_source="api", # type: ignore[arg-type] + ) + + assert usage.usage_source == ContractEnumUsageSource.API + + def test_source_invalid_rejected(self) -> None: + """Invalid usage_source string is rejected.""" + with pytest.raises(ValidationError): + ModelLlmUsage(usage_source="invalid_source") # type: ignore[arg-type] + + +# ============================================================================ +# Raw Provider Usage Field +# ============================================================================ + + +class TestRawProviderUsage: + """Tests for the raw_provider_usage audit trail field.""" + + def test_none_by_default(self) -> None: + """raw_provider_usage defaults to None.""" + usage = ModelLlmUsage() + + assert usage.raw_provider_usage is None + + def test_preserves_openai_format(self) -> None: + """OpenAI-style usage dict is preserved verbatim.""" + raw = {"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150} + usage = ModelLlmUsage( + tokens_input=100, + tokens_output=50, + raw_provider_usage=raw, + ) + + assert usage.raw_provider_usage == raw + + def test_preserves_ollama_format(self) -> None: + """Ollama-style usage dict with different field names is preserved.""" + raw = { + "prompt_eval_count": 100, + "eval_count": 50, + "total_duration": 123456789, + "load_duration": 1000, + } + usage = ModelLlmUsage( + tokens_input=100, + tokens_output=50, + raw_provider_usage=raw, + ) + + assert usage.raw_provider_usage == raw + assert usage.raw_provider_usage["total_duration"] == 123456789 + + def test_empty_dict_accepted(self) -> None: + """Empty dict is valid for raw_provider_usage.""" + usage = ModelLlmUsage(raw_provider_usage={}) + + assert usage.raw_provider_usage == {} + + def test_nested_dicts_preserved(self) -> None: + """Nested structures in raw usage data are preserved.""" + raw = { + "prompt_tokens": 100, + "completion_tokens_details": { + "reasoning_tokens": 30, + "accepted_prediction_tokens": 20, + }, + } + usage = ModelLlmUsage( + tokens_input=100, + tokens_output=50, + raw_provider_usage=raw, + ) + + assert usage.raw_provider_usage is not None + assert ( + usage.raw_provider_usage["completion_tokens_details"]["reasoning_tokens"] + == 30 + ) # type: ignore[index] + + +# ============================================================================ +# Immutability on New Fields +# ============================================================================ + + +class TestImmutabilityNewFields: + """Verify frozen=True applies to new fields.""" + + def test_usage_source_immutable(self) -> None: + """Cannot reassign usage_source on frozen model.""" + usage = ModelLlmUsage(usage_source=ContractEnumUsageSource.API) + + with pytest.raises(ValidationError): + usage.usage_source = ContractEnumUsageSource.MISSING # type: ignore[misc] + + def test_raw_provider_usage_immutable(self) -> None: + """Cannot reassign raw_provider_usage on frozen model.""" + usage = ModelLlmUsage(raw_provider_usage={"prompt_tokens": 100}) + + with pytest.raises(ValidationError): + usage.raw_provider_usage = {} # type: ignore[misc] + + +# ============================================================================ +# Serialization with New Fields +# ============================================================================ + + +class TestSerializationNewFields: + """Tests for serialization round-trips including new fields.""" + + def test_model_dump_includes_new_fields(self) -> None: + """model_dump includes usage_source and raw_provider_usage.""" + usage = ModelLlmUsage( + tokens_input=100, + tokens_output=50, + usage_source=ContractEnumUsageSource.API, + raw_provider_usage={"prompt_tokens": 100}, + ) + + data = usage.model_dump() + + assert data["usage_source"] == "api" + assert data["raw_provider_usage"] == {"prompt_tokens": 100} + + def test_model_dump_json_roundtrip(self) -> None: + """JSON round-trip preserves new fields.""" + usage = ModelLlmUsage( + tokens_input=100, + tokens_output=50, + usage_source=ContractEnumUsageSource.API, + raw_provider_usage={"prompt_tokens": 100, "completion_tokens": 50}, + ) + + json_str = usage.model_dump_json() + parsed = json.loads(json_str) + + assert parsed["usage_source"] == "api" + assert parsed["raw_provider_usage"]["prompt_tokens"] == 100 + assert parsed["raw_provider_usage"]["completion_tokens"] == 50 + + def test_model_validate_roundtrip(self) -> None: + """model_dump -> model_validate round-trip preserves all fields.""" + original = ModelLlmUsage( + tokens_input=200, + tokens_output=80, + usage_source=ContractEnumUsageSource.ESTIMATED, + raw_provider_usage={"estimated_via": "tiktoken"}, + ) + + data = original.model_dump() + restored = ModelLlmUsage.model_validate(data) + + assert restored == original + assert restored.usage_source == ContractEnumUsageSource.ESTIMATED + assert restored.raw_provider_usage == {"estimated_via": "tiktoken"} + + def test_model_dump_none_raw_usage(self) -> None: + """Serialization handles None raw_provider_usage.""" + usage = ModelLlmUsage() + + data = usage.model_dump() + + assert data["raw_provider_usage"] is None + assert data["usage_source"] == "missing" + + +# ============================================================================ +# Combined Field Interactions +# ============================================================================ + + +class TestCombinedFieldInteractions: + """Tests for interactions between token fields and new provenance fields.""" + + def test_api_source_with_full_tokens(self) -> None: + """API source with complete token data and raw usage.""" + usage = ModelLlmUsage( + tokens_input=500, + tokens_output=200, + cost_usd=0.05, + usage_source=ContractEnumUsageSource.API, + raw_provider_usage={ + "prompt_tokens": 500, + "completion_tokens": 200, + "total_tokens": 700, + }, + ) + + assert usage.tokens_total == 700 + assert usage.cost_usd == pytest.approx(0.05) + assert usage.usage_source == ContractEnumUsageSource.API + + def test_estimated_source_with_zero_cost(self) -> None: + """Estimated source with zero tokens and no cost.""" + usage = ModelLlmUsage( + tokens_input=0, + tokens_output=0, + usage_source=ContractEnumUsageSource.ESTIMATED, + ) + + assert usage.tokens_total == 0 + assert usage.usage_source == ContractEnumUsageSource.ESTIMATED + + def test_missing_source_with_zero_tokens(self) -> None: + """Missing source results in all-zero tokens.""" + usage = ModelLlmUsage( + usage_source=ContractEnumUsageSource.MISSING, + ) + + assert usage.tokens_input == 0 + assert usage.tokens_output == 0 + assert usage.tokens_total == 0 diff --git a/tests/unit/nodes/node_llm_inference_effect/handlers/test_handler_llm_openai_compatible_class.py b/tests/unit/nodes/node_llm_inference_effect/handlers/test_handler_llm_openai_compatible_class.py index d54ce62553..311c5809de 100644 --- a/tests/unit/nodes/node_llm_inference_effect/handlers/test_handler_llm_openai_compatible_class.py +++ b/tests/unit/nodes/node_llm_inference_effect/handlers/test_handler_llm_openai_compatible_class.py @@ -25,6 +25,8 @@ import pytest from omnibase_infra.enums import ( + EnumHandlerType, + EnumHandlerTypeCategory, EnumLlmFinishReason, EnumLlmOperationType, ) @@ -144,18 +146,17 @@ def test_stores_transport(self) -> None: assert handler._transport is transport - def test_no_handler_type_property(self) -> None: - """Handler does NOT implement ProtocolHandler/ProtocolMessageHandler. + def test_handler_classification_properties(self) -> None: + """Handler exposes handler_type and handler_category for classification. - Unlike HandlerLlmOllama, this handler operates at the infrastructure - layer with typed request/response models and does not expose - handler_type or handler_category properties. + Like HandlerLlmOllama, this handler provides classification properties + to support the handler plugin loader and dispatch infrastructure. """ transport = _make_transport() handler = HandlerLlmOpenaiCompatible(transport) - assert not hasattr(handler, "handler_type") - assert not hasattr(handler, "handler_category") + assert handler.handler_type == EnumHandlerType.INFRA_HANDLER + assert handler.handler_category == EnumHandlerTypeCategory.EFFECT # --------------------------------------------------------------------------- diff --git a/tests/unit/nodes/node_llm_inference_effect/handlers/test_handler_llm_openai_compatible_metrics.py b/tests/unit/nodes/node_llm_inference_effect/handlers/test_handler_llm_openai_compatible_metrics.py index ae967ce06f..30a48dd2a9 100644 --- a/tests/unit/nodes/node_llm_inference_effect/handlers/test_handler_llm_openai_compatible_metrics.py +++ b/tests/unit/nodes/node_llm_inference_effect/handlers/test_handler_llm_openai_compatible_metrics.py @@ -27,6 +27,7 @@ from omnibase_infra.nodes.node_llm_inference_effect.handlers.handler_llm_openai_compatible import ( HandlerLlmOpenaiCompatible, _compute_input_hash, + _parse_usage, ) from omnibase_infra.nodes.node_llm_inference_effect.models.model_llm_inference_request import ( ModelLlmInferenceRequest, @@ -463,3 +464,54 @@ async def test_response_content_unchanged(self) -> None: assert resp.generated_text == "Hello!" assert resp.usage.tokens_input == 100 assert resp.usage.tokens_output == 50 + + +# --------------------------------------------------------------------------- +# _parse_usage Inconsistent Total Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestParseUsageInconsistentTotal: + """Tests for _parse_usage when provider total_tokens != prompt + completion. + + Some LLM providers (e.g. those counting cached/reasoning/system tokens) + return a total_tokens value that exceeds prompt_tokens + completion_tokens. + The handler must not crash; instead it falls back to auto-computation. + """ + + def test_inconsistent_total_auto_computes(self) -> None: + """When total_tokens != prompt + completion, auto-compute the total.""" + raw: dict[str, Any] = { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 20, # Inconsistent: 20 != 10 + 5 + } + usage = _parse_usage(raw) + + assert usage.tokens_input == 10 + assert usage.tokens_output == 5 + assert usage.tokens_total == 15 # Auto-computed as 10 + 5 + + def test_inconsistent_total_preserves_raw_provider_usage(self) -> None: + """Raw provider data is preserved even when total is corrected.""" + raw: dict[str, Any] = { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 20, + } + usage = _parse_usage(raw) + + assert usage.raw_provider_usage is not None + assert usage.raw_provider_usage["total_tokens"] == 20 + + def test_consistent_total_passes_through(self) -> None: + """When total_tokens == prompt + completion, it passes through.""" + raw: dict[str, Any] = { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + } + usage = _parse_usage(raw) + + assert usage.tokens_total == 15 diff --git a/tests/unit/topics/test_platform_topic_suffixes.py b/tests/unit/topics/test_platform_topic_suffixes.py index 334e9e0e90..86c7a5438f 100644 --- a/tests/unit/topics/test_platform_topic_suffixes.py +++ b/tests/unit/topics/test_platform_topic_suffixes.py @@ -185,8 +185,8 @@ def test_intelligence_suffixes_use_correct_producers(self) -> None: ) def test_intelligence_topic_count(self) -> None: - """Intelligence spec registry should have 9 topics.""" - assert len(ALL_INTELLIGENCE_TOPIC_SPECS) == 9 + """Intelligence spec registry should have 10 topics.""" + assert len(ALL_INTELLIGENCE_TOPIC_SPECS) == 10 def test_intelligence_command_topics(self) -> None: """Intelligence command topics should be defined."""