diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 584c4bb235..87a9790d76 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -601,7 +601,7 @@ jobs: run: | uv run pytest tests/ \ --ignore=tests/integration/docker \ - -m "not slow and not chaos" \ + -m "not slow and not chaos and not kafka" \ --splits 10 \ --group ${{ matrix.split }} \ -n auto \ diff --git a/docker/migrations/forward/054_create_consumer_health_triage.sql b/docker/migrations/forward/054_create_consumer_health_triage.sql new file mode 100644 index 0000000000..3209536d27 --- /dev/null +++ b/docker/migrations/forward/054_create_consumer_health_triage.sql @@ -0,0 +1,111 @@ +-- ============================================================================= +-- MIGRATION: Create consumer_health_triage and consumer_restart_state tables +-- ============================================================================= +-- Ticket: OMN-5512 (Task 7: Create migration for consumer_health_triage table) +-- Epic: OMN-5529 (Runtime Health Event Pipeline) +-- Version: 1.0.0 +-- +-- PURPOSE: +-- Creates the consumer health triage tables for Layer 1 of the runtime +-- health event pipeline. consumer_health_triage tracks incidents by +-- fingerprint for graduated response. consumer_restart_state tracks +-- restart rate limiting by consumer identity. +-- +-- IDEMPOTENCY: +-- - CREATE TABLE IF NOT EXISTS is safe to re-run. +-- - CREATE INDEX IF NOT EXISTS is safe to re-run. +-- +-- ROLLBACK: +-- See rollback/rollback_054_create_consumer_health_triage.sql +-- ============================================================================= + +-- Incident tracking by fingerprint for graduated response +CREATE TABLE IF NOT EXISTS public.consumer_health_triage ( + id BIGSERIAL PRIMARY KEY, + fingerprint TEXT NOT NULL, + consumer_id TEXT NOT NULL, + consumer_group TEXT NOT NULL, + topic TEXT NOT NULL, + event_type TEXT NOT NULL, + severity TEXT NOT NULL, + incident_state TEXT NOT NULL DEFAULT 'open', + occurrence_count INT NOT NULL DEFAULT 1, + first_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + escalated_at TIMESTAMPTZ, + resolved_at TIMESTAMPTZ, + linear_ticket_id TEXT, + error_message TEXT DEFAULT '', + service_name TEXT DEFAULT '', + hostname TEXT DEFAULT '', + correlation_id TEXT +); + +-- Restart rate limiting by consumer identity +CREATE TABLE IF NOT EXISTS public.consumer_restart_state ( + id BIGSERIAL PRIMARY KEY, + consumer_id TEXT NOT NULL, + consumer_group TEXT NOT NULL, + topic TEXT NOT NULL, + last_restart_at TIMESTAMPTZ, + restart_count_30min INT NOT NULL DEFAULT 0, + restart_window_start TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_restart_success BOOLEAN, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (consumer_id, consumer_group, topic) +); + +-- ============================================================================= +-- INDEXES +-- ============================================================================= + +-- Triage table indexes +CREATE INDEX IF NOT EXISTS idx_consumer_health_triage_fingerprint + ON consumer_health_triage (fingerprint); + +CREATE INDEX IF NOT EXISTS idx_consumer_health_triage_consumer + ON consumer_health_triage (consumer_id, consumer_group); + +CREATE INDEX IF NOT EXISTS idx_consumer_health_triage_state + ON consumer_health_triage (incident_state); + +CREATE INDEX IF NOT EXISTS idx_consumer_health_triage_last_seen + ON consumer_health_triage (last_seen_at); + +CREATE INDEX IF NOT EXISTS idx_consumer_health_triage_fingerprint_state + ON consumer_health_triage (fingerprint, incident_state); + +-- Restart state indexes +CREATE INDEX IF NOT EXISTS idx_consumer_restart_state_consumer + ON consumer_restart_state (consumer_id, consumer_group, topic); + +-- ============================================================================= +-- COMMENTS +-- ============================================================================= + +COMMENT ON TABLE consumer_health_triage IS + 'Incident tracking for consumer health events (OMN-5512). ' + 'Each row tracks an incident by fingerprint for graduated response: ' + 'Slack warning -> Slack repeated -> restart command -> Linear ticket.'; + +COMMENT ON COLUMN consumer_health_triage.fingerprint IS + 'Stable hash of (consumer_id, event_type, topic) for deduplication.'; + +COMMENT ON COLUMN consumer_health_triage.incident_state IS + 'Lifecycle state: open, acknowledged, restart_pending, restart_succeeded, ' + 'restart_failed, ticketed, resolved.'; + +COMMENT ON COLUMN consumer_health_triage.occurrence_count IS + 'Number of times this fingerprint has been seen in the current incident.'; + +COMMENT ON TABLE consumer_restart_state IS + 'Restart rate limiting state per consumer identity (OMN-5512). ' + 'Prevents restart storms by tracking restart frequency within a 30-min window.'; + +-- Update migration sentinel +UPDATE public.db_metadata +SET migrations_complete = TRUE, + schema_version = '054', + updated_at = NOW() +WHERE id = TRUE AND owner_service = 'omnibase_infra'; diff --git a/docker/migrations/forward/055_create_runtime_error_triage.sql b/docker/migrations/forward/055_create_runtime_error_triage.sql new file mode 100644 index 0000000000..32f536c2cc --- /dev/null +++ b/docker/migrations/forward/055_create_runtime_error_triage.sql @@ -0,0 +1,93 @@ +-- ============================================================================= +-- MIGRATION: Create runtime_error_triage table +-- ============================================================================= +-- Ticket: OMN-5514 (Task 13: Create migration for runtime_error_triage table) +-- Epic: OMN-5529 (Runtime Health Event Pipeline) +-- Version: 1.0.0 +-- +-- PURPOSE: +-- Creates the runtime error triage table for Layer 2 of the runtime +-- health event pipeline. Tracks runtime error incidents by fingerprint +-- for graduated triage response with cross-layer correlation to Layer 1 +-- consumer health incidents. +-- +-- IDEMPOTENCY: +-- - CREATE TABLE IF NOT EXISTS is safe to re-run. +-- - CREATE INDEX IF NOT EXISTS is safe to re-run. +-- +-- ROLLBACK: +-- See rollback/rollback_055_create_runtime_error_triage.sql +-- ============================================================================= + +CREATE TABLE IF NOT EXISTS public.runtime_error_triage ( + id BIGSERIAL PRIMARY KEY, + fingerprint TEXT NOT NULL, + logger_name TEXT NOT NULL, + error_category TEXT NOT NULL, + severity TEXT NOT NULL, + incident_state TEXT NOT NULL DEFAULT 'open', + occurrence_count INT NOT NULL DEFAULT 1, + message_template TEXT NOT NULL DEFAULT '', + first_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + escalated_at TIMESTAMPTZ, + resolved_at TIMESTAMPTZ, + linear_ticket_id TEXT, + -- Cross-layer correlation: links to consumer_health_triage fingerprint + correlated_consumer_fingerprint TEXT, + service_name TEXT DEFAULT '', + hostname TEXT DEFAULT '', + correlation_id TEXT +); + +-- ============================================================================= +-- INDEXES +-- ============================================================================= + +CREATE INDEX IF NOT EXISTS idx_runtime_error_triage_fingerprint + ON runtime_error_triage (fingerprint); + +CREATE INDEX IF NOT EXISTS idx_runtime_error_triage_category + ON runtime_error_triage (error_category); + +CREATE INDEX IF NOT EXISTS idx_runtime_error_triage_state + ON runtime_error_triage (incident_state); + +CREATE INDEX IF NOT EXISTS idx_runtime_error_triage_last_seen + ON runtime_error_triage (last_seen_at); + +CREATE INDEX IF NOT EXISTS idx_runtime_error_triage_fingerprint_state + ON runtime_error_triage (fingerprint, incident_state); + +CREATE INDEX IF NOT EXISTS idx_runtime_error_triage_correlation + ON runtime_error_triage (correlated_consumer_fingerprint); + +-- ============================================================================= +-- COMMENTS +-- ============================================================================= + +COMMENT ON TABLE runtime_error_triage IS + 'Incident tracking for runtime error events (OMN-5514). ' + 'Each row tracks an error incident by fingerprint for graduated triage. ' + 'Supports cross-layer correlation with consumer_health_triage via ' + 'correlated_consumer_fingerprint.'; + +COMMENT ON COLUMN runtime_error_triage.fingerprint IS + 'Stable hash of (logger_name, message_template, error_category) for deduplication.'; + +COMMENT ON COLUMN runtime_error_triage.incident_state IS + 'Lifecycle state: open, acknowledged, suppressed, ticketed, resolved.'; + +COMMENT ON COLUMN runtime_error_triage.message_template IS + 'Templatized error message with variable parts replaced by placeholders.'; + +COMMENT ON COLUMN runtime_error_triage.correlated_consumer_fingerprint IS + 'Links to consumer_health_triage.fingerprint for cross-layer correlation. ' + 'NULL if no consumer health correlation exists.'; + +-- Update migration sentinel +UPDATE public.db_metadata +SET migrations_complete = TRUE, + schema_version = '055', + updated_at = NOW() +WHERE id = TRUE AND owner_service = 'omnibase_infra'; diff --git a/docker/migrations/schema_fingerprint.sha256 b/docker/migrations/schema_fingerprint.sha256 index 90a0a68e91..d9c4f463aa 100644 --- a/docker/migrations/schema_fingerprint.sha256 +++ b/docker/migrations/schema_fingerprint.sha256 @@ -1,6 +1,6 @@ # Schema migration fingerprint for omnibase_infra (auto-generated) # Regenerate: python scripts/check_schema_fingerprint.py stamp # Verify: python scripts/check_schema_fingerprint.py verify -sha256:38841cefe359f6b19d8b39de275e9ef93b1842ef4a5a983d61ea7e2fcab11482 -generated_at: 2026-03-17T20:46:21Z -migration_file_count: 32 +sha256:8974bc651d3282eb8ebb174ce224f653917600d543b60cb6c97b829f888eb30b +generated_at: 2026-03-19T20:54:52Z +migration_file_count: 34 diff --git a/scripts/check_contract_topic_parity.py b/scripts/check_contract_topic_parity.py index 5775f39f30..3d4108293e 100644 --- a/scripts/check_contract_topic_parity.py +++ b/scripts/check_contract_topic_parity.py @@ -318,6 +318,10 @@ "onex.evt.platform.resolution-decided.v1": "pre-migration; needs contract.yaml in omnibase_infra | owner: jonah | expiry: 2026-06-01", "onex.evt.platform.service-heartbeat.v1": "new topic added in OMN-5184; contract.yaml needed once consuming node is wired | owner: jonah | expiry: 2026-06-01", "onex.snapshot.platform.registration-snapshots.v1": "non-standard kind 'snapshot'; topic validated separately via ValidateTopicSuffix skip-list | owner: jonah | expiry: 2026-06-01", + # --- consumer health pipeline topics (OMN-5529) --- + "onex.evt.omnibase-infra.consumer-health.v1": "OMN-5515; contract.yaml will be added with NodeConsumerHealthTriageEffect in OMN-5520 | owner: jonah | expiry: 2026-06-01", + "onex.cmd.omnibase-infra.consumer-restart.v1": "OMN-5515; contract.yaml will be added with NodeConsumerHealthTriageEffect in OMN-5520 | owner: jonah | expiry: 2026-06-01", + "onex.evt.omnibase-infra.runtime-error.v1": "OMN-5517; contract.yaml will be added with NodeRuntimeErrorTriageEffect in OMN-5522 | owner: jonah | expiry: 2026-06-01", } # fmt: on diff --git a/src/omnibase_infra/enums/generated/enum_omnibase_infra_topic.py b/src/omnibase_infra/enums/generated/enum_omnibase_infra_topic.py index 343b534729..ff6763f06a 100644 --- a/src/omnibase_infra/enums/generated/enum_omnibase_infra_topic.py +++ b/src/omnibase_infra/enums/generated/enum_omnibase_infra_topic.py @@ -18,12 +18,15 @@ class EnumOmnibaseInfraTopic(str, Enum): All values are raw topic strings as declared in contract.yaml. Members are sorted by (kind, event_name, version). """ + CMD_CONSUMER_RESTART_V1 = "onex.cmd.omnibase-infra.consumer-restart.v1" # onex.cmd.omnibase-infra.consumer-restart.v1 EVT_BASELINES_COMPUTED_V1 = "onex.evt.omnibase-infra.baselines-computed.v1" # onex.evt.omnibase-infra.baselines-computed.v1 EVT_CIRCUIT_BREAKER_STATE_V1 = "onex.evt.omnibase-infra.circuit-breaker-state.v1" # onex.evt.omnibase-infra.circuit-breaker-state.v1 + EVT_CONSUMER_HEALTH_V1 = "onex.evt.omnibase-infra.consumer-health.v1" # onex.evt.omnibase-infra.consumer-health.v1 EVT_DB_ERROR_V1 = "onex.evt.omnibase-infra.db-error.v1" # onex.evt.omnibase-infra.db-error.v1 EVT_EFFECTIVENESS_DATA_CHANGED_V1 = "onex.evt.omnibase-infra.effectiveness-data-changed.v1" # onex.evt.omnibase-infra.effectiveness-data-changed.v1 EVT_GMAIL_INTENT_RECEIVED_V1 = "onex.evt.omnibase-infra.gmail-intent-received.v1" # onex.evt.omnibase-infra.gmail-intent-received.v1 EVT_LLM_CALL_COMPLETED_V1 = "onex.evt.omnibase-infra.llm-call-completed.v1" # onex.evt.omnibase-infra.llm-call-completed.v1 EVT_LLM_ENDPOINT_HEALTH_V1 = "onex.evt.omnibase-infra.llm-endpoint-health.v1" # onex.evt.omnibase-infra.llm-endpoint-health.v1 + EVT_RUNTIME_ERROR_V1 = "onex.evt.omnibase-infra.runtime-error.v1" # onex.evt.omnibase-infra.runtime-error.v1 EVT_SAVINGS_ESTIMATED_V1 = "onex.evt.omnibase-infra.savings-estimated.v1" # onex.evt.omnibase-infra.savings-estimated.v1 EVT_WIRING_HEALTH_SNAPSHOT_V1 = "onex.evt.omnibase-infra.wiring-health-snapshot.v1" # onex.evt.omnibase-infra.wiring-health-snapshot.v1 diff --git a/src/omnibase_infra/event_bus/consumer_health_emitter.py b/src/omnibase_infra/event_bus/consumer_health_emitter.py new file mode 100644 index 0000000000..49efd460c6 --- /dev/null +++ b/src/omnibase_infra/event_bus/consumer_health_emitter.py @@ -0,0 +1,198 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Best-effort consumer health event emitter for Layer 1 pipeline. + +Non-blocking, fire-and-forget emitter for consumer health events. +Holds a reference to an existing AIOKafkaProducer (does NOT own the +producer lifecycle). Emission failure is tolerated and logged. + +Features: + - Local rate limiter: max 1 event per fingerprint per 10 seconds + - Self-metrics: events_emitted, events_dropped, events_rate_limited + - Feature flag gated: ENABLE_CONSUMER_HEALTH_EMITTER (default off) + +Related Tickets: + - OMN-5516: Create ConsumerHealthEmitter + - OMN-5529: Runtime Health Event Pipeline (epic) +""" + +from __future__ import annotations + +import json +import logging +import os +import time +from typing import TYPE_CHECKING + +from omnibase_infra.event_bus.topic_constants import TOPIC_CONSUMER_HEALTH +from omnibase_infra.models.health.enum_consumer_health_event_type import ( + EnumConsumerHealthEventType, +) +from omnibase_infra.models.health.enum_consumer_health_severity import ( + EnumConsumerHealthSeverity, +) +from omnibase_infra.models.health.model_consumer_health_event import ( + ModelConsumerHealthEvent, +) + +if TYPE_CHECKING: + from uuid import UUID + + from aiokafka import AIOKafkaProducer + +logger = logging.getLogger(__name__) + +# Rate limit: max 1 event per fingerprint per 10 seconds +_RATE_LIMIT_SECONDS = 10.0 + + +class ConsumerHealthEmitter: + """Best-effort, non-blocking emitter for consumer health events. + + Does NOT own the producer lifecycle -- callers must pass an already-started + AIOKafkaProducer. Emission failure is tolerated (logged, counted). + + All emission is gated by ENABLE_CONSUMER_HEALTH_EMITTER env var (default off). + + Attributes: + events_emitted: Count of successfully emitted events. + events_dropped: Count of events that failed to emit. + events_rate_limited: Count of events suppressed by rate limiter. + """ + + def __init__( + self, + producer: AIOKafkaProducer, + *, + topic: str = TOPIC_CONSUMER_HEALTH, + ) -> None: + """Initialize the emitter. + + Args: + producer: An already-started AIOKafkaProducer. + topic: Topic to emit to (defaults to TOPIC_CONSUMER_HEALTH). + """ + self._producer = producer + self._topic = topic + self._rate_limit_cache: dict[str, float] = {} + + # Self-metrics + self.events_emitted: int = 0 + self.events_dropped: int = 0 + self.events_rate_limited: int = 0 + + @staticmethod + def is_enabled() -> bool: + """Check if consumer health emission is enabled via feature flag. + + Returns: + True if ENABLE_CONSUMER_HEALTH_EMITTER is truthy. + """ + return os.environ.get( # ONEX_EXCLUDE: env + "ENABLE_CONSUMER_HEALTH_EMITTER", "" + ).strip().lower() in { + "1", + "true", + "yes", + "on", + } + + def _is_rate_limited(self, fingerprint: str) -> bool: + """Check if an event with this fingerprint is rate-limited. + + Args: + fingerprint: Event fingerprint. + + Returns: + True if the event should be suppressed. + """ + now = time.monotonic() + last_emit = self._rate_limit_cache.get(fingerprint, 0.0) + if now - last_emit < _RATE_LIMIT_SECONDS: + return True + self._rate_limit_cache[fingerprint] = now + return False + + async def emit(self, event: ModelConsumerHealthEvent) -> None: + """Emit a consumer health event (fire-and-forget). + + Rate-limited per fingerprint. Emission failure is logged but + does not raise. + + Args: + event: The consumer health event to emit. + """ + if not self.is_enabled(): + return + + if self._is_rate_limited(event.fingerprint): + self.events_rate_limited += 1 + return + + try: + payload = json.dumps( + event.model_dump(mode="json"), + ).encode("utf-8") + await self._producer.send(self._topic, value=payload) + self.events_emitted += 1 + except Exception: # noqa: BLE001 - intentional catch-all for best-effort emission + self.events_dropped += 1 + logger.debug( + "Failed to emit consumer health event (fingerprint=%s)", + event.fingerprint, + exc_info=True, + ) + + async def emit_event( + self, + *, + consumer_identity: str, + consumer_group: str, + topic: str, + event_type: EnumConsumerHealthEventType, + severity: EnumConsumerHealthSeverity, + correlation_id: UUID | None = None, + rebalance_duration_ms: int | None = None, + partitions_assigned: int | None = None, + partitions_revoked: int | None = None, + error_message: str = "", + error_type: str = "", + hostname: str = "", + service_label: str = "", + ) -> None: + """Convenience method to construct and emit in one call. + + Args: + consumer_identity: Kafka consumer identity string. + consumer_group: Kafka consumer group ID. + topic: Primary Kafka topic. + event_type: Type of health event. + severity: Severity level. + correlation_id: Optional correlation ID. + rebalance_duration_ms: Rebalance duration in ms. + partitions_assigned: Partitions assigned after rebalance. + partitions_revoked: Partitions revoked during rebalance. + error_message: Error message if applicable. + error_type: Exception type name if applicable. + hostname: Machine hostname. + service_label: Service display label. + """ + event = ModelConsumerHealthEvent.create( + consumer_identity=consumer_identity, + consumer_group=consumer_group, + topic=topic, + event_type=event_type, + severity=severity, + correlation_id=correlation_id, + rebalance_duration_ms=rebalance_duration_ms, + partitions_assigned=partitions_assigned, + partitions_revoked=partitions_revoked, + error_message=error_message, + error_type=error_type, + hostname=hostname, + service_label=service_label, + ) + await self.emit(event) + + +__all__ = ["ConsumerHealthEmitter"] diff --git a/src/omnibase_infra/event_bus/event_bus_kafka.py b/src/omnibase_infra/event_bus/event_bus_kafka.py index f84c5075cf..0f926845cc 100644 --- a/src/omnibase_infra/event_bus/event_bus_kafka.py +++ b/src/omnibase_infra/event_bus/event_bus_kafka.py @@ -211,6 +211,7 @@ async def handler(msg): ModelTimeoutErrorContext, ProtocolConfigurationError, ) +from omnibase_infra.event_bus.consumer_health_emitter import ConsumerHealthEmitter from omnibase_infra.event_bus.mixin_kafka_broadcast import MixinKafkaBroadcast from omnibase_infra.event_bus.mixin_kafka_dlq import MixinKafkaDlq from omnibase_infra.event_bus.models import ( @@ -222,6 +223,12 @@ async def handler(msg): from omnibase_infra.event_bus.topic_violation_alerter import TopicViolationAlerter from omnibase_infra.mixins import MixinAsyncCircuitBreaker from omnibase_infra.models import ModelNodeIdentity +from omnibase_infra.models.health.enum_consumer_health_event_type import ( + EnumConsumerHealthEventType, +) +from omnibase_infra.models.health.enum_consumer_health_severity import ( + EnumConsumerHealthSeverity, +) from omnibase_infra.observability.wiring_health import MixinEmissionCounter from omnibase_infra.utils import apply_instance_discriminator, compute_consumer_group_id from omnibase_infra.utils.util_consumer_group import KAFKA_CONSUMER_GROUP_MAX_LENGTH @@ -462,6 +469,10 @@ def __init__( TopicViolationAlerter() if self._topic_enforcement_mode != "off" else None ) + # Consumer health emitter (OMN-5518) — initialized after producer starts + # Gated by ENABLE_CONSUMER_HEALTH_EMITTER env var + self._health_emitter: ConsumerHealthEmitter | None = None + # ========================================================================= # Factory Methods # ========================================================================= @@ -563,6 +574,14 @@ def environment(self) -> str: """ return self._environment + @property + def health_emitter(self) -> ConsumerHealthEmitter | None: + """Get the consumer health emitter (OMN-5518). + + Returns None if the feature flag is off or bus has not started. + """ + return self._health_emitter + # ========================================================================= # Auth / TLS helpers # ========================================================================= @@ -653,6 +672,13 @@ async def start(self) -> None: self._shutdown = False self._closing = False + # Initialize consumer health emitter if feature flag is on (OMN-5518) + if ConsumerHealthEmitter.is_enabled() and self._producer is not None: + self._health_emitter = ConsumerHealthEmitter(self._producer) + logger.info( + "ConsumerHealthEmitter initialized (ENABLE_CONSUMER_HEALTH_EMITTER=on)" + ) + # Reset circuit breaker on success async with self._circuit_breaker_lock: await self._reset_circuit_breaker() @@ -1588,6 +1614,23 @@ async def _start_consumer_for_topic(self, topic: str, group_id: str) -> None: }, ) + # Emit consumer started health event (OMN-5518) + if self._health_emitter is not None: + try: + await self._health_emitter.emit_event( + consumer_identity=f"eventbus.{topic}", + consumer_group=effective_group_id, + topic=topic, + event_type=EnumConsumerHealthEventType.CONSUMER_STARTED, + severity=EnumConsumerHealthSeverity.INFO, + correlation_id=correlation_id, + service_label="EventBusKafka", + ) + except Exception: # noqa: BLE001 - best-effort emission + logger.debug( + "Failed to emit consumer started health event", exc_info=True + ) + except TimeoutError as e: # Clean up consumer on failure to prevent resource leak try: @@ -1860,6 +1903,46 @@ async def _consume_loop(self, topic: str, correlation_id: UUID) -> None: "error_type": type(e).__name__, }, ) + + # Emit consumer health event if emitter is available (OMN-5518) + if self._health_emitter is not None: + # Classify error type for severity + error_type_name = type(e).__name__ + is_session_timeout = ( + "session" in error_type_name.lower() or "timeout" in str(e).lower() + ) + event_type = ( + EnumConsumerHealthEventType.SESSION_TIMEOUT + if is_session_timeout + else EnumConsumerHealthEventType.CONNECTION_LOST + ) + severity = ( + EnumConsumerHealthSeverity.CRITICAL + if is_session_timeout + else EnumConsumerHealthSeverity.ERROR + ) + try: + # Derive consumer group from subscriber registry + _subs = self._subscribers.get(topic, []) + _consumer_group = _subs[0][0] if _subs else "unknown" + await self._health_emitter.emit_event( + consumer_identity=f"eventbus.{topic}", + consumer_group=_consumer_group, + topic=topic, + event_type=event_type, + severity=severity, + correlation_id=correlation_id, + error_message=str(e)[:500], + error_type=error_type_name, + hostname=os.environ.get("HOSTNAME", ""), # ONEX_EXCLUDE: env + service_label="EventBusKafka", + ) + except Exception: # noqa: BLE001 - best-effort emission must not propagate + logger.debug( + "Failed to emit health event for consumer loop error", + exc_info=True, + ) + # Don't raise - allow task to complete and cleanup to proceed finally: diff --git a/src/omnibase_infra/event_bus/mixin_consumer_health.py b/src/omnibase_infra/event_bus/mixin_consumer_health.py new file mode 100644 index 0000000000..24afd70ce4 --- /dev/null +++ b/src/omnibase_infra/event_bus/mixin_consumer_health.py @@ -0,0 +1,180 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Mixin providing consumer health emission for standalone consumers. + +Standalone Kafka consumers (those NOT managed by EventBusKafka) use this +mixin to emit health events and handle restart commands. + +Usage: + class MyConsumer(MixinConsumerHealth): + async def start(self): + await self._init_health_emitter(producer, consumer_identity="my-consumer", ...) + ... + + async def _handle_restart(self) -> None: + # Implement consumer-specific restart logic + ... + +Related Tickets: + - OMN-5519: Create MixinConsumerHealth for standalone consumers + - OMN-5529: Runtime Health Event Pipeline (epic) +""" + +from __future__ import annotations + +import logging +import os +from typing import TYPE_CHECKING + +from omnibase_infra.event_bus.consumer_health_emitter import ConsumerHealthEmitter +from omnibase_infra.models.health.enum_consumer_health_event_type import ( + EnumConsumerHealthEventType, +) +from omnibase_infra.models.health.enum_consumer_health_severity import ( + EnumConsumerHealthSeverity, +) + +if TYPE_CHECKING: + from uuid import UUID + + from aiokafka import AIOKafkaProducer + +logger = logging.getLogger(__name__) + + +class MixinConsumerHealth: + """Mixin providing consumer health emission for standalone consumers. + + Provides: + - Health event emission via ConsumerHealthEmitter + - Convenience methods for common event types + - Feature flag gating via ENABLE_CONSUMER_HEALTH_EMITTER + + Subclasses should call ``_init_health_emitter()`` during startup and + use ``_emit_health_event()`` to emit events. + """ + + _health_emitter: ConsumerHealthEmitter | None + + def _init_health_emitter( + self, + producer: AIOKafkaProducer, + *, + consumer_identity: str, + consumer_group: str, + topic: str, + service_label: str = "", + hostname: str = "", + ) -> None: + """Initialize the health emitter. + + Args: + producer: An already-started AIOKafkaProducer. + consumer_identity: Kafka consumer identity string. + consumer_group: Consumer group ID. + topic: Primary topic the consumer subscribes to. + service_label: Service display label. + hostname: Machine hostname. + """ + self._health_emitter = ConsumerHealthEmitter(producer) + self._health_consumer_identity = consumer_identity + self._health_consumer_group = consumer_group + self._health_topic = topic + self._health_service_label = service_label + self._health_hostname = hostname + + async def _emit_health_event( + self, + event_type: EnumConsumerHealthEventType, + severity: EnumConsumerHealthSeverity, + *, + correlation_id: UUID | None = None, + error_message: str = "", + error_type: str = "", + rebalance_duration_ms: int | None = None, + partitions_assigned: int | None = None, + partitions_revoked: int | None = None, + ) -> None: + """Emit a consumer health event if emitter is initialized and enabled. + + Args: + event_type: Type of health event. + severity: Severity level. + correlation_id: Optional correlation ID. + error_message: Error message if applicable. + error_type: Exception type name if applicable. + rebalance_duration_ms: Rebalance duration in ms. + partitions_assigned: Partitions assigned after rebalance. + partitions_revoked: Partitions revoked during rebalance. + """ + emitter = getattr(self, "_health_emitter", None) + if emitter is None: + return + + try: + await emitter.emit_event( + consumer_identity=self._health_consumer_identity, + consumer_group=self._health_consumer_group, + topic=self._health_topic, + event_type=event_type, + severity=severity, + correlation_id=correlation_id, + error_message=error_message, + error_type=error_type, + rebalance_duration_ms=rebalance_duration_ms, + partitions_assigned=partitions_assigned, + partitions_revoked=partitions_revoked, + hostname=self._health_hostname, + service_label=self._health_service_label, + ) + except Exception: # noqa: BLE001 - best-effort emission + logger.debug( + "Failed to emit health event for %s", + getattr(self, "_health_consumer_identity", "unknown"), + exc_info=True, + ) + + async def _emit_heartbeat_failure( + self, + error_message: str = "", + *, + correlation_id: UUID | None = None, + ) -> None: + """Convenience: emit heartbeat failure event.""" + await self._emit_health_event( + EnumConsumerHealthEventType.HEARTBEAT_FAILURE, + EnumConsumerHealthSeverity.ERROR, + error_message=error_message, + correlation_id=correlation_id, + ) + + async def _emit_session_timeout( + self, + error_message: str = "", + *, + correlation_id: UUID | None = None, + ) -> None: + """Convenience: emit session timeout event.""" + await self._emit_health_event( + EnumConsumerHealthEventType.SESSION_TIMEOUT, + EnumConsumerHealthSeverity.CRITICAL, + error_message=error_message, + correlation_id=correlation_id, + ) + + async def _emit_consumer_started(self) -> None: + """Convenience: emit consumer started event.""" + await self._emit_health_event( + EnumConsumerHealthEventType.CONSUMER_STARTED, + EnumConsumerHealthSeverity.INFO, + ) + + async def _emit_consumer_stopped(self) -> None: + """Convenience: emit consumer stopped event.""" + await self._emit_health_event( + EnumConsumerHealthEventType.CONSUMER_STOPPED, + EnumConsumerHealthSeverity.WARNING, + ) + + +__all__ = ["MixinConsumerHealth"] diff --git a/src/omnibase_infra/event_bus/topic_constants.py b/src/omnibase_infra/event_bus/topic_constants.py index 60d8122651..c7a69e1e2a 100644 --- a/src/omnibase_infra/event_bus/topic_constants.py +++ b/src/omnibase_infra/event_bus/topic_constants.py @@ -594,6 +594,44 @@ def derive_dlq_topic_for_event_type( Ticket: OMN-5545 """ +# Consumer Health topics (OMN-5515 / OMN-5529) +TOPIC_CONSUMER_HEALTH: Final[str] = "onex.evt.omnibase-infra.consumer-health.v1" +"""Consumer health events emitted by ConsumerHealthEmitter. + +Emitted on heartbeat failures, session timeouts, rebalance events, +and other consumer lifecycle state changes. Consumed by +NodeConsumerHealthTriageEffect for graduated response. + +Producer: ConsumerHealthEmitter (EventBusKafka, standalone consumers) +Consumer: NodeConsumerHealthTriageEffect, Omnidash /consumer-health dashboard +Ticket: OMN-5515 +""" + +TOPIC_CONSUMER_RESTART_CMD: Final[str] = "onex.cmd.omnibase-infra.consumer-restart.v1" +"""Consumer restart commands issued by the triage node. + +Published when graduated response escalates to automated restart. +Each standalone consumer subscribes to this topic to receive restart +commands targeted at its consumer identity. + +Producer: NodeConsumerHealthTriageEffect +Consumer: MixinConsumerHealth (standalone consumers) +Ticket: OMN-5515 +""" + +# Runtime Error topics (OMN-5517 / OMN-5529) +TOPIC_RUNTIME_ERROR: Final[str] = "onex.evt.omnibase-infra.runtime-error.v1" +"""Runtime error events emitted by RuntimeLogEventBridge. + +Captures ERROR/WARNING log records from allowlisted Python loggers, +fingerprints and deduplicates them, and emits structured events. +Consumed by NodeRuntimeErrorTriageEffect for triage. + +Producer: RuntimeLogEventBridge (logging.Handler) +Consumer: NodeRuntimeErrorTriageEffect, Omnidash /runtime-errors dashboard +Ticket: OMN-5517 +""" + TOPIC_HOOK_CONTEXT_INJECTED: Final[str] = "onex.evt.omniclaude.hook-context-injected.v1" """Hook context injection events from omniclaude UserPromptSubmit hooks. @@ -635,6 +673,11 @@ def derive_dlq_topic_for_event_type( "TOPIC_LLM_CALL_COMPLETED_INFRA", # LLM Endpoint Health Topics "TOPIC_LLM_ENDPOINT_HEALTH", + # Consumer Health Topics (OMN-5515) + "TOPIC_CONSUMER_HEALTH", + "TOPIC_CONSUMER_RESTART_CMD", + # Runtime Error Topics (OMN-5517) + "TOPIC_RUNTIME_ERROR", # Wiring Health Snapshot Topics (OMN-5292) "TOPIC_WIRING_HEALTH_SNAPSHOT", # Savings Estimation Topics (OMN-5545) diff --git a/src/omnibase_infra/models/health/__init__.py b/src/omnibase_infra/models/health/__init__.py index 2622e02d1a..eac52da4db 100644 --- a/src/omnibase_infra/models/health/__init__.py +++ b/src/omnibase_infra/models/health/__init__.py @@ -2,6 +2,27 @@ # SPDX-License-Identifier: MIT """Health check models for infrastructure components.""" +from omnibase_infra.models.health.enum_consumer_health_event_type import ( + EnumConsumerHealthEventType, +) +from omnibase_infra.models.health.enum_consumer_health_severity import ( + EnumConsumerHealthSeverity, +) +from omnibase_infra.models.health.enum_consumer_incident_state import ( + EnumConsumerIncidentState, +) +from omnibase_infra.models.health.enum_runtime_error_category import ( + EnumRuntimeErrorCategory, +) +from omnibase_infra.models.health.enum_runtime_error_severity import ( + EnumRuntimeErrorSeverity, +) +from omnibase_infra.models.health.model_consumer_health_event import ( + ModelConsumerHealthEvent, +) +from omnibase_infra.models.health.model_consumer_restart_command import ( + ModelConsumerRestartCommand, +) from omnibase_infra.models.health.model_health_check_result import ( ModelHealthCheckResult, ) @@ -14,10 +35,21 @@ from omnibase_infra.models.health.model_llm_endpoint_status import ( ModelLlmEndpointStatus, ) +from omnibase_infra.models.health.model_runtime_error_event import ( + ModelRuntimeErrorEvent, +) __all__ = [ + "EnumConsumerHealthEventType", + "EnumConsumerHealthSeverity", + "EnumConsumerIncidentState", + "EnumRuntimeErrorCategory", + "EnumRuntimeErrorSeverity", + "ModelConsumerHealthEvent", + "ModelConsumerRestartCommand", "ModelHealthCheckResult", "ModelLlmEndpointHealthConfig", "ModelLlmEndpointHealthEvent", "ModelLlmEndpointStatus", + "ModelRuntimeErrorEvent", ] diff --git a/src/omnibase_infra/models/health/enum_consumer_health_event_type.py b/src/omnibase_infra/models/health/enum_consumer_health_event_type.py new file mode 100644 index 0000000000..330d4c3e79 --- /dev/null +++ b/src/omnibase_infra/models/health/enum_consumer_health_event_type.py @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Consumer health event type classification enum. + +Ticket: OMN-5511 +""" + +from __future__ import annotations + +from enum import StrEnum + + +class EnumConsumerHealthEventType(StrEnum): + """Classification of consumer health events.""" + + HEARTBEAT_FAILURE = "heartbeat_failure" + SESSION_TIMEOUT = "session_timeout" + REBALANCE_START = "rebalance_start" + REBALANCE_COMPLETE = "rebalance_complete" + CONSUMER_STOPPED = "consumer_stopped" + CONSUMER_STARTED = "consumer_started" + POLL_TIMEOUT = "poll_timeout" + CONNECTION_LOST = "connection_lost" + + +__all__ = ["EnumConsumerHealthEventType"] diff --git a/src/omnibase_infra/models/health/enum_consumer_health_severity.py b/src/omnibase_infra/models/health/enum_consumer_health_severity.py new file mode 100644 index 0000000000..908258351e --- /dev/null +++ b/src/omnibase_infra/models/health/enum_consumer_health_severity.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Consumer health severity level enum. + +Ticket: OMN-5511 +""" + +from __future__ import annotations + +from enum import StrEnum + + +class EnumConsumerHealthSeverity(StrEnum): + """Severity level for consumer health events.""" + + INFO = "info" + WARNING = "warning" + ERROR = "error" + CRITICAL = "critical" + + +__all__ = ["EnumConsumerHealthSeverity"] diff --git a/src/omnibase_infra/models/health/enum_consumer_incident_state.py b/src/omnibase_infra/models/health/enum_consumer_incident_state.py new file mode 100644 index 0000000000..c8a5488bbe --- /dev/null +++ b/src/omnibase_infra/models/health/enum_consumer_incident_state.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Consumer health incident lifecycle state enum. + +Ticket: OMN-5511 +""" + +from __future__ import annotations + +from enum import StrEnum + + +class EnumConsumerIncidentState(StrEnum): + """Lifecycle state of a consumer health incident.""" + + OPEN = "open" + ACKNOWLEDGED = "acknowledged" + RESTART_PENDING = "restart_pending" + RESTART_SUCCEEDED = "restart_succeeded" + RESTART_FAILED = "restart_failed" + TICKETED = "ticketed" + RESOLVED = "resolved" + + +__all__ = ["EnumConsumerIncidentState"] diff --git a/src/omnibase_infra/models/health/enum_runtime_error_category.py b/src/omnibase_infra/models/health/enum_runtime_error_category.py new file mode 100644 index 0000000000..4ae9073998 --- /dev/null +++ b/src/omnibase_infra/models/health/enum_runtime_error_category.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Runtime error category classification enum. + +Ticket: OMN-5513 +""" + +from __future__ import annotations + +from enum import StrEnum + + +class EnumRuntimeErrorCategory(StrEnum): + """Classification of runtime errors by subsystem.""" + + KAFKA_CONSUMER = "kafka_consumer" + KAFKA_PRODUCER = "kafka_producer" + DATABASE = "database" + HTTP_CLIENT = "http_client" + HTTP_SERVER = "http_server" + RUNTIME = "runtime" + UNKNOWN = "unknown" + + +__all__ = ["EnumRuntimeErrorCategory"] diff --git a/src/omnibase_infra/models/health/enum_runtime_error_severity.py b/src/omnibase_infra/models/health/enum_runtime_error_severity.py new file mode 100644 index 0000000000..5817783b91 --- /dev/null +++ b/src/omnibase_infra/models/health/enum_runtime_error_severity.py @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Runtime error severity level enum. + +Ticket: OMN-5513 +""" + +from __future__ import annotations + +from enum import StrEnum + + +class EnumRuntimeErrorSeverity(StrEnum): + """Severity level for runtime error events.""" + + WARNING = "warning" + ERROR = "error" + CRITICAL = "critical" + + +__all__ = ["EnumRuntimeErrorSeverity"] diff --git a/src/omnibase_infra/models/health/model_consumer_health_event.py b/src/omnibase_infra/models/health/model_consumer_health_event.py new file mode 100644 index 0000000000..df8ef64e95 --- /dev/null +++ b/src/omnibase_infra/models/health/model_consumer_health_event.py @@ -0,0 +1,188 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Consumer health event model for Layer 1 health pipeline. + +Structured event model emitted by ConsumerHealthEmitter when consumer +lifecycle events occur (heartbeat failures, rebalances, session timeouts). +The fingerprint field enables deduplication and incident grouping in the +triage node. + +Related Tickets: + - OMN-5511: Create consumer health event models and enums + - OMN-5529: Runtime Health Event Pipeline (epic) +""" + +from __future__ import annotations + +import hashlib +from datetime import UTC, datetime +from uuid import UUID, uuid4 + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_infra.models.health.enum_consumer_health_event_type import ( + EnumConsumerHealthEventType, +) +from omnibase_infra.models.health.enum_consumer_health_severity import ( + EnumConsumerHealthSeverity, +) + + +def _compute_fingerprint( + consumer_identity: str, + event_type: str, + topic: str, +) -> str: + """Compute a stable fingerprint for deduplication and incident grouping. + + Args: + consumer_identity: Unique consumer identifier. + event_type: The event type string. + topic: The Kafka topic the consumer is subscribed to. + + Returns: + Hex digest fingerprint string. + """ + raw = f"{consumer_identity}:{event_type}:{topic}" + return hashlib.sha256(raw.encode()).hexdigest()[:16] + + +class ModelConsumerHealthEvent(BaseModel): + """Structured consumer health event for Kafka emission. + + Emitted by ConsumerHealthEmitter when consumer lifecycle events + occur (heartbeat failures, rebalances, session timeouts, etc.). + """ + + model_config = ConfigDict( + frozen=True, + extra="forbid", + from_attributes=True, + ) + + # Identity + event_id: UUID = Field( + default_factory=uuid4, description="Unique event identifier." + ) + correlation_id: UUID = Field( + default_factory=uuid4, + description="Correlation ID for tracing across services.", + ) + consumer_identity: str = Field( + ..., description="Unique identifier for the consumer instance." + ) + consumer_group: str = Field(..., description="Kafka consumer group ID.") + topic: str = Field( + ..., description="Primary Kafka topic the consumer is subscribed to." + ) + + # Classification + event_type: EnumConsumerHealthEventType = Field( + ..., description="Type of consumer health event." + ) + severity: EnumConsumerHealthSeverity = Field( + ..., description="Severity level of the event." + ) + + # Fingerprint for deduplication + fingerprint: str = Field( + ..., + description="Stable hash for deduplication and incident grouping.", + ) + + # Rebalance metrics (populated for rebalance events) + rebalance_duration_ms: int | None = Field( + default=None, + description="Duration of rebalance in milliseconds (for REBALANCE_COMPLETE).", + ) + partitions_assigned: int | None = Field( + default=None, + description="Number of partitions assigned after rebalance.", + ) + partitions_revoked: int | None = Field( + default=None, + description="Number of partitions revoked during rebalance.", + ) + + # Error context + error_message: str = Field( + default="", + description="Error message if applicable. Empty if no error.", + ) + error_type: str = Field( + default="", + description="Exception type name if applicable. Empty if no error.", + ) + + # Metadata + hostname: str = Field( + default="", description="Hostname of the machine running the consumer." + ) + service_label: str = Field( + default="", description="Display name of the service owning the consumer." + ) + emitted_at: datetime = Field( + default_factory=lambda: datetime.now(UTC), + description="Timestamp when the event was emitted.", + ) + + @classmethod + def create( + cls, + *, + consumer_identity: str, + consumer_group: str, + topic: str, + event_type: EnumConsumerHealthEventType, + severity: EnumConsumerHealthSeverity, + correlation_id: UUID | None = None, + rebalance_duration_ms: int | None = None, + partitions_assigned: int | None = None, + partitions_revoked: int | None = None, + error_message: str = "", + error_type: str = "", + hostname: str = "", + service_label: str = "", + ) -> ModelConsumerHealthEvent: + """Factory method that auto-computes fingerprint. + + Args: + consumer_identity: Unique consumer identifier. + consumer_group: Kafka consumer group ID. + topic: Primary Kafka topic. + event_type: Type of health event. + severity: Severity level. + correlation_id: Optional correlation ID (auto-generated if None). + rebalance_duration_ms: Rebalance duration in ms. + partitions_assigned: Partitions assigned after rebalance. + partitions_revoked: Partitions revoked during rebalance. + error_message: Error message if applicable. + error_type: Exception type name if applicable. + hostname: Machine hostname. + service_label: Service display name. + + Returns: + A new ModelConsumerHealthEvent with computed fingerprint. + """ + fingerprint = _compute_fingerprint(consumer_identity, event_type.value, topic) + return cls( + consumer_identity=consumer_identity, + consumer_group=consumer_group, + topic=topic, + event_type=event_type, + severity=severity, + fingerprint=fingerprint, + correlation_id=correlation_id or uuid4(), + rebalance_duration_ms=rebalance_duration_ms, + partitions_assigned=partitions_assigned, + partitions_revoked=partitions_revoked, + error_message=error_message, + error_type=error_type, + hostname=hostname, + service_label=service_label, + ) + + +__all__ = [ + "ModelConsumerHealthEvent", +] diff --git a/src/omnibase_infra/models/health/model_consumer_restart_command.py b/src/omnibase_infra/models/health/model_consumer_restart_command.py new file mode 100644 index 0000000000..6168831a45 --- /dev/null +++ b/src/omnibase_infra/models/health/model_consumer_restart_command.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Consumer restart command model for Layer 1 health pipeline. + +Published to the consumer restart command topic by the triage node +when graduated response escalates to automated restart. + +Related Tickets: + - OMN-5511: Create consumer health event models and enums + - OMN-5529: Runtime Health Event Pipeline (epic) +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Literal +from uuid import UUID, uuid4 + +from pydantic import BaseModel, ConfigDict, Field + + +class ModelConsumerRestartCommand(BaseModel): + """Command to restart a specific consumer instance. + + Published to the consumer restart command topic by the triage node + when graduated response escalates to automated restart. + """ + + model_config = ConfigDict( + frozen=True, + extra="forbid", + from_attributes=True, + ) + + command_id: UUID = Field( + default_factory=uuid4, description="Unique command identifier." + ) + correlation_id: UUID = Field( + default_factory=uuid4, + description="Correlation ID linking to the triggering health event.", + ) + consumer_identity: str = Field(..., description="Target consumer to restart.") + consumer_group: str = Field( + ..., description="Consumer group of the target consumer." + ) + topic: str = Field(..., description="Topic the consumer is subscribed to.") + reason: str = Field( + ..., description="Human-readable reason for the restart command." + ) + fingerprint: str = Field( + ..., description="Fingerprint of the incident triggering the restart." + ) + intent_type: Literal["consumer_health.restart"] = Field( + default="consumer_health.restart", + description="Intent type for routing.", + ) + issued_at: datetime = Field( + default_factory=lambda: datetime.now(UTC), + description="Timestamp when the command was issued.", + ) + + +__all__ = ["ModelConsumerRestartCommand"] diff --git a/src/omnibase_infra/models/health/model_runtime_error_event.py b/src/omnibase_infra/models/health/model_runtime_error_event.py new file mode 100644 index 0000000000..03c0592269 --- /dev/null +++ b/src/omnibase_infra/models/health/model_runtime_error_event.py @@ -0,0 +1,195 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Runtime error event model for Layer 2 health pipeline. + +Structured event model for runtime log errors captured by +RuntimeLogEventBridge (a Python logging.Handler) and emitted to Kafka. + +Each event carries a fingerprint for deduplication, error category for +triage routing, and local occurrence count for rate-limited aggregation. + +Related Tickets: + - OMN-5513: Create runtime error event model + - OMN-5529: Runtime Health Event Pipeline (epic) +""" + +from __future__ import annotations + +import hashlib +from datetime import UTC, datetime +from uuid import UUID, uuid4 + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_infra.models.health.enum_runtime_error_category import ( + EnumRuntimeErrorCategory, +) +from omnibase_infra.models.health.enum_runtime_error_severity import ( + EnumRuntimeErrorSeverity, +) + + +def _compute_error_fingerprint( + logger_name: str, + message_template: str, + error_category: str, +) -> str: + """Compute a stable fingerprint for error deduplication. + + Args: + logger_name: Name of the Python logger that emitted the record. + message_template: Templatized message (variable parts replaced). + error_category: Error category string. + + Returns: + Hex digest fingerprint string. + """ + raw = f"{logger_name}:{error_category}:{message_template}" + return hashlib.sha256(raw.encode()).hexdigest()[:16] + + +class ModelRuntimeErrorEvent(BaseModel): + """Structured runtime error event for Kafka emission. + + Emitted by RuntimeLogEventBridge when ERROR/WARNING log records + are captured from allowlisted loggers. + """ + + model_config = ConfigDict( + frozen=True, + extra="forbid", + from_attributes=True, + ) + + # Identity + event_id: UUID = Field( + default_factory=uuid4, description="Unique event identifier." + ) + correlation_id: UUID = Field( + default_factory=uuid4, + description="Correlation ID for tracing across services.", + ) + + # Logger info + logger_family: str = Field( + ..., description="Name of the Python logger that emitted the record." + ) + log_level: str = Field(..., description="Log level (ERROR, WARNING, CRITICAL).") + + # Message + message_template: str = Field( + ..., + description=( + "Templatized message with variable parts replaced by placeholders. " + "Used for fingerprint computation." + ), + ) + raw_message: str = Field(..., description="Original unmodified log message.") + + # Classification + error_category: EnumRuntimeErrorCategory = Field( + ..., description="Subsystem classification of the error." + ) + severity: EnumRuntimeErrorSeverity = Field( + ..., description="Severity level of the error event." + ) + + # Fingerprint for deduplication + fingerprint: str = Field( + ..., + description="Stable hash for deduplication and incident grouping.", + ) + + # Aggregation + occurrence_count_local: int = Field( + default=1, + description=( + "Number of occurrences of this fingerprint seen locally " + "before emission (rate-limited aggregation)." + ), + ) + + # Error context + exception_type: str = Field( + default="", + description="Exception class name if the log record included exc_info.", + ) + exception_message: str = Field( + default="", + description="Exception message if the log record included exc_info.", + ) + stack_trace: str = Field( + default="", + description="Truncated stack trace if available.", + ) + + # Metadata + hostname: str = Field(default="", description="Hostname of the machine.") + service_label: str = Field(default="", description="Display name of the service.") + emitted_at: datetime = Field( + default_factory=lambda: datetime.now(UTC), + description="Timestamp when the event was emitted.", + ) + + @classmethod + def create( + cls, + *, + logger_family: str, + log_level: str, + message_template: str, + raw_message: str, + error_category: EnumRuntimeErrorCategory, + severity: EnumRuntimeErrorSeverity, + correlation_id: UUID | None = None, + occurrence_count_local: int = 1, + exception_type: str = "", + exception_message: str = "", + stack_trace: str = "", + hostname: str = "", + service_label: str = "", + ) -> ModelRuntimeErrorEvent: + """Factory method that auto-computes fingerprint. + + Args: + logger_family: Python logger name. + log_level: Log level string. + message_template: Templatized message. + raw_message: Original log message. + error_category: Subsystem classification. + severity: Severity level. + correlation_id: Optional correlation ID. + occurrence_count_local: Local occurrence count. + exception_type: Exception class name. + exception_message: Exception message. + stack_trace: Truncated stack trace. + hostname: Machine hostname. + service_label: Service display name. + + Returns: + A new ModelRuntimeErrorEvent with computed fingerprint. + """ + fingerprint = _compute_error_fingerprint( + logger_family, message_template, error_category.value + ) + return cls( + logger_family=logger_family, + log_level=log_level, + message_template=message_template, + raw_message=raw_message, + error_category=error_category, + severity=severity, + fingerprint=fingerprint, + correlation_id=correlation_id or uuid4(), + occurrence_count_local=occurrence_count_local, + exception_type=exception_type, + exception_message=exception_message, + stack_trace=stack_trace, + hostname=hostname, + service_label=service_label, + ) + + +__all__ = [ + "ModelRuntimeErrorEvent", +] diff --git a/src/omnibase_infra/nodes/node_consumer_health_triage_effect/__init__.py b/src/omnibase_infra/nodes/node_consumer_health_triage_effect/__init__.py new file mode 100644 index 0000000000..1f0227d616 --- /dev/null +++ b/src/omnibase_infra/nodes/node_consumer_health_triage_effect/__init__.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""NodeConsumerHealthTriageEffect — graduated triage for consumer health events.""" + +from omnibase_infra.nodes.node_consumer_health_triage_effect.node import ( + NodeConsumerHealthTriageEffect, +) + +__all__ = ["NodeConsumerHealthTriageEffect"] diff --git a/src/omnibase_infra/nodes/node_consumer_health_triage_effect/contract.yaml b/src/omnibase_infra/nodes/node_consumer_health_triage_effect/contract.yaml new file mode 100644 index 0000000000..b7c28455dc --- /dev/null +++ b/src/omnibase_infra/nodes/node_consumer_health_triage_effect/contract.yaml @@ -0,0 +1,149 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# Copyright (c) 2025 OmniNode Team +# +# ONEX Node Contract +# Node: NodeConsumerHealthTriageEffect +# +# Consumes onex.evt.omnibase-infra.consumer-health.v1 events and applies +# graduated triage: Slack warning -> Slack repeated -> restart -> Linear. +# +# Related Tickets: +# - OMN-5520: Create NodeConsumerHealthTriageEffect +# - OMN-5529: Runtime Health Event Pipeline (epic) +# Contract identifiers +name: "node_consumer_health_triage_effect" +contract_name: "node_consumer_health_triage_effect" +node_name: "node_consumer_health_triage_effect" +contract_version: + major: 1 + minor: 0 + patch: 0 +node_version: + major: 1 + minor: 0 + patch: 0 +# Node type +node_type: "EFFECT_GENERIC" +# Description +description: > + Effect node that consumes consumer health events from onex.evt.omnibase-infra.consumer-health.v1 and applies graduated triage response. First occurrence triggers Slack WARNING, second triggers Slack REPEATED, third in 30 minutes triggers restart command (gated by ENABLE_CONSUMER_AUTO_RESTART), restart failure triggers Linear ticket. Uses consumer_health_triage and consumer_restart_state PostgreSQL tables. + +# Event bus subscription +event_bus: + version: + major: 1 + minor: 0 + patch: 0 + subscribe_topics: + - "onex.evt.omnibase-infra.consumer-health.v1" + publish_topics: + - "onex.cmd.omnibase-infra.consumer-restart.v1" +# I/O models +input_model: + name: "ModelConsumerHealthEvent" + module: "omnibase_infra.models.health.model_consumer_health_event" + description: "Structured consumer health event from Kafka." +output_model: + name: "ModelTriageResult" + module: "omnibase_infra.nodes.node_consumer_health_triage_effect.handlers.handler_consumer_health_triage" + description: "Result of the graduated triage response." +# Handler routing +handler_routing: + routing_strategy: "operation_match" + handlers: + - operation: "triage_health_event" + handler: + name: "HandlerConsumerHealthTriage" + module: "omnibase_infra.nodes.node_consumer_health_triage_effect.handlers.handler_consumer_health_triage" + description: "Graduated triage: Slack warning -> repeated -> restart -> Linear" + output_fields: + - fingerprint + - action + - incident_state + - occurrence_count +# Error handling +error_handling: + retry_policy: + max_retries: 3 + initial_delay_ms: 1000 + max_delay_ms: 8000 + exponential_base: 2 + retry_on: + - "DB_POOL_TIMEOUT" + - "SLACK_API_ERROR" + error_types: + - name: "DB_POOL_UNAVAILABLE" + description: "asyncpg pool not available for triage state" + recoverable: false + retry_strategy: "none" + - name: "DB_POOL_TIMEOUT" + description: "asyncpg pool query timed out" + recoverable: true + retry_strategy: "exponential_backoff" + - name: "SLACK_API_ERROR" + description: "Slack notification failed" + recoverable: true + retry_strategy: "exponential_backoff" + - name: "LINEAR_API_ERROR" + description: "Linear ticket creation failed" + recoverable: true + retry_strategy: "exponential_backoff" +# IO operations +io_operations: + - operation: "triage_health_event" + description: "Apply graduated triage response to consumer health event" + input_fields: + - fingerprint + - consumer_identity + - consumer_group + - topic + - event_type + - severity + - error_message + output_fields: + - fingerprint + - action + - incident_state + - occurrence_count +# Dependencies +dependencies: + - name: "db_pool" + type: "service" + description: "asyncpg pool for consumer_health_triage and consumer_restart_state tables" + required: true + - name: "producer" + type: "service" + description: "AIOKafkaProducer for restart commands" + required: false + - name: "slack_handler" + type: "service" + description: "Slack webhook handler for notifications" + required: false + - name: "linear_handler" + type: "service" + description: "Linear API handler for ticket creation" + required: false +# Feature flags +feature_flags: + - name: "ENABLE_CONSUMER_HEALTH_TRIAGE" + env_var: "ENABLE_CONSUMER_HEALTH_TRIAGE" + default: "false" + description: "Master gate for consumer health triage" + - name: "ENABLE_CONSUMER_AUTO_RESTART" + env_var: "ENABLE_CONSUMER_AUTO_RESTART" + default: "false" + description: "Gate for automated restart commands" +# Metadata +metadata: + author: "OmniNode Team" + license: "MIT" + created: "2026-03-19" + updated: "2026-03-19" + tags: + - effect + - health + - triage + - consumer + - kafka + - graduated-response diff --git a/src/omnibase_infra/nodes/node_consumer_health_triage_effect/handlers/__init__.py b/src/omnibase_infra/nodes/node_consumer_health_triage_effect/handlers/__init__.py new file mode 100644 index 0000000000..87b100c0ae --- /dev/null +++ b/src/omnibase_infra/nodes/node_consumer_health_triage_effect/handlers/__init__.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Handlers for NodeConsumerHealthTriageEffect.""" + +from omnibase_infra.nodes.node_consumer_health_triage_effect.handlers.handler_consumer_health_triage import ( + HandlerConsumerHealthTriage, +) + +__all__ = ["HandlerConsumerHealthTriage"] diff --git a/src/omnibase_infra/nodes/node_consumer_health_triage_effect/handlers/handler_consumer_health_triage.py b/src/omnibase_infra/nodes/node_consumer_health_triage_effect/handlers/handler_consumer_health_triage.py new file mode 100644 index 0000000000..c45d38d2ea --- /dev/null +++ b/src/omnibase_infra/nodes/node_consumer_health_triage_effect/handlers/handler_consumer_health_triage.py @@ -0,0 +1,416 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Handler for consumer health triage with graduated response. + +Graduated response policy: + 1st occurrence -> Slack WARNING notification + 2nd occurrence -> Slack REPEATED notification + 3rd in 30 min -> Emit restart command (gated by ENABLE_CONSUMER_AUTO_RESTART) + Restart failure -> Create Linear ticket + +All state is tracked in PostgreSQL tables: + - consumer_health_triage: incident tracking by fingerprint + - consumer_restart_state: restart rate limiting by consumer identity + +Gated by ENABLE_CONSUMER_HEALTH_TRIAGE env var (default off). + +Related Tickets: + - OMN-5520: Create NodeConsumerHealthTriageEffect + - OMN-5529: Runtime Health Event Pipeline (epic) +""" + +from __future__ import annotations + +import json +import logging +import os +from collections.abc import Awaitable, Callable +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING, cast + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_infra.enums import EnumHandlerType, EnumHandlerTypeCategory +from omnibase_infra.event_bus.topic_constants import TOPIC_CONSUMER_RESTART_CMD +from omnibase_infra.models.health.enum_consumer_incident_state import ( + EnumConsumerIncidentState, +) +from omnibase_infra.models.health.model_consumer_health_event import ( + ModelConsumerHealthEvent, +) +from omnibase_infra.models.health.model_consumer_restart_command import ( + ModelConsumerRestartCommand, +) + +if TYPE_CHECKING: + from aiokafka import AIOKafkaProducer + from asyncpg import Pool + +logger = logging.getLogger(__name__) + +# Graduated response thresholds +_RESTART_THRESHOLD = 3 # 3rd occurrence triggers restart +_RESTART_WINDOW_MINUTES = 30 # Window for counting occurrences +_MAX_RESTARTS_PER_WINDOW = 2 # Max restarts in the window + + +class ModelTriageResult(BaseModel): + """Result of a consumer health triage operation.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + fingerprint: str = Field(..., description="Event fingerprint that was triaged.") + action: str = Field( + ..., + description="Action taken: 'slack_warning', 'slack_repeated', 'restart_command', 'linear_ticket', 'suppressed'.", + ) + incident_state: str = Field(..., description="New incident state after triage.") + occurrence_count: int = Field( + ..., description="Total occurrences for this fingerprint." + ) + + +class HandlerConsumerHealthTriage: + """Handler that applies graduated triage response to consumer health events. + + Dependencies (injected via constructor): + - db_pool: asyncpg connection pool for PostgreSQL + - producer: AIOKafkaProducer for emitting restart commands + - slack_handler: Optional callable for Slack notifications + - linear_handler: Optional callable for Linear ticket creation + + Feature Flags: + - ENABLE_CONSUMER_HEALTH_TRIAGE: Master gate (default off) + - ENABLE_CONSUMER_AUTO_RESTART: Restart command gate (default off) + """ + + def __init__( + self, + db_pool: Pool, + producer: AIOKafkaProducer | None = None, + *, + slack_handler: Callable[[str], Awaitable[None]] | None = None, + linear_handler: Callable[..., Awaitable[None]] | None = None, + ) -> None: + """Initialize the triage handler. + + Args: + db_pool: asyncpg connection pool. + producer: AIOKafkaProducer for restart commands (optional). + slack_handler: Async callable accepting a message string. + linear_handler: Async callable accepting title and description kwargs. + """ + self._db_pool = db_pool + self._producer = producer + self._slack_handler = slack_handler + self._linear_handler = linear_handler + + @property + def handler_type(self) -> EnumHandlerType: + """Architectural role: infrastructure handler for health triage.""" + return EnumHandlerType.INFRA_HANDLER + + @property + def handler_category(self) -> EnumHandlerTypeCategory: + """Behavioral classification: side-effecting triage operation.""" + return EnumHandlerTypeCategory.EFFECT + + @staticmethod + def is_enabled() -> bool: + """Check if consumer health triage is enabled.""" + return os.environ.get( # ONEX_EXCLUDE: env + "ENABLE_CONSUMER_HEALTH_TRIAGE", "" + ).strip().lower() in {"1", "true", "yes", "on"} + + @staticmethod + def is_auto_restart_enabled() -> bool: + """Check if automated restart is enabled.""" + return os.environ.get( # ONEX_EXCLUDE: env + "ENABLE_CONSUMER_AUTO_RESTART", "" + ).strip().lower() in {"1", "true", "yes", "on"} + + async def handle(self, event: ModelConsumerHealthEvent) -> ModelTriageResult: + """Apply graduated triage to a consumer health event. + + Args: + event: The consumer health event to triage. + + Returns: + ModelTriageResult with action taken and new state. + """ + if not self.is_enabled(): + return ModelTriageResult( + fingerprint=event.fingerprint, + action="suppressed", + incident_state=EnumConsumerIncidentState.OPEN, + occurrence_count=0, + ) + + # Upsert incident in consumer_health_triage table + incident = await self._upsert_incident(event) + occurrence_count = cast("int", incident["occurrence_count"]) + + # Graduated response logic + if occurrence_count >= _RESTART_THRESHOLD and self.is_auto_restart_enabled(): + # Check restart rate limit before issuing restart + can_restart = await self._check_restart_rate_limit(event) + if can_restart: + await self._emit_restart_command(event) + await self._update_incident_state( + event.fingerprint, EnumConsumerIncidentState.RESTART_PENDING + ) + return ModelTriageResult( + fingerprint=event.fingerprint, + action="restart_command", + incident_state=EnumConsumerIncidentState.RESTART_PENDING, + occurrence_count=occurrence_count, + ) + else: + # Rate limited — escalate to Linear ticket + await self._create_linear_ticket(event, occurrence_count) + await self._update_incident_state( + event.fingerprint, EnumConsumerIncidentState.TICKETED + ) + return ModelTriageResult( + fingerprint=event.fingerprint, + action="linear_ticket", + incident_state=EnumConsumerIncidentState.TICKETED, + occurrence_count=occurrence_count, + ) + elif occurrence_count >= 2: + # 2nd+ occurrence — Slack repeated notification + await self._send_slack_notification(event, repeated=True) + return ModelTriageResult( + fingerprint=event.fingerprint, + action="slack_repeated", + incident_state=EnumConsumerIncidentState.OPEN, + occurrence_count=occurrence_count, + ) + else: + # 1st occurrence — Slack warning + await self._send_slack_notification(event, repeated=False) + return ModelTriageResult( + fingerprint=event.fingerprint, + action="slack_warning", + incident_state=EnumConsumerIncidentState.OPEN, + occurrence_count=occurrence_count, + ) + + async def _upsert_incident( + self, event: ModelConsumerHealthEvent + ) -> dict[str, object]: + """Upsert incident row in consumer_health_triage table. + + Returns dict with at least 'occurrence_count'. + """ + async with self._db_pool.acquire() as conn: + row = await conn.fetchrow( + """ + INSERT INTO consumer_health_triage ( + fingerprint, consumer_id, consumer_group, topic, + event_type, severity, incident_state, + occurrence_count, first_seen_at, last_seen_at, + error_message, service_name, hostname, correlation_id + ) VALUES ($1, $2, $3, $4, $5, $6, $7, 1, NOW(), NOW(), $8, $9, $10, $11) + ON CONFLICT (fingerprint) WHERE incident_state IN ('open', 'acknowledged') + DO UPDATE SET + occurrence_count = consumer_health_triage.occurrence_count + 1, + last_seen_at = NOW(), + severity = EXCLUDED.severity, + error_message = EXCLUDED.error_message, + correlation_id = EXCLUDED.correlation_id + RETURNING occurrence_count, incident_state + """, + event.fingerprint, + event.consumer_identity, + event.consumer_group, + event.topic, + event.event_type.value, + event.severity.value, + EnumConsumerIncidentState.OPEN.value, + event.error_message, + event.service_label, + event.hostname, + str(event.correlation_id), + ) + if row is None: + # Insert without ON CONFLICT (fingerprint has resolved/ticketed incident) + row = await conn.fetchrow( + """ + INSERT INTO consumer_health_triage ( + fingerprint, consumer_id, consumer_group, topic, + event_type, severity, incident_state, + occurrence_count, first_seen_at, last_seen_at, + error_message, service_name, hostname, correlation_id + ) VALUES ($1, $2, $3, $4, $5, $6, $7, 1, NOW(), NOW(), $8, $9, $10, $11) + RETURNING occurrence_count, incident_state + """, + event.fingerprint, + event.consumer_identity, + event.consumer_group, + event.topic, + event.event_type.value, + event.severity.value, + EnumConsumerIncidentState.OPEN.value, + event.error_message, + event.service_label, + event.hostname, + str(event.correlation_id), + ) + return ( + dict(row) if row else {"occurrence_count": 1, "incident_state": "open"} + ) + + async def _update_incident_state( + self, fingerprint: str, state: EnumConsumerIncidentState + ) -> None: + """Update the incident state for a fingerprint.""" + async with self._db_pool.acquire() as conn: + await conn.execute( + """ + UPDATE consumer_health_triage + SET incident_state = $1, escalated_at = CASE WHEN $1 IN ('restart_pending', 'ticketed') THEN NOW() ELSE escalated_at END + WHERE fingerprint = $2 AND incident_state NOT IN ('resolved') + """, + state.value, + fingerprint, + ) + + async def _check_restart_rate_limit(self, event: ModelConsumerHealthEvent) -> bool: + """Check if a restart is allowed for this consumer within the rate window. + + Returns True if restart is allowed. + """ + async with self._db_pool.acquire() as conn: + row = await conn.fetchrow( + """ + SELECT restart_count_30min, restart_window_start + FROM consumer_restart_state + WHERE consumer_id = $1 AND consumer_group = $2 AND topic = $3 + """, + event.consumer_identity, + event.consumer_group, + event.topic, + ) + if row is None: + # No prior restarts — allowed + return True + + window_start = row["restart_window_start"] + count = row["restart_count_30min"] + window_cutoff = datetime.now(UTC) - timedelta( + minutes=_RESTART_WINDOW_MINUTES + ) + + if window_start < window_cutoff: + # Window expired — reset + return True + + return count < _MAX_RESTARTS_PER_WINDOW + + async def _emit_restart_command(self, event: ModelConsumerHealthEvent) -> None: + """Emit a restart command to the consumer restart topic.""" + if self._producer is None: + logger.warning("Cannot emit restart command: no producer available") + return + + command = ModelConsumerRestartCommand( + consumer_identity=event.consumer_identity, + consumer_group=event.consumer_group, + topic=event.topic, + reason=f"Graduated triage: {event.event_type.value} occurred {_RESTART_THRESHOLD}+ times", + fingerprint=event.fingerprint, + correlation_id=event.correlation_id, + ) + + try: + payload = json.dumps(command.model_dump(mode="json")).encode("utf-8") + await self._producer.send(TOPIC_CONSUMER_RESTART_CMD, value=payload) + logger.info( + "Emitted restart command for consumer %s", + event.consumer_identity, + ) + + # Update restart state + async with self._db_pool.acquire() as conn: + await conn.execute( + """ + INSERT INTO consumer_restart_state ( + consumer_id, consumer_group, topic, + last_restart_at, restart_count_30min, restart_window_start + ) VALUES ($1, $2, $3, NOW(), 1, NOW()) + ON CONFLICT (consumer_id, consumer_group, topic) + DO UPDATE SET + last_restart_at = NOW(), + restart_count_30min = CASE + WHEN consumer_restart_state.restart_window_start < NOW() - INTERVAL '30 minutes' + THEN 1 + ELSE consumer_restart_state.restart_count_30min + 1 + END, + restart_window_start = CASE + WHEN consumer_restart_state.restart_window_start < NOW() - INTERVAL '30 minutes' + THEN NOW() + ELSE consumer_restart_state.restart_window_start + END, + updated_at = NOW() + """, + event.consumer_identity, + event.consumer_group, + event.topic, + ) + except Exception: # noqa: BLE001 - best-effort restart command + logger.warning( + "Failed to emit restart command for %s", + event.consumer_identity, + exc_info=True, + ) + + async def _send_slack_notification( + self, event: ModelConsumerHealthEvent, *, repeated: bool + ) -> None: + """Send a Slack notification for the health event.""" + if self._slack_handler is None: + logger.debug("Slack handler not configured, skipping notification") + return + + try: + prefix = "REPEATED" if repeated else "WARNING" + message = ( + f"[{prefix}] Consumer health: {event.event_type.value} " + f"on {event.topic} ({event.consumer_identity})" + ) + if event.error_message: + message += f"\nError: {event.error_message}" + await self._slack_handler(message) + except Exception: # noqa: BLE001 - best-effort notification + logger.debug("Failed to send Slack notification", exc_info=True) + + async def _create_linear_ticket( + self, event: ModelConsumerHealthEvent, occurrence_count: int + ) -> None: + """Create a Linear ticket for an unresolvable consumer health issue.""" + if self._linear_handler is None: + logger.debug("Linear handler not configured, skipping ticket creation") + return + + try: + title = ( + f"Consumer health: {event.event_type.value} on {event.topic} " + f"({occurrence_count} occurrences)" + ) + description = ( + f"Consumer: {event.consumer_identity}\n" + f"Group: {event.consumer_group}\n" + f"Topic: {event.topic}\n" + f"Event: {event.event_type.value}\n" + f"Severity: {event.severity.value}\n" + f"Occurrences: {occurrence_count}\n" + f"Error: {event.error_message}\n" + f"Fingerprint: {event.fingerprint}\n" + ) + await self._linear_handler(title=title, description=description) + except Exception: # noqa: BLE001 - best-effort ticket creation + logger.debug("Failed to create Linear ticket", exc_info=True) + + +__all__ = ["HandlerConsumerHealthTriage", "ModelTriageResult"] diff --git a/src/omnibase_infra/nodes/node_consumer_health_triage_effect/node.py b/src/omnibase_infra/nodes/node_consumer_health_triage_effect/node.py new file mode 100644 index 0000000000..c934e47cf5 --- /dev/null +++ b/src/omnibase_infra/nodes/node_consumer_health_triage_effect/node.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Node Consumer Health Triage Effect — graduated response for consumer health events. + +Consumes onex.evt.omnibase-infra.consumer-health.v1 events and applies a +graduated triage response: + 1st occurrence -> Slack WARNING + 2nd occurrence -> Slack REPEATED + 3rd in 30 min -> Restart command (gated by ENABLE_CONSUMER_AUTO_RESTART) + Restart failure -> Linear ticket + +Gated by ENABLE_CONSUMER_HEALTH_TRIAGE (default off). + +Architecture: + onex.evt.omnibase-infra.consumer-health.v1 (Kafka) + -> NodeConsumerHealthTriageEffect (this declarative shell) + -> HandlerConsumerHealthTriage + -> PostgreSQL consumer_health_triage + consumer_restart_state tables + -> Slack webhook + Linear API (escalation) + -> onex.cmd.omnibase-infra.consumer-restart.v1 (restart commands) + +Related Tickets: + - OMN-5520: Create NodeConsumerHealthTriageEffect + - OMN-5529: Runtime Health Event Pipeline (epic) +""" + +from __future__ import annotations + +from omnibase_core.models.container import ModelONEXContainer +from omnibase_core.nodes.node_effect import NodeEffect + + +class NodeConsumerHealthTriageEffect(NodeEffect): + """Declarative effect node for consumer health triage with graduated response. + + This effect node is a lightweight shell that defines the I/O contract + for consumer health triage operations. All routing and execution + logic is driven by contract.yaml — this class contains NO custom + routing code. + + Supported Operations (defined in contract.yaml handler_routing): + - triage_health_event: Apply graduated triage to a consumer health event + + Dependency Injection: + Callers must instantiate HandlerConsumerHealthTriage with + db_pool, slack_handler, and optional linear_handler, then pass + via the container. This node does not retain handler instances + as attributes. + + Example: + ```python + from omnibase_core.models.container import ModelONEXContainer + from omnibase_infra.nodes.node_consumer_health_triage_effect import ( + NodeConsumerHealthTriageEffect, + ) + from omnibase_infra.nodes.node_consumer_health_triage_effect.handlers import ( + HandlerConsumerHealthTriage, + ) + + container = ModelONEXContainer() + effect = NodeConsumerHealthTriageEffect(container) + + handler = HandlerConsumerHealthTriage( + db_pool=pool, + event_bus=event_bus, + ) + result = await handler.handle(health_event) + ``` + """ + + def __init__(self, container: ModelONEXContainer) -> None: + super().__init__(container) + + # Pure declarative shell — all behaviour defined in contract.yaml + + +__all__ = ["NodeConsumerHealthTriageEffect"] diff --git a/src/omnibase_infra/nodes/node_runtime_error_triage_effect/__init__.py b/src/omnibase_infra/nodes/node_runtime_error_triage_effect/__init__.py new file mode 100644 index 0000000000..7c4f3e2daf --- /dev/null +++ b/src/omnibase_infra/nodes/node_runtime_error_triage_effect/__init__.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""NodeRuntimeErrorTriageEffect — first-match-wins triage for runtime error events.""" + +from omnibase_infra.nodes.node_runtime_error_triage_effect.node import ( + NodeRuntimeErrorTriageEffect, +) + +__all__ = ["NodeRuntimeErrorTriageEffect"] diff --git a/src/omnibase_infra/nodes/node_runtime_error_triage_effect/contract.yaml b/src/omnibase_infra/nodes/node_runtime_error_triage_effect/contract.yaml new file mode 100644 index 0000000000..a48f82eacf --- /dev/null +++ b/src/omnibase_infra/nodes/node_runtime_error_triage_effect/contract.yaml @@ -0,0 +1,128 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# Copyright (c) 2025 OmniNode Team +# +# ONEX Node Contract +# Node: NodeRuntimeErrorTriageEffect +# +# Consumes onex.evt.omnibase-infra.runtime-error.v1 events and applies +# first-match-wins triage with cross-layer correlation to consumer health. +# +# Related Tickets: +# - OMN-5522: Create NodeRuntimeErrorTriageEffect +# - OMN-5529: Runtime Health Event Pipeline (epic) +# Contract identifiers +name: "node_runtime_error_triage_effect" +contract_name: "node_runtime_error_triage_effect" +node_name: "node_runtime_error_triage_effect" +contract_version: + major: 1 + minor: 0 + patch: 0 +node_version: + major: 1 + minor: 0 + patch: 0 +# Node type +node_type: "EFFECT_GENERIC" +# Description +description: > + Effect node that consumes runtime error events from onex.evt.omnibase-infra.runtime-error.v1 and applies first-match-wins triage. Default rules cover aiokafka, asyncpg, and aiohttp patterns. Includes cross-layer correlation with Layer 1 consumer health incidents. Uses runtime_error_triage PostgreSQL table. + +# Event bus subscription +event_bus: + version: + major: 1 + minor: 0 + patch: 0 + subscribe_topics: + - "onex.evt.omnibase-infra.runtime-error.v1" + publish_topics: [] +# I/O models +input_model: + name: "ModelRuntimeErrorEvent" + module: "omnibase_infra.models.health.model_runtime_error_event" + description: "Structured runtime error event from Kafka." +output_model: + name: "ModelRuntimeErrorTriageResult" + module: "omnibase_infra.nodes.node_runtime_error_triage_effect.handlers.handler_runtime_error_triage" + description: "Result of the first-match-wins triage." +# Handler routing +handler_routing: + routing_strategy: "operation_match" + handlers: + - operation: "triage_runtime_error" + handler: + name: "HandlerRuntimeErrorTriage" + module: "omnibase_infra.nodes.node_runtime_error_triage_effect.handlers.handler_runtime_error_triage" + description: "First-match-wins triage with cross-layer correlation" + output_fields: + - fingerprint + - action + - matched_rule + - incident_state + - occurrence_count + - correlated_consumer_fingerprint +# Error handling +error_handling: + retry_policy: + max_retries: 3 + initial_delay_ms: 1000 + max_delay_ms: 8000 + exponential_base: 2 + retry_on: + - "DB_POOL_TIMEOUT" + error_types: + - name: "DB_POOL_UNAVAILABLE" + description: "asyncpg pool not available" + recoverable: false + retry_strategy: "none" + - name: "DB_POOL_TIMEOUT" + description: "asyncpg pool query timed out" + recoverable: true + retry_strategy: "exponential_backoff" +# IO operations +io_operations: + - operation: "triage_runtime_error" + description: "Apply first-match-wins triage to runtime error event" + input_fields: + - fingerprint + - logger_family + - error_category + - severity + - raw_message + - message_template + output_fields: + - fingerprint + - action + - rule_name + - incident_state + - occurrence_count + - correlated_consumer_fingerprint +# Dependencies +dependencies: + - name: "db_pool" + type: "service" + description: "asyncpg pool for runtime_error_triage table" + required: true + - name: "slack_handler" + type: "service" + description: "Slack webhook handler for notifications" + required: false + - name: "linear_handler" + type: "service" + description: "Linear API handler for ticket creation" + required: false +# Metadata +metadata: + author: "OmniNode Team" + license: "MIT" + created: "2026-03-19" + updated: "2026-03-19" + tags: + - effect + - health + - triage + - runtime-error + - cross-layer + - rule-engine diff --git a/src/omnibase_infra/nodes/node_runtime_error_triage_effect/handlers/__init__.py b/src/omnibase_infra/nodes/node_runtime_error_triage_effect/handlers/__init__.py new file mode 100644 index 0000000000..5e046a3bba --- /dev/null +++ b/src/omnibase_infra/nodes/node_runtime_error_triage_effect/handlers/__init__.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Handlers for NodeRuntimeErrorTriageEffect.""" + +from omnibase_infra.nodes.node_runtime_error_triage_effect.handlers.handler_runtime_error_triage import ( + HandlerRuntimeErrorTriage, +) + +__all__ = ["HandlerRuntimeErrorTriage"] diff --git a/src/omnibase_infra/nodes/node_runtime_error_triage_effect/handlers/handler_runtime_error_triage.py b/src/omnibase_infra/nodes/node_runtime_error_triage_effect/handlers/handler_runtime_error_triage.py new file mode 100644 index 0000000000..2d1201d301 --- /dev/null +++ b/src/omnibase_infra/nodes/node_runtime_error_triage_effect/handlers/handler_runtime_error_triage.py @@ -0,0 +1,375 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Handler for runtime error triage with first-match-wins rule engine. + +Applies a configurable set of triage rules to runtime error events. +Rules are evaluated in priority order; first match determines the +triage action (suppress, alert, ticket). + +Includes cross-layer correlation with Layer 1 consumer health incidents: +when a runtime error's logger_family matches a known Kafka consumer pattern, +the handler queries consumer_health_triage for active incidents with the +same consumer identity. + +Related Tickets: + - OMN-5522: Create NodeRuntimeErrorTriageEffect + - OMN-5529: Runtime Health Event Pipeline (epic) +""" + +from __future__ import annotations + +import logging +from collections.abc import Awaitable, Callable +from datetime import UTC, datetime +from typing import TYPE_CHECKING, cast + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_infra.enums import EnumHandlerType, EnumHandlerTypeCategory +from omnibase_infra.models.health.enum_runtime_error_category import ( + EnumRuntimeErrorCategory, +) +from omnibase_infra.models.health.model_runtime_error_event import ( + ModelRuntimeErrorEvent, +) +from omnibase_infra.nodes.node_runtime_error_triage_effect.models.model_triage_rule import ( + DEFAULT_TRIAGE_RULES, + ModelTriageRule, +) + +if TYPE_CHECKING: + from asyncpg import Pool + +logger = logging.getLogger(__name__) + +# Categories that may correlate with Layer 1 consumer health events +_KAFKA_CATEGORIES = { + EnumRuntimeErrorCategory.KAFKA_CONSUMER, + EnumRuntimeErrorCategory.KAFKA_PRODUCER, +} + + +class ModelRuntimeErrorTriageResult(BaseModel): + """Result of a runtime error triage operation.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + fingerprint: str = Field(..., description="Event fingerprint that was triaged.") + action: str = Field( + ..., + description="Action taken: 'suppress', 'alert', 'ticket'.", + ) + matched_rule: str = Field( + ..., description="Identifier of the triage rule that matched." + ) + incident_state: str = Field(..., description="New incident state after triage.") + occurrence_count: int = Field( + ..., description="Total occurrences for this fingerprint." + ) + correlated_consumer_fingerprint: str | None = Field( + default=None, + description="Fingerprint of correlated Layer 1 consumer health incident.", + ) + + +class HandlerRuntimeErrorTriage: + """Handler that applies first-match-wins triage to runtime error events. + + Dependencies (injected via constructor): + - db_pool: asyncpg connection pool for PostgreSQL + - rules: Optional list of triage rules (defaults to DEFAULT_TRIAGE_RULES) + - slack_handler: Optional callable for Slack notifications + - linear_handler: Optional callable for Linear ticket creation + """ + + def __init__( + self, + db_pool: Pool, + *, + rules: list[ModelTriageRule] | None = None, + slack_handler: Callable[[str], Awaitable[None]] | None = None, + linear_handler: Callable[..., Awaitable[None]] | None = None, + ) -> None: + """Initialize the triage handler. + + Args: + db_pool: asyncpg connection pool. + rules: Triage rules in priority order. Defaults to DEFAULT_TRIAGE_RULES. + slack_handler: Async callable accepting a message string. + linear_handler: Async callable accepting title and description kwargs. + """ + self._db_pool = db_pool + self._rules = sorted(rules or DEFAULT_TRIAGE_RULES, key=lambda r: r.priority) + self._slack_handler = slack_handler + self._linear_handler = linear_handler + + @property + def handler_type(self) -> EnumHandlerType: + """Architectural role: infrastructure handler for error triage.""" + return EnumHandlerType.INFRA_HANDLER + + @property + def handler_category(self) -> EnumHandlerTypeCategory: + """Behavioral classification: side-effecting triage operation.""" + return EnumHandlerTypeCategory.EFFECT + + async def handle( + self, event: ModelRuntimeErrorEvent + ) -> ModelRuntimeErrorTriageResult: + """Apply first-match-wins triage to a runtime error event. + + Args: + event: The runtime error event to triage. + + Returns: + ModelRuntimeErrorTriageResult with action taken and metadata. + """ + # Find matching rule + matched_rule = self._find_matching_rule(event) + if matched_rule is None: + # Should not happen with catch-all rule, but handle gracefully + matched_rule = ModelTriageRule(name="fallback", action="alert") + + # Cross-layer correlation for Kafka-related errors + correlated_fingerprint = None + if event.error_category in _KAFKA_CATEGORIES: + correlated_fingerprint = await self._correlate_with_layer1(event) + + # Upsert incident in runtime_error_triage table + incident = await self._upsert_incident( + event, matched_rule, correlated_fingerprint + ) + occurrence_count = cast("int", incident["occurrence_count"]) + incident_state = cast("str", incident["incident_state"]) + + # Execute the triage action + if matched_rule.action == "suppress": + # Check if suppression window is active + if incident_state == "suppressed": + return ModelRuntimeErrorTriageResult( + fingerprint=event.fingerprint, + action="suppress", + matched_rule=matched_rule.name, + incident_state=incident_state, + occurrence_count=occurrence_count, + correlated_consumer_fingerprint=correlated_fingerprint, + ) + # First time or suppression expired — suppress and alert once + await self._update_incident_state(event.fingerprint, "suppressed") + await self._send_slack_notification( + event, matched_rule, correlated_fingerprint + ) + return ModelRuntimeErrorTriageResult( + fingerprint=event.fingerprint, + action="suppress", + matched_rule=matched_rule.name, + incident_state="suppressed", + occurrence_count=occurrence_count, + correlated_consumer_fingerprint=correlated_fingerprint, + ) + + elif matched_rule.action == "ticket": + await self._create_linear_ticket( + event, occurrence_count, correlated_fingerprint + ) + await self._update_incident_state(event.fingerprint, "ticketed") + return ModelRuntimeErrorTriageResult( + fingerprint=event.fingerprint, + action="ticket", + matched_rule=matched_rule.name, + incident_state="ticketed", + occurrence_count=occurrence_count, + correlated_consumer_fingerprint=correlated_fingerprint, + ) + + else: + # Default: alert via Slack + await self._send_slack_notification( + event, matched_rule, correlated_fingerprint + ) + return ModelRuntimeErrorTriageResult( + fingerprint=event.fingerprint, + action="alert", + matched_rule=matched_rule.name, + incident_state=incident_state, + occurrence_count=occurrence_count, + correlated_consumer_fingerprint=correlated_fingerprint, + ) + + def _find_matching_rule( + self, event: ModelRuntimeErrorEvent + ) -> ModelTriageRule | None: + """Find the first matching triage rule for the event.""" + for rule in self._rules: + if rule.matches( + event.logger_family, event.error_category, event.raw_message + ): + return rule + return None + + async def _correlate_with_layer1(self, event: ModelRuntimeErrorEvent) -> str | None: + """Check for active Layer 1 consumer health incidents that may correlate. + + Queries consumer_health_triage for open incidents where the logger + family or error message suggests the same consumer is affected. + + Returns the correlated fingerprint or None. + """ + try: + async with self._db_pool.acquire() as conn: + row = await conn.fetchrow( + """ + SELECT fingerprint + FROM consumer_health_triage + WHERE incident_state IN ('open', 'acknowledged', 'restart_pending') + AND last_seen_at > NOW() - INTERVAL '30 minutes' + ORDER BY last_seen_at DESC + LIMIT 1 + """, + ) + return row["fingerprint"] if row else None + except Exception: # noqa: BLE001 - cross-layer correlation is best-effort + logger.debug("Failed to correlate with Layer 1", exc_info=True) + return None + + async def _upsert_incident( + self, + event: ModelRuntimeErrorEvent, + rule: ModelTriageRule, + correlated_fingerprint: str | None, + ) -> dict[str, object]: + """Upsert incident row in runtime_error_triage table.""" + async with self._db_pool.acquire() as conn: + row = await conn.fetchrow( + """ + INSERT INTO runtime_error_triage ( + fingerprint, logger_name, error_category, severity, + incident_state, occurrence_count, message_template, + first_seen_at, last_seen_at, + correlated_consumer_fingerprint, service_name, + hostname, correlation_id + ) VALUES ($1, $2, $3, $4, $5, 1, $6, NOW(), NOW(), $7, $8, $9, $10) + ON CONFLICT (fingerprint) WHERE incident_state IN ('open', 'suppressed') + DO UPDATE SET + occurrence_count = runtime_error_triage.occurrence_count + 1, + last_seen_at = NOW(), + severity = EXCLUDED.severity, + correlated_consumer_fingerprint = COALESCE( + EXCLUDED.correlated_consumer_fingerprint, + runtime_error_triage.correlated_consumer_fingerprint + ), + correlation_id = EXCLUDED.correlation_id + RETURNING occurrence_count, incident_state + """, + event.fingerprint, + event.logger_family, + event.error_category.value, + event.severity.value, + "open", + event.message_template, + correlated_fingerprint, + event.service_label, + event.hostname, + str(event.correlation_id), + ) + if row is None: + # Existing incident is ticketed/resolved — create new + row = await conn.fetchrow( + """ + INSERT INTO runtime_error_triage ( + fingerprint, logger_name, error_category, severity, + incident_state, occurrence_count, message_template, + first_seen_at, last_seen_at, + correlated_consumer_fingerprint, service_name, + hostname, correlation_id + ) VALUES ($1, $2, $3, $4, 'open', 1, $5, NOW(), NOW(), $6, $7, $8, $9) + RETURNING occurrence_count, incident_state + """, + event.fingerprint, + event.logger_family, + event.error_category.value, + event.severity.value, + event.message_template, + correlated_fingerprint, + event.service_label, + event.hostname, + str(event.correlation_id), + ) + return ( + dict(row) if row else {"occurrence_count": 1, "incident_state": "open"} + ) + + async def _update_incident_state(self, fingerprint: str, state: str) -> None: + """Update the incident state for a fingerprint.""" + async with self._db_pool.acquire() as conn: + await conn.execute( + """ + UPDATE runtime_error_triage + SET incident_state = $1, + escalated_at = CASE WHEN $1 IN ('ticketed') THEN NOW() ELSE escalated_at END + WHERE fingerprint = $2 AND incident_state NOT IN ('resolved') + """, + state, + fingerprint, + ) + + async def _send_slack_notification( + self, + event: ModelRuntimeErrorEvent, + rule: ModelTriageRule, + correlated_fingerprint: str | None, + ) -> None: + """Send a Slack notification for the runtime error event.""" + if self._slack_handler is None: + logger.debug("Slack handler not configured, skipping notification") + return + + try: + message = ( + f"[{event.severity.value.upper()}] Runtime error: {event.error_category.value}\n" + f"Logger: {event.logger_family}\n" + f"Message: {event.raw_message[:200]}\n" + f"Rule: {rule.name}" + ) + if correlated_fingerprint: + message += f"\nCorrelated with consumer health incident: {correlated_fingerprint}" + await self._slack_handler(message) + except Exception: # noqa: BLE001 - best-effort notification + logger.debug("Failed to send Slack notification", exc_info=True) + + async def _create_linear_ticket( + self, + event: ModelRuntimeErrorEvent, + occurrence_count: int, + correlated_fingerprint: str | None, + ) -> None: + """Create a Linear ticket for the runtime error.""" + if self._linear_handler is None: + logger.debug("Linear handler not configured, skipping ticket creation") + return + + try: + title = ( + f"Runtime error: {event.error_category.value} in {event.logger_family} " + f"({occurrence_count} occurrences)" + ) + description = ( + f"Logger: {event.logger_family}\n" + f"Category: {event.error_category.value}\n" + f"Severity: {event.severity.value}\n" + f"Message: {event.raw_message[:500]}\n" + f"Fingerprint: {event.fingerprint}\n" + f"Occurrences: {occurrence_count}\n" + ) + if correlated_fingerprint: + description += ( + f"\nCorrelated consumer health incident: {correlated_fingerprint}" + ) + if event.stack_trace: + description += f"\n\nStack trace:\n```\n{event.stack_trace[:1000]}\n```" + await self._linear_handler(title=title, description=description) + except Exception: # noqa: BLE001 - best-effort ticket creation + logger.debug("Failed to create Linear ticket", exc_info=True) + + +__all__ = ["HandlerRuntimeErrorTriage", "ModelRuntimeErrorTriageResult"] diff --git a/src/omnibase_infra/nodes/node_runtime_error_triage_effect/models/__init__.py b/src/omnibase_infra/nodes/node_runtime_error_triage_effect/models/__init__.py new file mode 100644 index 0000000000..e572e4d225 --- /dev/null +++ b/src/omnibase_infra/nodes/node_runtime_error_triage_effect/models/__init__.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Models for NodeRuntimeErrorTriageEffect.""" + +from omnibase_infra.nodes.node_runtime_error_triage_effect.models.model_triage_rule import ( + ModelTriageRule, +) + +__all__ = ["ModelTriageRule"] diff --git a/src/omnibase_infra/nodes/node_runtime_error_triage_effect/models/model_triage_rule.py b/src/omnibase_infra/nodes/node_runtime_error_triage_effect/models/model_triage_rule.py new file mode 100644 index 0000000000..ccabba1d0e --- /dev/null +++ b/src/omnibase_infra/nodes/node_runtime_error_triage_effect/models/model_triage_rule.py @@ -0,0 +1,151 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Triage rule model for first-match-wins runtime error triage. + +Each rule matches on a combination of logger_family prefix, error_category, +and optional message pattern. Rules are evaluated in order; first match wins. + +Related Tickets: + - OMN-5522: Create NodeRuntimeErrorTriageEffect + - OMN-5529: Runtime Health Event Pipeline (epic) +""" + +from __future__ import annotations + +import re +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_infra.models.health.enum_runtime_error_category import ( + EnumRuntimeErrorCategory, +) + + +class ModelTriageRule(BaseModel): + """A single triage rule for the first-match-wins rule engine. + + Rules are evaluated in priority order. First match determines the + triage action for the runtime error event. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + name: str = Field(..., description="Human-readable rule name.") + priority: int = Field( + default=100, + description="Lower number = higher priority. Rules are sorted by priority.", + ) + + # Match conditions (all must match for rule to fire) + logger_prefix: str = Field( + default="", + description="Logger family prefix to match (e.g., 'aiokafka'). Empty = match all.", + ) + error_category: EnumRuntimeErrorCategory | None = Field( + default=None, + description="Error category to match. None = match all.", + ) + message_pattern: str = Field( + default="", + description="Regex pattern to match against raw_message. Empty = match all.", + ) + + # Action + action: Literal["suppress", "alert", "ticket"] = Field( + default="alert", + description="Action to take: suppress (ignore), alert (Slack), ticket (Linear).", + ) + suppress_duration_minutes: int = Field( + default=60, + description="For 'suppress' action: suppress further alerts for this many minutes.", + ) + + def matches( + self, + logger_family: str, + error_category: EnumRuntimeErrorCategory, + raw_message: str, + ) -> bool: + """Check if this rule matches the given error event attributes. + + Args: + logger_family: Logger name from the error event. + error_category: Error category from the event. + raw_message: Raw error message. + + Returns: + True if all conditions match. + """ + # Check logger prefix + if self.logger_prefix and not logger_family.startswith(self.logger_prefix): + return False + + # Check error category + if self.error_category is not None and self.error_category != error_category: + return False + + # Check message pattern + if self.message_pattern: + try: + if not re.search(self.message_pattern, raw_message, re.IGNORECASE): + return False + except re.error: + # Invalid regex — treat as non-match + return False + + return True + + +# Default triage rules for common runtime errors +DEFAULT_TRIAGE_RULES: list[ModelTriageRule] = [ + ModelTriageRule( + name="aiokafka_heartbeat_failure", + priority=10, + logger_prefix="aiokafka", + error_category=EnumRuntimeErrorCategory.KAFKA_CONSUMER, + message_pattern=r"heartbeat|session.*timeout|coordinator.*dead", + action="alert", + ), + ModelTriageRule( + name="aiokafka_rebalance", + priority=20, + logger_prefix="aiokafka", + error_category=EnumRuntimeErrorCategory.KAFKA_CONSUMER, + message_pattern=r"rebalance|partition.*revoked", + action="suppress", + suppress_duration_minutes=5, + ), + ModelTriageRule( + name="asyncpg_connection_error", + priority=10, + logger_prefix="asyncpg", + error_category=EnumRuntimeErrorCategory.DATABASE, + message_pattern=r"connection.*refused|connection.*reset|cannot connect", + action="alert", + ), + ModelTriageRule( + name="asyncpg_query_error", + priority=30, + logger_prefix="asyncpg", + error_category=EnumRuntimeErrorCategory.DATABASE, + message_pattern=r"relation.*does not exist|column.*does not exist", + action="ticket", + ), + ModelTriageRule( + name="aiohttp_connection_error", + priority=20, + logger_prefix="aiohttp", + error_category=EnumRuntimeErrorCategory.HTTP_CLIENT, + message_pattern=r"connection.*refused|timeout|server disconnected", + action="alert", + ), + ModelTriageRule( + name="catch_all", + priority=999, + action="alert", + ), +] + + +__all__ = ["DEFAULT_TRIAGE_RULES", "ModelTriageRule"] diff --git a/src/omnibase_infra/nodes/node_runtime_error_triage_effect/node.py b/src/omnibase_infra/nodes/node_runtime_error_triage_effect/node.py new file mode 100644 index 0000000000..dbf387dcd4 --- /dev/null +++ b/src/omnibase_infra/nodes/node_runtime_error_triage_effect/node.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Node Runtime Error Triage Effect — first-match-wins triage for runtime errors. + +Consumes onex.evt.omnibase-infra.runtime-error.v1 events and applies a +first-match-wins triage rule engine with cross-layer correlation to +Layer 1 consumer health incidents. + +Default rules cover aiokafka, asyncpg, and aiohttp error patterns. + +Architecture: + onex.evt.omnibase-infra.runtime-error.v1 (Kafka) + -> NodeRuntimeErrorTriageEffect (this declarative shell) + -> HandlerRuntimeErrorTriage + -> PostgreSQL runtime_error_triage table + -> Cross-layer correlation with consumer_health_triage + -> Slack webhook + Linear API (escalation) + +Related Tickets: + - OMN-5522: Create NodeRuntimeErrorTriageEffect + - OMN-5529: Runtime Health Event Pipeline (epic) +""" + +from __future__ import annotations + +from omnibase_core.models.container import ModelONEXContainer +from omnibase_core.nodes.node_effect import NodeEffect + + +class NodeRuntimeErrorTriageEffect(NodeEffect): + """Declarative effect node for runtime error triage. + + This effect node is a lightweight shell that defines the I/O contract + for runtime error triage operations. All routing and execution logic + is driven by contract.yaml — this class contains NO custom routing code. + + Supported Operations (defined in contract.yaml handler_routing): + - triage_runtime_error: Apply first-match-wins triage with cross-layer correlation + + Example: + ```python + from omnibase_core.models.container import ModelONEXContainer + from omnibase_infra.nodes.node_runtime_error_triage_effect import ( + NodeRuntimeErrorTriageEffect, + ) + from omnibase_infra.nodes.node_runtime_error_triage_effect.handlers import ( + HandlerRuntimeErrorTriage, + ) + + container = ModelONEXContainer() + effect = NodeRuntimeErrorTriageEffect(container) + + handler = HandlerRuntimeErrorTriage(db_pool=pool) + result = await handler.handle(runtime_error_event) + ``` + """ + + def __init__(self, container: ModelONEXContainer) -> None: + super().__init__(container) + + # Pure declarative shell — all behaviour defined in contract.yaml + + +__all__ = ["NodeRuntimeErrorTriageEffect"] diff --git a/src/omnibase_infra/observability/runtime_log_event_bridge.py b/src/omnibase_infra/observability/runtime_log_event_bridge.py new file mode 100644 index 0000000000..827e5ba20f --- /dev/null +++ b/src/omnibase_infra/observability/runtime_log_event_bridge.py @@ -0,0 +1,339 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""RuntimeLogEventBridge - Python logging.Handler that emits structured Kafka events. + +Best-effort capture of ERROR/WARNING log records from allowlisted loggers. +Decoupled via asyncio.Queue to avoid blocking the logging thread. + +Features: + - 3-layer circular logging prevention (bridge logger excluded) + - Queue-based decoupling (sync logging -> async emission) + - Allowlist-only attachment (NOT root logger) + - Rate limited: max 5 events per fingerprint per 5 minutes + - Self-metrics: events_emitted, events_dropped, events_rate_limited + +Feature flag: ENABLE_RUNTIME_LOG_BRIDGE (default off) + +Related Tickets: + - OMN-5521: Create RuntimeLogEventBridge + - OMN-5529: Runtime Health Event Pipeline (epic) +""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import logging +import os +import re +import time +from typing import TYPE_CHECKING + +from omnibase_infra.event_bus.topic_constants import TOPIC_RUNTIME_ERROR +from omnibase_infra.models.health.enum_runtime_error_category import ( + EnumRuntimeErrorCategory, +) +from omnibase_infra.models.health.enum_runtime_error_severity import ( + EnumRuntimeErrorSeverity, +) +from omnibase_infra.models.health.model_runtime_error_event import ( + ModelRuntimeErrorEvent, +) + +if TYPE_CHECKING: + from aiokafka import AIOKafkaProducer + +# Bridge's own logger -- must never be captured by the bridge itself +_bridge_logger = logging.getLogger(f"{__name__}._bridge") + +# Rate limit: max 5 events per fingerprint per 5 minutes +_RATE_LIMIT_WINDOW_SECONDS = 300.0 +_RATE_LIMIT_MAX_PER_WINDOW = 5 + +# Queue size for decoupling sync logging from async emission +_DEFAULT_QUEUE_SIZE = 1000 + +# Pattern to templatize log messages (replace numbers, UUIDs, IPs, etc.) +_TEMPLATIZE_PATTERNS = [ + ( + re.compile( + r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", re.I + ), + "{}", + ), + (re.compile(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b"), "{}"), + (re.compile(r"\b\d+\.\d+\b"), "{}"), + (re.compile(r"\b\d+\b"), "{}"), +] + +# Logger name prefix -> error category mapping +_CATEGORY_MAP: dict[str, EnumRuntimeErrorCategory] = { + "aiokafka.consumer": EnumRuntimeErrorCategory.KAFKA_CONSUMER, + "aiokafka.producer": EnumRuntimeErrorCategory.KAFKA_PRODUCER, + "aiokafka": EnumRuntimeErrorCategory.KAFKA_CONSUMER, + "asyncpg": EnumRuntimeErrorCategory.DATABASE, + "aiohttp.client": EnumRuntimeErrorCategory.HTTP_CLIENT, + "aiohttp.server": EnumRuntimeErrorCategory.HTTP_SERVER, + "aiohttp": EnumRuntimeErrorCategory.HTTP_CLIENT, + "uvicorn": EnumRuntimeErrorCategory.HTTP_SERVER, +} + + +def _templatize_message(msg: str) -> str: + """Replace variable parts of a log message with placeholders. + + Args: + msg: Raw log message. + + Returns: + Templatized message suitable for fingerprinting. + """ + result = msg + for pattern, replacement in _TEMPLATIZE_PATTERNS: + result = pattern.sub(replacement, result) + return result + + +def _categorize_logger(logger_family: str) -> EnumRuntimeErrorCategory: + """Map a logger name to an error category. + + Args: + logger_family: Python logger name. + + Returns: + Best-match error category. + """ + for prefix, category in _CATEGORY_MAP.items(): + if logger_family.startswith(prefix): + return category + return EnumRuntimeErrorCategory.UNKNOWN + + +def _log_level_to_severity(levelno: int) -> EnumRuntimeErrorSeverity: + """Map Python log level to error severity. + + Args: + levelno: Python logging level number. + + Returns: + Corresponding severity enum. + """ + if levelno >= logging.CRITICAL: + return EnumRuntimeErrorSeverity.CRITICAL + if levelno >= logging.ERROR: + return EnumRuntimeErrorSeverity.ERROR + return EnumRuntimeErrorSeverity.WARNING + + +class RuntimeLogEventBridge(logging.Handler): + """Python logging.Handler that captures log records as structured Kafka events. + + Attach to specific named loggers only (NOT root logger). + Uses an asyncio.Queue for sync-to-async decoupling. + + Attributes: + events_emitted: Count of successfully emitted events. + events_dropped: Count of events that failed to emit or were queue-full. + events_rate_limited: Count of events suppressed by rate limiter. + """ + + def __init__( + self, + producer: AIOKafkaProducer, + *, + topic: str = TOPIC_RUNTIME_ERROR, + queue_size: int = _DEFAULT_QUEUE_SIZE, + hostname: str = "", + service_label: str = "", + ) -> None: + """Initialize the bridge. + + Args: + producer: An already-started AIOKafkaProducer. + topic: Topic to emit to. + queue_size: Max queue size for decoupling. + hostname: Machine hostname. + service_label: Service display label. + """ + super().__init__(level=logging.WARNING) + self._producer = producer + self._topic = topic + self._hostname = hostname + self._service_label = service_label + self._queue: asyncio.Queue[ModelRuntimeErrorEvent] = asyncio.Queue( + maxsize=queue_size + ) + self._rate_limit_cache: dict[str, list[float]] = {} + self._running = False + self._drain_task: asyncio.Task[None] | None = None + + # Self-metrics + self.events_emitted: int = 0 + self.events_dropped: int = 0 + self.events_rate_limited: int = 0 + + @staticmethod + def is_enabled() -> bool: + """Check if runtime log bridge is enabled via feature flag.""" + return os.environ.get( # ONEX_EXCLUDE: env + "ENABLE_RUNTIME_LOG_BRIDGE", "" + ).strip().lower() in { + "1", + "true", + "yes", + "on", + } + + def emit(self, record: logging.LogRecord) -> None: + """Handle a log record (called by Python logging framework). + + Converts to ModelRuntimeErrorEvent and enqueues for async emission. + Non-blocking -- drops if queue is full. + + Args: + record: The log record to process. + """ + if not self.is_enabled(): + return + + # Circular prevention: skip records from our own bridge logger + if record.name.startswith(__name__): + return + + try: + raw_message = self.format(record) if record.msg else str(record.msg) + message_template = _templatize_message(raw_message) + logger_family = record.name + error_category = _categorize_logger(logger_family) + severity = _log_level_to_severity(record.levelno) + + # Rate limiting + fingerprint = hashlib.sha256( + f"{logger_family}:{error_category}:{message_template}".encode() + ).hexdigest()[:16] + + if self._is_rate_limited(fingerprint): + self.events_rate_limited += 1 + return + + # Extract exception info + exception_type = "" + exception_message = "" + stack_trace = "" + if record.exc_info and record.exc_info[1]: + exc = record.exc_info[1] + exception_type = type(exc).__name__ + exception_message = str(exc)[:500] + if record.exc_text: + stack_trace = record.exc_text[:2000] + + event = ModelRuntimeErrorEvent.create( + logger_family=logger_family, + log_level=record.levelname, + message_template=message_template, + raw_message=raw_message[:2000], + error_category=error_category, + severity=severity, + exception_type=exception_type, + exception_message=exception_message, + stack_trace=stack_trace, + hostname=self._hostname, + service_label=self._service_label, + ) + + try: + self._queue.put_nowait(event) + except asyncio.QueueFull: + self.events_dropped += 1 + + except Exception: # noqa: BLE001 - must never raise from logging handler + self.events_dropped += 1 + + def _is_rate_limited(self, fingerprint: str) -> bool: + """Check if an event with this fingerprint is rate-limited.""" + now = time.monotonic() + timestamps = self._rate_limit_cache.get(fingerprint, []) + + # Remove expired entries + timestamps = [t for t in timestamps if now - t < _RATE_LIMIT_WINDOW_SECONDS] + + if len(timestamps) >= _RATE_LIMIT_MAX_PER_WINDOW: + return True + + timestamps.append(now) + self._rate_limit_cache[fingerprint] = timestamps + return False + + async def start(self) -> None: + """Start the async drain loop.""" + self._running = True + self._drain_task = asyncio.create_task(self._drain_loop()) + + async def stop(self) -> None: + """Stop the drain loop and flush remaining events.""" + self._running = False + if self._drain_task: + self._drain_task.cancel() + try: + await self._drain_task + except asyncio.CancelledError: + pass + + async def _drain_loop(self) -> None: + """Background task: drain queue and emit to Kafka.""" + while self._running: + try: + event = await asyncio.wait_for(self._queue.get(), timeout=1.0) + await self._emit_to_kafka(event) + except TimeoutError: + continue + except asyncio.CancelledError: + # Flush remaining + while not self._queue.empty(): + try: + event = self._queue.get_nowait() + await self._emit_to_kafka(event) + except asyncio.QueueEmpty: + break + raise + + async def _emit_to_kafka(self, event: ModelRuntimeErrorEvent) -> None: + """Emit a single event to Kafka.""" + try: + payload = json.dumps( + event.model_dump(mode="json"), + ).encode("utf-8") + await self._producer.send(self._topic, value=payload) + self.events_emitted += 1 + except Exception: # noqa: BLE001 - best-effort emission + self.events_dropped += 1 + _bridge_logger.debug( + "Failed to emit runtime error event (fingerprint=%s)", + event.fingerprint, + exc_info=True, + ) + + def attach_to_loggers(self, logger_names: list[str]) -> None: + """Attach this handler to the specified named loggers. + + Args: + logger_names: List of logger names to attach to. + """ + for name in logger_names: + target_logger = logging.getLogger(name) + target_logger.addHandler(self) + _bridge_logger.debug("Attached RuntimeLogEventBridge to logger: %s", name) + + def detach_from_loggers(self, logger_names: list[str]) -> None: + """Detach this handler from the specified named loggers. + + Args: + logger_names: List of logger names to detach from. + """ + for name in logger_names: + target_logger = logging.getLogger(name) + target_logger.removeHandler(self) + + +__all__ = ["RuntimeLogEventBridge"] diff --git a/src/omnibase_infra/runtime/service_kernel.py b/src/omnibase_infra/runtime/service_kernel.py index 0288594354..f2dd884d6a 100644 --- a/src/omnibase_infra/runtime/service_kernel.py +++ b/src/omnibase_infra/runtime/service_kernel.py @@ -95,6 +95,7 @@ from omnibase_infra.nodes.node_registration_orchestrator.plugin import ( PluginRegistration, ) +from omnibase_infra.observability.runtime_log_event_bridge import RuntimeLogEventBridge from omnibase_infra.runtime.handler_registry import RegistryProtocolBinding from omnibase_infra.runtime.models import ( ModelDomainPluginConfig, @@ -618,6 +619,7 @@ async def bootstrap() -> int: contract_unsub_deregistered: Callable[[], Awaitable[None]] | None = None contract_unsub_heartbeat: Callable[[], Awaitable[None]] | None = None plugin_config: ModelDomainPluginConfig | None = None + runtime_log_bridge: RuntimeLogEventBridge | None = None correlation_id = generate_correlation_id() bootstrap_start_time = time.time() @@ -959,6 +961,49 @@ async def bootstrap() -> int: exc_info=True, ) + # 3.6. Initialize RuntimeLogEventBridge if enabled (OMN-5525) + # Captures ERROR/WARNING log records from allowlisted loggers and emits + # them as structured Kafka events. Requires a dedicated producer. + if use_kafka and RuntimeLogEventBridge.is_enabled() and kafka_bootstrap_servers: + try: + from aiokafka import AIOKafkaProducer as _BridgeProducer + + _bridge_producer = _BridgeProducer( + bootstrap_servers=kafka_bootstrap_servers, + ) + await _bridge_producer.start() + + runtime_log_bridge = RuntimeLogEventBridge( + producer=_bridge_producer, + hostname=os.environ.get("HOSTNAME", ""), + service_label="onex-kernel", + ) + + # Parse allowlist from env or use defaults + allowlist_raw = os.environ.get( + "RUNTIME_LOG_BRIDGE_ALLOWLIST", + "aiokafka.consumer,asyncpg,aiohttp", + ) + allowlist = [ + name.strip() for name in allowlist_raw.split(",") if name.strip() + ] + runtime_log_bridge.attach_to_loggers(allowlist) + await runtime_log_bridge.start() + + logger.info( + "RuntimeLogEventBridge started (loggers=%s, correlation_id=%s)", + allowlist, + correlation_id, + ) + except Exception: # noqa: BLE001 — best-effort, never blocks startup + logger.warning( + "Failed to start RuntimeLogEventBridge, continuing without it " + "(correlation_id=%s)", + correlation_id, + exc_info=True, + ) + runtime_log_bridge = None + # 4. Create and wire container for dependency injection container_start_time = time.time() container = ModelONEXContainer() @@ -2027,6 +2072,33 @@ def windows_handler(signum: int, frame: object) -> None: contract_unsub_deregistered = None contract_unsub_heartbeat = None + # Stop RuntimeLogEventBridge (OMN-5525) + if runtime_log_bridge is not None: + try: + allowlist_raw = os.environ.get( + "RUNTIME_LOG_BRIDGE_ALLOWLIST", + "aiokafka.consumer,asyncpg,aiohttp", + ) + allowlist = [ + name.strip() for name in allowlist_raw.split(",") if name.strip() + ] + runtime_log_bridge.detach_from_loggers(allowlist) + await runtime_log_bridge.stop() + # Stop the bridge's producer + if runtime_log_bridge._producer is not None: + await runtime_log_bridge._producer.stop() + logger.debug( + "RuntimeLogEventBridge stopped (correlation_id=%s)", + correlation_id, + ) + except Exception: # noqa: BLE001 — boundary: best-effort cleanup + logger.warning( + "Error stopping RuntimeLogEventBridge (correlation_id=%s)", + correlation_id, + exc_info=True, + ) + runtime_log_bridge = None + # Stop health server (fast, non-blocking) if health_server is not None: try: @@ -2180,6 +2252,18 @@ def windows_handler(signum: int, frame: object) -> None: correlation_id, ) + # Cleanup RuntimeLogEventBridge (OMN-5525) + if runtime_log_bridge is not None: + try: + await runtime_log_bridge.stop() + if runtime_log_bridge._producer is not None: + await runtime_log_bridge._producer.stop() + except Exception: # noqa: BLE001 — boundary: best-effort cleanup + logger.warning( + "Failed to stop RuntimeLogEventBridge during cleanup (correlation_id=%s)", + correlation_id, + ) + if health_server is not None: try: await health_server.stop() diff --git a/src/omnibase_infra/services/observability/agent_actions/consumer.py b/src/omnibase_infra/services/observability/agent_actions/consumer.py index 689bb7cb55..05a1b31cd6 100644 --- a/src/omnibase_infra/services/observability/agent_actions/consumer.py +++ b/src/omnibase_infra/services/observability/agent_actions/consumer.py @@ -73,6 +73,8 @@ from omnibase_core.errors import OnexError from omnibase_core.types import JsonType +from omnibase_infra.event_bus.consumer_health_emitter import ConsumerHealthEmitter +from omnibase_infra.event_bus.mixin_consumer_health import MixinConsumerHealth from omnibase_infra.services.observability.agent_actions.config import ( ConfigAgentActionsConsumer, ) @@ -497,7 +499,7 @@ async def snapshot(self) -> dict[str, object]: # ============================================================================= -class AgentActionsConsumer: +class AgentActionsConsumer(MixinConsumerHealth): """Async Kafka consumer for agent observability events. Consumes events from multiple observability topics and persists them @@ -564,6 +566,8 @@ def __init__(self, config: ConfigAgentActionsConsumer) -> None: # Dead Letter Queue producer (Phase 2 - OMN-1768) self._dlq_producer: AIOKafkaProducer | None = None + # Dedicated health producer when DLQ is disabled (OMN-5523) + self._health_producer: AIOKafkaProducer | None = None # Health check server self._health_app: web.Application | None = None @@ -733,6 +737,24 @@ async def start(self) -> None: self._running = True self._shutdown_event.clear() + # Initialize consumer health emitter (OMN-5523) + if ConsumerHealthEmitter.is_enabled(): + # Use DLQ producer if available, otherwise create dedicated one + health_prod = self._dlq_producer + if health_prod is None: + self._health_producer = AIOKafkaProducer( + bootstrap_servers=self._config.kafka_bootstrap_servers, + ) + await self._health_producer.start() + health_prod = self._health_producer + self._init_health_emitter( + health_prod, + consumer_identity="agent-actions-consumer", + consumer_group=self._config.kafka_group_id, + topic=",".join(self._config.topics), + service_label="AgentActionsConsumer", + ) + logger.info( "AgentActionsConsumer started", extra={ @@ -817,6 +839,18 @@ async def _cleanup_resources(self, correlation_id: UUID) -> None: self._health_app = None + # Stop dedicated health producer (OMN-5523) + if self._health_producer is not None: + try: + await self._health_producer.stop() + except Exception: # noqa: BLE001 — boundary: logs warning and degrades + logger.warning( + "Error stopping health producer", + extra={"consumer_id": self._consumer_id}, + ) + finally: + self._health_producer = None + # Stop DLQ producer (Phase 2 - OMN-1768) if self._dlq_producer is not None: try: diff --git a/src/omnibase_infra/services/observability/context_audit/consumer.py b/src/omnibase_infra/services/observability/context_audit/consumer.py index ead268dc8c..efa0af5afa 100644 --- a/src/omnibase_infra/services/observability/context_audit/consumer.py +++ b/src/omnibase_infra/services/observability/context_audit/consumer.py @@ -65,6 +65,8 @@ from aiokafka import AIOKafkaConsumer, AIOKafkaProducer, TopicPartition from aiokafka.errors import KafkaError +from omnibase_infra.event_bus.consumer_health_emitter import ConsumerHealthEmitter +from omnibase_infra.event_bus.mixin_consumer_health import MixinConsumerHealth from omnibase_infra.services.observability.context_audit.config import ( ConfigContextAuditConsumer, ) @@ -249,7 +251,7 @@ async def snapshot(self) -> dict[str, object]: # ============================================================================= -class ContextAuditConsumer: +class ContextAuditConsumer(MixinConsumerHealth): """Async Kafka consumer for context integrity audit observability events. Subscribes to all context audit topics and persists events to PostgreSQL @@ -338,6 +340,17 @@ async def start(self) -> None: await self._producer.start() self._running = True + + # Initialize consumer health emitter (OMN-5523) + if ConsumerHealthEmitter.is_enabled() and self._producer is not None: + self._init_health_emitter( + self._producer, + consumer_identity="context-audit-consumer", + consumer_group=self.config.kafka_group_id, + topic=",".join(self.config.topics), + service_label="ContextAuditConsumer", + ) + logger.info("ContextAuditConsumer started successfully") async def stop(self) -> None: diff --git a/src/omnibase_infra/services/observability/injection_effectiveness/consumer.py b/src/omnibase_infra/services/observability/injection_effectiveness/consumer.py index a2b8b0cb35..47f0890f36 100644 --- a/src/omnibase_infra/services/observability/injection_effectiveness/consumer.py +++ b/src/omnibase_infra/services/observability/injection_effectiveness/consumer.py @@ -63,12 +63,14 @@ import asyncpg from aiohttp import web -from aiokafka import AIOKafkaConsumer, TopicPartition +from aiokafka import AIOKafkaConsumer, AIOKafkaProducer, TopicPartition from aiokafka.errors import KafkaError from pydantic import BaseModel, ValidationError from omnibase_core.errors import OnexError from omnibase_core.types import JsonType +from omnibase_infra.event_bus.consumer_health_emitter import ConsumerHealthEmitter +from omnibase_infra.event_bus.mixin_consumer_health import MixinConsumerHealth from omnibase_infra.services.observability.injection_effectiveness.config import ( ConfigInjectionEffectivenessConsumer, ) @@ -335,7 +337,7 @@ async def snapshot(self) -> dict[str, object]: # ============================================================================= -class InjectionEffectivenessConsumer: +class InjectionEffectivenessConsumer(MixinConsumerHealth): """Async Kafka consumer for injection effectiveness events. Consumes events from multiple injection effectiveness topics and persists them @@ -397,6 +399,7 @@ def __init__(self, config: ConfigInjectionEffectivenessConsumer) -> None: self._consumer: AIOKafkaConsumer | None = None self._pool: asyncpg.Pool | None = None self._writer: WriterInjectionEffectivenessPostgres | None = None + self._health_producer: AIOKafkaProducer | None = None self._running = False self._shutdown_event = asyncio.Event() @@ -537,6 +540,20 @@ async def start(self) -> None: self._running = True self._shutdown_event.clear() + # Initialize consumer health emitter (OMN-5523) + if ConsumerHealthEmitter.is_enabled(): + self._health_producer = AIOKafkaProducer( + bootstrap_servers=self._config.kafka_bootstrap_servers, + ) + await self._health_producer.start() + self._init_health_emitter( + self._health_producer, + consumer_identity="injection-effectiveness-consumer", + consumer_group=self._config.kafka_group_id, + topic=",".join(self._config.topics), + service_label="InjectionEffectivenessConsumer", + ) + logger.info( "InjectionEffectivenessConsumer started", extra={ @@ -621,6 +638,18 @@ async def _cleanup_resources(self, correlation_id: UUID) -> None: self._health_app = None + # Stop health producer (OMN-5523) + if self._health_producer is not None: + try: + await self._health_producer.stop() + except Exception: # noqa: BLE001 — boundary: logs warning and degrades + logger.warning( + "Error stopping health producer", + extra={"consumer_id": self._consumer_id}, + ) + finally: + self._health_producer = None + # Stop Kafka consumer if self._consumer is not None: try: diff --git a/src/omnibase_infra/services/observability/llm_cost_aggregation/consumer.py b/src/omnibase_infra/services/observability/llm_cost_aggregation/consumer.py index cb17d5b3fd..144fd5297a 100644 --- a/src/omnibase_infra/services/observability/llm_cost_aggregation/consumer.py +++ b/src/omnibase_infra/services/observability/llm_cost_aggregation/consumer.py @@ -64,7 +64,7 @@ import asyncpg from aiohttp import web -from aiokafka import AIOKafkaConsumer, TopicPartition +from aiokafka import AIOKafkaConsumer, AIOKafkaProducer, TopicPartition from aiokafka.errors import KafkaError from aiokafka.structs import OffsetAndMetadata from pydantic import ValidationError @@ -73,6 +73,8 @@ from omnibase_core.types import JsonType from omnibase_infra.enums import EnumInfraTransportType from omnibase_infra.errors import InfraConnectionError, ModelInfraErrorContext +from omnibase_infra.event_bus.consumer_health_emitter import ConsumerHealthEmitter +from omnibase_infra.event_bus.mixin_consumer_health import MixinConsumerHealth from omnibase_infra.services.observability.llm_cost_aggregation.config import ( ConfigLlmCostAggregation, ) @@ -345,7 +347,7 @@ async def health_snapshot(self) -> HealthSnapshot: # ============================================================================= -class ServiceLlmCostAggregator: +class ServiceLlmCostAggregator(MixinConsumerHealth): """Async Kafka consumer for LLM cost aggregation. Consumes LLM call completed events, writes raw metrics to @@ -379,6 +381,7 @@ def __init__(self, config: ConfigLlmCostAggregation) -> None: self._consumer: AIOKafkaConsumer | None = None self._pool: asyncpg.Pool | None = None self._writer: WriterLlmCostAggregationPostgres | None = None + self._health_producer: AIOKafkaProducer | None = None self._running = False self._shutdown_event = asyncio.Event() @@ -486,6 +489,20 @@ async def start(self) -> None: self._running = True self._shutdown_event.clear() + # Initialize consumer health emitter (OMN-5523) + if ConsumerHealthEmitter.is_enabled(): + self._health_producer = AIOKafkaProducer( + bootstrap_servers=self._config.kafka_bootstrap_servers, + ) + await self._health_producer.start() + self._init_health_emitter( + self._health_producer, + consumer_identity="llm-cost-aggregation-consumer", + consumer_group=self._config.kafka_group_id, + topic=",".join(self._config.topics), + service_label="ServiceLlmCostAggregator", + ) + logger.info( "ServiceLlmCostAggregator started", extra={ @@ -548,6 +565,18 @@ async def _cleanup_resources(self, correlation_id: UUID) -> None: self._health_app = None + # Stop health producer (OMN-5523) + if self._health_producer is not None: + try: + await self._health_producer.stop() + except Exception: # noqa: BLE001 — boundary: logs warning and degrades + logger.warning( + "Error stopping health producer", + extra={"consumer_id": self._consumer_id}, + ) + finally: + self._health_producer = None + if self._consumer is not None: try: await self._consumer.stop() diff --git a/src/omnibase_infra/services/observability/skill_lifecycle/consumer.py b/src/omnibase_infra/services/observability/skill_lifecycle/consumer.py index 0cb32bf7b9..1bc5942cd1 100644 --- a/src/omnibase_infra/services/observability/skill_lifecycle/consumer.py +++ b/src/omnibase_infra/services/observability/skill_lifecycle/consumer.py @@ -62,6 +62,8 @@ from aiokafka import AIOKafkaConsumer, AIOKafkaProducer, TopicPartition from aiokafka.errors import KafkaError +from omnibase_infra.event_bus.consumer_health_emitter import ConsumerHealthEmitter +from omnibase_infra.event_bus.mixin_consumer_health import MixinConsumerHealth from omnibase_infra.services.observability.skill_lifecycle.config import ( ConfigSkillLifecycleConsumer, ) @@ -273,7 +275,7 @@ async def snapshot(self) -> dict[str, object]: TOPIC_COMPLETED = "onex.evt.omniclaude.skill-completed.v1" -class SkillLifecycleConsumer: +class SkillLifecycleConsumer(MixinConsumerHealth): """Async Kafka consumer for skill lifecycle observability events. Subscribes to skill-started and skill-completed topics and persists @@ -363,6 +365,17 @@ async def start(self) -> None: await self._producer.start() self._running = True + + # Initialize consumer health emitter (OMN-5523) + if ConsumerHealthEmitter.is_enabled() and self._producer is not None: + self._init_health_emitter( + self._producer, + consumer_identity="skill-lifecycle-consumer", + consumer_group=self.config.kafka_group_id, + topic=",".join(self.config.topics), + service_label="SkillLifecycleConsumer", + ) + logger.info("SkillLifecycleConsumer started successfully") async def stop(self) -> None: diff --git a/src/omnibase_infra/services/session/consumer.py b/src/omnibase_infra/services/session/consumer.py index 0a31ea6af7..e4962e4d2b 100644 --- a/src/omnibase_infra/services/session/consumer.py +++ b/src/omnibase_infra/services/session/consumer.py @@ -60,10 +60,12 @@ from enum import StrEnum from uuid import UUID, uuid4 -from aiokafka import AIOKafkaConsumer +from aiokafka import AIOKafkaConsumer, AIOKafkaProducer from aiokafka.errors import KafkaError from pydantic import ValidationError +from omnibase_infra.event_bus.consumer_health_emitter import ConsumerHealthEmitter +from omnibase_infra.event_bus.mixin_consumer_health import MixinConsumerHealth from omnibase_infra.services.session.config_consumer import ConfigSessionConsumer from omnibase_infra.services.session.protocol_session_aggregator import ( ProtocolSessionAggregator, @@ -195,7 +197,7 @@ async def snapshot(self) -> dict[str, object]: # ============================================================================= -class SessionEventConsumer: +class SessionEventConsumer(MixinConsumerHealth): """Kafka consumer for Claude Code hook events. Consumes events from session/prompt/tool topics and processes @@ -266,6 +268,7 @@ def __init__( self._config = config self._aggregator = aggregator self._consumer: AIOKafkaConsumer | None = None + self._health_producer: AIOKafkaProducer | None = None self._running = False self._shutdown_event = asyncio.Event() @@ -376,6 +379,20 @@ async def start(self) -> None: self._running = True self._shutdown_event.clear() + # Initialize consumer health emitter (OMN-5523) + if ConsumerHealthEmitter.is_enabled(): + self._health_producer = AIOKafkaProducer( + bootstrap_servers=self._config.bootstrap_servers, + ) + await self._health_producer.start() + self._init_health_emitter( + self._health_producer, + consumer_identity="session-event-consumer", + consumer_group=self._config.group_id, + topic=",".join(self._config.topics), + service_label="SessionEventConsumer", + ) + logger.info( "SessionEventConsumer started", extra={ @@ -433,6 +450,15 @@ async def stop(self) -> None: if self._consumer is not None and self._consumer_paused: await self._resume_consumer(correlation_id) + # Stop health producer (OMN-5523) + if self._health_producer is not None: + try: + await self._health_producer.stop() + except Exception: # noqa: BLE001 — boundary: logs warning and degrades + logger.warning("Error stopping health producer", exc_info=True) + finally: + self._health_producer = None + # Close consumer connection if self._consumer is not None: try: diff --git a/src/omnibase_infra/topics/__init__.py b/src/omnibase_infra/topics/__init__.py index 233ae37261..b1694abf4a 100644 --- a/src/omnibase_infra/topics/__init__.py +++ b/src/omnibase_infra/topics/__init__.py @@ -33,6 +33,8 @@ ALL_VALIDATION_TOPIC_SPECS, SUFFIX_BASELINES_COMPUTED, SUFFIX_CIRCUIT_BREAKER_STATE, + SUFFIX_CONSUMER_HEALTH, + SUFFIX_CONSUMER_RESTART_CMD, SUFFIX_CONTRACT_DEREGISTERED, SUFFIX_CONTRACT_REGISTERED, SUFFIX_FSM_STATE_TRANSITIONS, @@ -96,6 +98,7 @@ SUFFIX_REGISTRY_REQUEST_INTROSPECTION, SUFFIX_REQUEST_INTROSPECTION, SUFFIX_RESOLUTION_DECIDED, + SUFFIX_RUNTIME_ERROR, SUFFIX_RUNTIME_TICK, SUFFIX_SERVICE_HEARTBEAT, SUFFIX_TOPIC_CATALOG_CHANGED, @@ -155,6 +158,10 @@ "SUFFIX_CIRCUIT_BREAKER_STATE", "SUFFIX_GMAIL_ARCHIVE_PURGED", "SUFFIX_WIRING_HEALTH_SNAPSHOT", + # Runtime health event pipeline (OMN-5529) + "SUFFIX_CONSUMER_HEALTH", + "SUFFIX_CONSUMER_RESTART_CMD", + "SUFFIX_RUNTIME_ERROR", # OmniMemory domain suffix constants "SUFFIX_OMNIMEMORY_DOCUMENT_DISCOVERED", "SUFFIX_OMNIMEMORY_DOCUMENT_CHANGED", diff --git a/src/omnibase_infra/topics/platform_topic_suffixes.py b/src/omnibase_infra/topics/platform_topic_suffixes.py index ee8efec8c1..7048fab4ba 100644 --- a/src/omnibase_infra/topics/platform_topic_suffixes.py +++ b/src/omnibase_infra/topics/platform_topic_suffixes.py @@ -441,6 +441,44 @@ Consumer: omnidash /circuit-breaker dashboard (OMN-5293) """ +# ============================================================================= +# OMNIBASE_INFRA CONSUMER HEALTH TOPIC SUFFIXES (OMN-5515 / OMN-5529) +# ============================================================================= + +SUFFIX_CONSUMER_HEALTH: str = "onex.evt.omnibase-infra.consumer-health.v1" +"""Topic suffix for consumer health events. + +Published by ConsumerHealthEmitter when consumer lifecycle events occur +(heartbeat failures, session timeouts, rebalances, etc.). + +Producer: ConsumerHealthEmitter (EventBusKafka, standalone consumers via MixinConsumerHealth) +Consumer: NodeConsumerHealthTriageEffect, omnidash /consumer-health dashboard +""" + +SUFFIX_CONSUMER_RESTART_CMD: str = "onex.cmd.omnibase-infra.consumer-restart.v1" +"""Topic suffix for consumer restart commands. + +Published by NodeConsumerHealthTriageEffect when graduated response +escalates to automated restart. + +Producer: NodeConsumerHealthTriageEffect +Consumer: MixinConsumerHealth (standalone consumers) +""" + +# ============================================================================= +# OMNIBASE_INFRA RUNTIME ERROR TOPIC SUFFIXES (OMN-5517 / OMN-5529) +# ============================================================================= + +SUFFIX_RUNTIME_ERROR: str = "onex.evt.omnibase-infra.runtime-error.v1" +"""Topic suffix for runtime error events. + +Published by RuntimeLogEventBridge (logging.Handler) when ERROR/WARNING +log records are captured from allowlisted Python loggers. + +Producer: RuntimeLogEventBridge +Consumer: NodeRuntimeErrorTriageEffect, omnidash /runtime-errors dashboard +""" + # Full topic name (not a suffix) — named as such to be unambiguous. # Used by monitor_logs.py postgres error emitter and downstream consumers. TOPIC_DB_ERROR_V1: str = "onex.evt.omnibase-infra.db-error.v1" @@ -494,6 +532,33 @@ "cleanup.policy": "delete", }, # 7 days ), + # Consumer health events (3 partitions — event-driven, OMN-5515) + ModelTopicSpec( + suffix=SUFFIX_CONSUMER_HEALTH, + partitions=3, + kafka_config={ + "retention.ms": "604800000", + "cleanup.policy": "delete", + }, # 7 days + ), + # Consumer restart commands (1 partition — low-throughput commands, OMN-5515) + ModelTopicSpec( + suffix=SUFFIX_CONSUMER_RESTART_CMD, + partitions=1, + kafka_config={ + "retention.ms": "86400000", + "cleanup.policy": "delete", + }, # 1 day — commands are short-lived + ), + # Runtime error events (3 partitions — event-driven, OMN-5517) + ModelTopicSpec( + suffix=SUFFIX_RUNTIME_ERROR, + partitions=3, + kafka_config={ + "retention.ms": "604800000", + "cleanup.policy": "delete", + }, # 7 days + ), ) """Omnibase_infra domain topic specs for internal effect nodes. diff --git a/tests/integration/test_consumer_health_pipeline.py b/tests/integration/test_consumer_health_pipeline.py new file mode 100644 index 0000000000..8a697ed46d --- /dev/null +++ b/tests/integration/test_consumer_health_pipeline.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Integration tests for Layer 1: Consumer Health Pipeline. + +Tests the end-to-end flow from ConsumerHealthEmitter through to triage handlers. + +OMN-5524: Integration test for Layer 1 end-to-end flow. + +Requires: running Kafka/Redpanda broker on localhost:19092. + +Tests: + - Emit → consume → triage graduated response + - Restart command flow (3rd occurrence triggers restart) + - Restart failure → Linear ticket creation (mocked) + - Rate limiting prevents flood + - Emitter self-metrics accuracy +""" + +from __future__ import annotations + +import asyncio +import json +import os +from collections.abc import AsyncGenerator +from typing import Any +from unittest.mock import patch +from uuid import uuid4 + +import pytest +from aiokafka import AIOKafkaConsumer, AIOKafkaProducer +from aiokafka.structs import ConsumerRecord + +from omnibase_infra.event_bus.consumer_health_emitter import ConsumerHealthEmitter +from omnibase_infra.event_bus.topic_constants import TOPIC_CONSUMER_HEALTH +from omnibase_infra.models.health.enum_consumer_health_event_type import ( + EnumConsumerHealthEventType, +) +from omnibase_infra.models.health.enum_consumer_health_severity import ( + EnumConsumerHealthSeverity, +) +from omnibase_infra.models.health.model_consumer_health_event import ( + ModelConsumerHealthEvent, +) + +pytestmark = [ + pytest.mark.integration, + pytest.mark.kafka, +] + +BOOTSTRAP_SERVERS = "localhost:19092" + + +@pytest.fixture +async def kafka_producer() -> AsyncGenerator[AIOKafkaProducer, None]: + """Create and start a Kafka producer for tests.""" + producer = AIOKafkaProducer( + bootstrap_servers=BOOTSTRAP_SERVERS, + ) + await producer.start() + yield producer + await producer.stop() + + +@pytest.fixture +async def kafka_consumer() -> AsyncGenerator[AIOKafkaConsumer, None]: + """Create and start a Kafka consumer for the health topic.""" + group_id = f"test-consumer-health-{uuid4().hex[:8]}" + consumer = AIOKafkaConsumer( + TOPIC_CONSUMER_HEALTH, + bootstrap_servers=BOOTSTRAP_SERVERS, + group_id=group_id, + auto_offset_reset="latest", + enable_auto_commit=True, + consumer_timeout_ms=5000, + ) + await consumer.start() + yield consumer + await consumer.stop() + + +class TestConsumerHealthEmitterIntegration: + """Tests for ConsumerHealthEmitter → Kafka → Consumer flow.""" + + @pytest.mark.asyncio + async def test_emit_event_produces_to_kafka( + self, kafka_producer: AIOKafkaProducer, kafka_consumer: AIOKafkaConsumer + ) -> None: + """Verify emitter produces a valid event to Kafka and it can be consumed.""" + with patch.dict(os.environ, {"ENABLE_CONSUMER_HEALTH_EMITTER": "true"}): + emitter = ConsumerHealthEmitter(kafka_producer) + + # Emit a heartbeat failure event + await emitter.emit_event( + consumer_identity="test-consumer-1", + consumer_group="test-group", + topic="test.topic.v1", + event_type=EnumConsumerHealthEventType.HEARTBEAT_FAILURE, + severity=EnumConsumerHealthSeverity.ERROR, + error_message="Heartbeat session expired", + error_type="SessionExpiredError", + ) + + assert emitter.events_emitted == 1 + assert emitter.events_dropped == 0 + + # Consume and validate + msg = await asyncio.wait_for(_get_one_message(kafka_consumer), timeout=10.0) + assert msg is not None + assert msg.value is not None + + payload = json.loads(msg.value) + assert payload["consumer_identity"] == "test-consumer-1" + assert payload["event_type"] == "HEARTBEAT_FAILURE" + assert payload["severity"] == "ERROR" + + @pytest.mark.asyncio + async def test_emitter_rate_limiting( + self, kafka_producer: AIOKafkaProducer + ) -> None: + """Verify rate limiter suppresses duplicate events within window.""" + with patch.dict(os.environ, {"ENABLE_CONSUMER_HEALTH_EMITTER": "true"}): + emitter = ConsumerHealthEmitter(kafka_producer) + + # Emit same event type rapidly 5 times + for _ in range(5): + await emitter.emit_event( + consumer_identity="test-consumer-rl", + consumer_group="test-group", + topic="test.topic.v1", + event_type=EnumConsumerHealthEventType.HEARTBEAT_FAILURE, + severity=EnumConsumerHealthSeverity.ERROR, + error_message="Same error", + ) + + # First should emit, subsequent should be rate-limited + assert emitter.events_emitted == 1 + assert emitter.events_rate_limited == 4 + + @pytest.mark.asyncio + async def test_emitter_disabled_by_default( + self, kafka_producer: AIOKafkaProducer + ) -> None: + """Verify emitter is a no-op when feature flag is off.""" + with patch.dict(os.environ, {}, clear=False): + # Ensure the flag is not set + os.environ.pop("ENABLE_CONSUMER_HEALTH_EMITTER", None) + + emitter = ConsumerHealthEmitter(kafka_producer) + assert not ConsumerHealthEmitter.is_enabled() + + await emitter.emit_event( + consumer_identity="test-consumer-disabled", + consumer_group="test-group", + topic="test.topic.v1", + event_type=EnumConsumerHealthEventType.CONSUMER_STARTED, + severity=EnumConsumerHealthSeverity.INFO, + ) + + # Should be dropped (feature disabled) + assert emitter.events_emitted == 0 + + @pytest.mark.asyncio + async def test_emitter_self_metrics(self, kafka_producer: AIOKafkaProducer) -> None: + """Verify emitter tracks self-metrics accurately.""" + with patch.dict(os.environ, {"ENABLE_CONSUMER_HEALTH_EMITTER": "true"}): + emitter = ConsumerHealthEmitter(kafka_producer) + + # Emit distinct events (different fingerprints) + for i in range(3): + await emitter.emit_event( + consumer_identity=f"test-consumer-{i}", + consumer_group="test-group", + topic="test.topic.v1", + event_type=EnumConsumerHealthEventType.CONSUMER_STARTED, + severity=EnumConsumerHealthSeverity.INFO, + ) + + assert emitter.events_emitted == 3 + assert emitter.events_dropped == 0 + assert emitter.events_rate_limited == 0 + + +class TestMixinConsumerHealthIntegration: + """Tests for MixinConsumerHealth wiring in standalone consumers.""" + + @pytest.mark.asyncio + async def test_mixin_init_health_emitter( + self, kafka_producer: AIOKafkaProducer + ) -> None: + """Verify mixin correctly initializes health emitter.""" + from omnibase_infra.event_bus.mixin_consumer_health import MixinConsumerHealth + + class TestConsumer(MixinConsumerHealth): + pass + + with patch.dict(os.environ, {"ENABLE_CONSUMER_HEALTH_EMITTER": "true"}): + consumer = TestConsumer() + consumer._init_health_emitter( + kafka_producer, + consumer_identity="test-mixin-consumer", + consumer_group="test-group", + topic="test.topic.v1", + service_label="TestConsumer", + ) + + assert consumer._health_emitter is not None + assert consumer._health_consumer_identity == "test-mixin-consumer" + + # Emit via convenience method + await consumer._emit_consumer_started() + + assert consumer._health_emitter.events_emitted == 1 + + +async def _get_one_message( + consumer: AIOKafkaConsumer, +) -> ConsumerRecord[Any, Any] | None: + """Poll for a single message from a Kafka consumer.""" + async for msg in consumer: + return msg + return None diff --git a/tests/integration/test_runtime_log_bridge_pipeline.py b/tests/integration/test_runtime_log_bridge_pipeline.py new file mode 100644 index 0000000000..355ac4c523 --- /dev/null +++ b/tests/integration/test_runtime_log_bridge_pipeline.py @@ -0,0 +1,281 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Integration tests for Layer 2: Runtime Log Event Bridge Pipeline. + +Tests the end-to-end flow from Python logging → RuntimeLogEventBridge → Kafka. + +OMN-5526: Integration test for Layer 2. + +Requires: running Kafka/Redpanda broker on localhost:19092. + +Tests: + - Log error → Kafka event flow + - Circular logging prevention + - Rate limiting + - Bridge metrics accuracy + - Allowlist-only attachment +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import os +from collections.abc import AsyncGenerator +from typing import Any +from uuid import uuid4 + +import pytest +from aiokafka import AIOKafkaConsumer, AIOKafkaProducer +from aiokafka.structs import ConsumerRecord + +from omnibase_infra.event_bus.topic_constants import TOPIC_RUNTIME_ERROR +from omnibase_infra.observability.runtime_log_event_bridge import ( + RuntimeLogEventBridge, +) + +pytestmark = [ + pytest.mark.integration, + pytest.mark.kafka, +] + +BOOTSTRAP_SERVERS = "localhost:19092" + + +@pytest.fixture +async def kafka_producer() -> AsyncGenerator[AIOKafkaProducer, None]: + """Create and start a Kafka producer for the bridge.""" + producer = AIOKafkaProducer( + bootstrap_servers=BOOTSTRAP_SERVERS, + ) + await producer.start() + yield producer + await producer.stop() + + +@pytest.fixture +async def kafka_consumer() -> AsyncGenerator[AIOKafkaConsumer, None]: + """Create and start a Kafka consumer for the runtime error topic.""" + group_id = f"test-runtime-error-{uuid4().hex[:8]}" + consumer = AIOKafkaConsumer( + TOPIC_RUNTIME_ERROR, + bootstrap_servers=BOOTSTRAP_SERVERS, + group_id=group_id, + auto_offset_reset="latest", + enable_auto_commit=True, + consumer_timeout_ms=5000, + ) + await consumer.start() + yield consumer + await consumer.stop() + + +class _EnableBridgeCtx: + """Context manager to enable the runtime log bridge feature flag.""" + + def __enter__(self) -> _EnableBridgeCtx: + os.environ["ENABLE_RUNTIME_LOG_BRIDGE"] = "true" + return self + + def __exit__(self, *args: object) -> None: + os.environ.pop("ENABLE_RUNTIME_LOG_BRIDGE", None) + + +class TestRuntimeLogBridgeIntegration: + """Tests for RuntimeLogEventBridge → Kafka flow.""" + + @pytest.mark.asyncio + async def test_log_error_produces_kafka_event( + self, + kafka_producer: AIOKafkaProducer, + kafka_consumer: AIOKafkaConsumer, + ) -> None: + """Verify an ERROR log record flows through the bridge to Kafka.""" + os.environ["ENABLE_RUNTIME_LOG_BRIDGE"] = "true" + try: + bridge = RuntimeLogEventBridge( + producer=kafka_producer, + hostname="test-host", + service_label="test-service", + ) + await bridge.start() + + # Create a test logger and attach bridge + test_logger_name = f"test.logger.{uuid4().hex[:8]}" + bridge.attach_to_loggers([test_logger_name]) + + test_logger = logging.getLogger(test_logger_name) + test_logger.setLevel(logging.WARNING) + + # Emit an error log + test_logger.error("Database connection failed to host %s", "db-primary") + + # Give the drain loop time to process + await asyncio.sleep(0.5) + + assert bridge.events_emitted == 1 + assert bridge.events_dropped == 0 + + # Consume and validate + msg = await asyncio.wait_for(_get_one_message(kafka_consumer), timeout=10.0) + assert msg is not None + assert msg.value is not None + + payload = json.loads(msg.value) + assert payload["log_level"] == "ERROR" + assert payload["hostname"] == "test-host" + assert payload["service_label"] == "test-service" + assert "Database connection failed" in payload["raw_message"] + + # Cleanup + bridge.detach_from_loggers([test_logger_name]) + await bridge.stop() + finally: + os.environ.pop("ENABLE_RUNTIME_LOG_BRIDGE", None) + + @pytest.mark.asyncio + async def test_circular_logging_prevention( + self, + kafka_producer: AIOKafkaProducer, + ) -> None: + """Verify the bridge does not capture its own log records.""" + os.environ["ENABLE_RUNTIME_LOG_BRIDGE"] = "true" + try: + bridge = RuntimeLogEventBridge( + producer=kafka_producer, + hostname="test-host", + service_label="test-service", + ) + await bridge.start() + + # The bridge's own logger name starts with the module name + bridge_logger = logging.getLogger( + "omnibase_infra.observability.runtime_log_event_bridge._bridge" + ) + bridge_logger.setLevel(logging.WARNING) + + # Attach bridge to its own module logger (should be filtered) + bridge.attach_to_loggers( + ["omnibase_infra.observability.runtime_log_event_bridge"] + ) + + # Emit via bridge's internal logger + bridge_logger.error("This should NOT be captured") + await asyncio.sleep(0.3) + + # Bridge should not have emitted anything (circular prevention) + assert bridge.events_emitted == 0 + + bridge.detach_from_loggers( + ["omnibase_infra.observability.runtime_log_event_bridge"] + ) + await bridge.stop() + finally: + os.environ.pop("ENABLE_RUNTIME_LOG_BRIDGE", None) + + @pytest.mark.asyncio + async def test_rate_limiting( + self, + kafka_producer: AIOKafkaProducer, + ) -> None: + """Verify rate limiter suppresses duplicate log events.""" + os.environ["ENABLE_RUNTIME_LOG_BRIDGE"] = "true" + try: + bridge = RuntimeLogEventBridge( + producer=kafka_producer, + hostname="test-host", + service_label="test-service", + ) + await bridge.start() + + test_logger_name = f"test.ratelimit.{uuid4().hex[:8]}" + bridge.attach_to_loggers([test_logger_name]) + test_logger = logging.getLogger(test_logger_name) + test_logger.setLevel(logging.WARNING) + + # Emit same error message 10 times rapidly + for _ in range(10): + test_logger.error("Repeated error message") + + await asyncio.sleep(0.5) + + # Max 5 per fingerprint per 5-minute window + assert bridge.events_emitted <= 5 + assert bridge.events_rate_limited >= 5 + + bridge.detach_from_loggers([test_logger_name]) + await bridge.stop() + finally: + os.environ.pop("ENABLE_RUNTIME_LOG_BRIDGE", None) + + @pytest.mark.asyncio + async def test_bridge_disabled_by_default( + self, + kafka_producer: AIOKafkaProducer, + ) -> None: + """Verify bridge is a no-op when feature flag is off.""" + os.environ.pop("ENABLE_RUNTIME_LOG_BRIDGE", None) + + bridge = RuntimeLogEventBridge( + producer=kafka_producer, + ) + assert not RuntimeLogEventBridge.is_enabled() + + test_logger_name = f"test.disabled.{uuid4().hex[:8]}" + bridge.attach_to_loggers([test_logger_name]) + test_logger = logging.getLogger(test_logger_name) + test_logger.setLevel(logging.WARNING) + + test_logger.error("This should be silently dropped") + await asyncio.sleep(0.2) + + # No events should be emitted or queued + assert bridge.events_emitted == 0 + + bridge.detach_from_loggers([test_logger_name]) + + @pytest.mark.asyncio + async def test_bridge_metrics_accuracy( + self, + kafka_producer: AIOKafkaProducer, + ) -> None: + """Verify bridge self-metrics track correctly.""" + os.environ["ENABLE_RUNTIME_LOG_BRIDGE"] = "true" + try: + bridge = RuntimeLogEventBridge( + producer=kafka_producer, + hostname="test-host", + service_label="test-service", + ) + await bridge.start() + + test_logger_name = f"test.metrics.{uuid4().hex[:8]}" + bridge.attach_to_loggers([test_logger_name]) + test_logger = logging.getLogger(test_logger_name) + test_logger.setLevel(logging.WARNING) + + # Emit 3 distinct errors + test_logger.error("Error A unique %s", uuid4()) + test_logger.error("Error B unique %s", uuid4()) + test_logger.warning("Warning C unique %s", uuid4()) + + await asyncio.sleep(0.5) + + assert bridge.events_emitted == 3 + assert bridge.events_dropped == 0 + assert bridge.events_rate_limited == 0 + + bridge.detach_from_loggers([test_logger_name]) + await bridge.stop() + finally: + os.environ.pop("ENABLE_RUNTIME_LOG_BRIDGE", None) + + +async def _get_one_message( + consumer: AIOKafkaConsumer, +) -> ConsumerRecord[Any, Any] | None: + """Poll for a single message from a Kafka consumer.""" + async for msg in consumer: + return msg + return None diff --git a/tests/unit/event_bus/test_consumer_health_emitter.py b/tests/unit/event_bus/test_consumer_health_emitter.py new file mode 100644 index 0000000000..dc198800b0 --- /dev/null +++ b/tests/unit/event_bus/test_consumer_health_emitter.py @@ -0,0 +1,129 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Unit tests for ConsumerHealthEmitter. + +Ticket: OMN-5516 +""" + +from __future__ import annotations + +import asyncio +from unittest.mock import AsyncMock, patch + +import pytest + +from omnibase_infra.event_bus.consumer_health_emitter import ConsumerHealthEmitter +from omnibase_infra.models.health.enum_consumer_health_event_type import ( + EnumConsumerHealthEventType, +) +from omnibase_infra.models.health.enum_consumer_health_severity import ( + EnumConsumerHealthSeverity, +) +from omnibase_infra.models.health.model_consumer_health_event import ( + ModelConsumerHealthEvent, +) + + +@pytest.fixture +def mock_producer() -> AsyncMock: + """Create a mock AIOKafkaProducer.""" + producer = AsyncMock() + producer.send = AsyncMock() + return producer + + +@pytest.fixture +def emitter(mock_producer: AsyncMock) -> ConsumerHealthEmitter: + """Create a ConsumerHealthEmitter with mock producer.""" + return ConsumerHealthEmitter(mock_producer) + + +def _make_event( + consumer_identity: str = "consumer-1", + event_type: EnumConsumerHealthEventType = EnumConsumerHealthEventType.HEARTBEAT_FAILURE, +) -> ModelConsumerHealthEvent: + return ModelConsumerHealthEvent.create( + consumer_identity=consumer_identity, + consumer_group="group-1", + topic="test-topic", + event_type=event_type, + severity=EnumConsumerHealthSeverity.ERROR, + ) + + +@pytest.mark.unit +class TestConsumerHealthEmitter: + """Tests for ConsumerHealthEmitter.""" + + @patch.dict("os.environ", {"ENABLE_CONSUMER_HEALTH_EMITTER": "true"}) + async def test_emit_sends_to_producer( + self, emitter: ConsumerHealthEmitter, mock_producer: AsyncMock + ) -> None: + event = _make_event() + await emitter.emit(event) + mock_producer.send.assert_called_once() + assert emitter.events_emitted == 1 + + @patch.dict("os.environ", {"ENABLE_CONSUMER_HEALTH_EMITTER": ""}) + async def test_emit_skipped_when_disabled( + self, emitter: ConsumerHealthEmitter, mock_producer: AsyncMock + ) -> None: + event = _make_event() + await emitter.emit(event) + mock_producer.send.assert_not_called() + assert emitter.events_emitted == 0 + + @patch.dict("os.environ", {"ENABLE_CONSUMER_HEALTH_EMITTER": "true"}) + async def test_rate_limiting( + self, emitter: ConsumerHealthEmitter, mock_producer: AsyncMock + ) -> None: + event = _make_event() + await emitter.emit(event) + await emitter.emit(event) # Same fingerprint, should be rate-limited + assert mock_producer.send.call_count == 1 + assert emitter.events_emitted == 1 + assert emitter.events_rate_limited == 1 + + @patch.dict("os.environ", {"ENABLE_CONSUMER_HEALTH_EMITTER": "true"}) + async def test_different_fingerprints_not_rate_limited( + self, emitter: ConsumerHealthEmitter, mock_producer: AsyncMock + ) -> None: + event1 = _make_event(consumer_identity="c1") + event2 = _make_event(consumer_identity="c2") + await emitter.emit(event1) + await emitter.emit(event2) + assert mock_producer.send.call_count == 2 + assert emitter.events_emitted == 2 + + @patch.dict("os.environ", {"ENABLE_CONSUMER_HEALTH_EMITTER": "true"}) + async def test_emit_failure_is_tolerated( + self, emitter: ConsumerHealthEmitter, mock_producer: AsyncMock + ) -> None: + mock_producer.send.side_effect = Exception("broker down") + event = _make_event() + await emitter.emit(event) # Should not raise + assert emitter.events_dropped == 1 + assert emitter.events_emitted == 0 + + def test_is_enabled_returns_false_by_default(self) -> None: + with patch.dict("os.environ", {}, clear=True): + assert not ConsumerHealthEmitter.is_enabled() + + def test_is_enabled_returns_true_when_set(self) -> None: + with patch.dict("os.environ", {"ENABLE_CONSUMER_HEALTH_EMITTER": "true"}): + assert ConsumerHealthEmitter.is_enabled() + + @patch.dict("os.environ", {"ENABLE_CONSUMER_HEALTH_EMITTER": "true"}) + async def test_emit_event_convenience( + self, emitter: ConsumerHealthEmitter, mock_producer: AsyncMock + ) -> None: + await emitter.emit_event( + consumer_identity="c1", + consumer_group="g1", + topic="t1", + event_type=EnumConsumerHealthEventType.SESSION_TIMEOUT, + severity=EnumConsumerHealthSeverity.CRITICAL, + error_message="session expired", + ) + mock_producer.send.assert_called_once() + assert emitter.events_emitted == 1 diff --git a/tests/unit/models/health/__init__.py b/tests/unit/models/health/__init__.py new file mode 100644 index 0000000000..327f1172ed --- /dev/null +++ b/tests/unit/models/health/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT diff --git a/tests/unit/models/health/test_model_consumer_health_event.py b/tests/unit/models/health/test_model_consumer_health_event.py new file mode 100644 index 0000000000..3cf55baacf --- /dev/null +++ b/tests/unit/models/health/test_model_consumer_health_event.py @@ -0,0 +1,204 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Unit tests for consumer health event models and enums. + +Ticket: OMN-5511 +""" + +from __future__ import annotations + +from uuid import UUID + +import pytest + +from omnibase_infra.models.health.enum_consumer_health_event_type import ( + EnumConsumerHealthEventType, +) +from omnibase_infra.models.health.enum_consumer_health_severity import ( + EnumConsumerHealthSeverity, +) +from omnibase_infra.models.health.enum_consumer_incident_state import ( + EnumConsumerIncidentState, +) +from omnibase_infra.models.health.model_consumer_health_event import ( + ModelConsumerHealthEvent, + _compute_fingerprint, +) +from omnibase_infra.models.health.model_consumer_restart_command import ( + ModelConsumerRestartCommand, +) + + +@pytest.mark.unit +class TestEnumConsumerHealthEventType: + """Tests for EnumConsumerHealthEventType.""" + + def test_all_values_exist(self) -> None: + assert EnumConsumerHealthEventType.HEARTBEAT_FAILURE == "heartbeat_failure" + assert EnumConsumerHealthEventType.SESSION_TIMEOUT == "session_timeout" + assert EnumConsumerHealthEventType.REBALANCE_START == "rebalance_start" + assert EnumConsumerHealthEventType.REBALANCE_COMPLETE == "rebalance_complete" + assert EnumConsumerHealthEventType.CONSUMER_STOPPED == "consumer_stopped" + assert EnumConsumerHealthEventType.CONSUMER_STARTED == "consumer_started" + assert EnumConsumerHealthEventType.POLL_TIMEOUT == "poll_timeout" + assert EnumConsumerHealthEventType.CONNECTION_LOST == "connection_lost" + + def test_is_str_enum(self) -> None: + assert isinstance(EnumConsumerHealthEventType.HEARTBEAT_FAILURE, str) + + +@pytest.mark.unit +class TestEnumConsumerHealthSeverity: + """Tests for EnumConsumerHealthSeverity.""" + + def test_all_values_exist(self) -> None: + assert EnumConsumerHealthSeverity.INFO == "info" + assert EnumConsumerHealthSeverity.WARNING == "warning" + assert EnumConsumerHealthSeverity.ERROR == "error" + assert EnumConsumerHealthSeverity.CRITICAL == "critical" + + +@pytest.mark.unit +class TestEnumConsumerIncidentState: + """Tests for EnumConsumerIncidentState.""" + + def test_all_values_exist(self) -> None: + assert EnumConsumerIncidentState.OPEN == "open" + assert EnumConsumerIncidentState.ACKNOWLEDGED == "acknowledged" + assert EnumConsumerIncidentState.RESTART_PENDING == "restart_pending" + assert EnumConsumerIncidentState.RESTART_SUCCEEDED == "restart_succeeded" + assert EnumConsumerIncidentState.RESTART_FAILED == "restart_failed" + assert EnumConsumerIncidentState.TICKETED == "ticketed" + assert EnumConsumerIncidentState.RESOLVED == "resolved" + + +@pytest.mark.unit +class TestComputeFingerprint: + """Tests for _compute_fingerprint.""" + + def test_deterministic(self) -> None: + fp1 = _compute_fingerprint("consumer-1", "heartbeat_failure", "topic-a") + fp2 = _compute_fingerprint("consumer-1", "heartbeat_failure", "topic-a") + assert fp1 == fp2 + + def test_different_inputs_different_fingerprints(self) -> None: + fp1 = _compute_fingerprint("consumer-1", "heartbeat_failure", "topic-a") + fp2 = _compute_fingerprint("consumer-2", "heartbeat_failure", "topic-a") + assert fp1 != fp2 + + def test_length(self) -> None: + fp = _compute_fingerprint("c", "e", "t") + assert len(fp) == 16 + + +@pytest.mark.unit +class TestModelConsumerHealthEvent: + """Tests for ModelConsumerHealthEvent.""" + + def test_create_factory(self) -> None: + event = ModelConsumerHealthEvent.create( + consumer_identity="consumer-1", + consumer_group="group-1", + topic="test-topic", + event_type=EnumConsumerHealthEventType.HEARTBEAT_FAILURE, + severity=EnumConsumerHealthSeverity.ERROR, + ) + assert event.consumer_identity == "consumer-1" + assert event.consumer_group == "group-1" + assert event.topic == "test-topic" + assert event.event_type == EnumConsumerHealthEventType.HEARTBEAT_FAILURE + assert event.severity == EnumConsumerHealthSeverity.ERROR + assert len(event.fingerprint) == 16 + assert isinstance(event.event_id, UUID) + assert isinstance(event.correlation_id, UUID) + + def test_create_with_rebalance_metrics(self) -> None: + event = ModelConsumerHealthEvent.create( + consumer_identity="consumer-1", + consumer_group="group-1", + topic="test-topic", + event_type=EnumConsumerHealthEventType.REBALANCE_COMPLETE, + severity=EnumConsumerHealthSeverity.WARNING, + rebalance_duration_ms=1500, + partitions_assigned=3, + partitions_revoked=1, + ) + assert event.rebalance_duration_ms == 1500 + assert event.partitions_assigned == 3 + assert event.partitions_revoked == 1 + + def test_frozen_model(self) -> None: + event = ModelConsumerHealthEvent.create( + consumer_identity="c", + consumer_group="g", + topic="t", + event_type=EnumConsumerHealthEventType.CONSUMER_STARTED, + severity=EnumConsumerHealthSeverity.INFO, + ) + with pytest.raises(Exception): + event.consumer_identity = "other" # type: ignore[misc] + + def test_serialization_roundtrip(self) -> None: + event = ModelConsumerHealthEvent.create( + consumer_identity="consumer-1", + consumer_group="group-1", + topic="test-topic", + event_type=EnumConsumerHealthEventType.SESSION_TIMEOUT, + severity=EnumConsumerHealthSeverity.CRITICAL, + error_message="session timed out", + hostname="host-1", + service_label="my-service", + ) + data = event.model_dump(mode="json") + restored = ModelConsumerHealthEvent.model_validate(data) + assert restored.consumer_identity == event.consumer_identity + assert restored.fingerprint == event.fingerprint + assert restored.error_message == "session timed out" + + def test_fingerprint_stability(self) -> None: + e1 = ModelConsumerHealthEvent.create( + consumer_identity="c1", + consumer_group="g1", + topic="t1", + event_type=EnumConsumerHealthEventType.HEARTBEAT_FAILURE, + severity=EnumConsumerHealthSeverity.ERROR, + ) + e2 = ModelConsumerHealthEvent.create( + consumer_identity="c1", + consumer_group="g1", + topic="t1", + event_type=EnumConsumerHealthEventType.HEARTBEAT_FAILURE, + severity=EnumConsumerHealthSeverity.WARNING, # different severity + ) + # Same consumer_identity + event_type + topic = same fingerprint + assert e1.fingerprint == e2.fingerprint + + +@pytest.mark.unit +class TestModelConsumerRestartCommand: + """Tests for ModelConsumerRestartCommand.""" + + def test_create(self) -> None: + cmd = ModelConsumerRestartCommand( + consumer_identity="consumer-1", + consumer_group="group-1", + topic="test-topic", + reason="3rd heartbeat failure in 30 minutes", + fingerprint="abc123", + ) + assert cmd.consumer_identity == "consumer-1" + assert cmd.intent_type == "consumer_health.restart" + assert isinstance(cmd.command_id, UUID) + + def test_serialization_roundtrip(self) -> None: + cmd = ModelConsumerRestartCommand( + consumer_identity="c", + consumer_group="g", + topic="t", + reason="test", + fingerprint="fp", + ) + data = cmd.model_dump(mode="json") + restored = ModelConsumerRestartCommand.model_validate(data) + assert restored.consumer_identity == cmd.consumer_identity + assert restored.intent_type == "consumer_health.restart" diff --git a/tests/unit/models/health/test_model_runtime_error_event.py b/tests/unit/models/health/test_model_runtime_error_event.py new file mode 100644 index 0000000000..2efbb908ae --- /dev/null +++ b/tests/unit/models/health/test_model_runtime_error_event.py @@ -0,0 +1,164 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Unit tests for runtime error event model. + +Ticket: OMN-5513 +""" + +from __future__ import annotations + +from uuid import UUID + +import pytest + +from omnibase_infra.models.health.enum_runtime_error_category import ( + EnumRuntimeErrorCategory, +) +from omnibase_infra.models.health.enum_runtime_error_severity import ( + EnumRuntimeErrorSeverity, +) +from omnibase_infra.models.health.model_runtime_error_event import ( + ModelRuntimeErrorEvent, + _compute_error_fingerprint, +) + + +@pytest.mark.unit +class TestEnumRuntimeErrorCategory: + """Tests for EnumRuntimeErrorCategory.""" + + def test_all_values_exist(self) -> None: + assert EnumRuntimeErrorCategory.KAFKA_CONSUMER == "kafka_consumer" + assert EnumRuntimeErrorCategory.KAFKA_PRODUCER == "kafka_producer" + assert EnumRuntimeErrorCategory.DATABASE == "database" + assert EnumRuntimeErrorCategory.HTTP_CLIENT == "http_client" + assert EnumRuntimeErrorCategory.HTTP_SERVER == "http_server" + assert EnumRuntimeErrorCategory.RUNTIME == "runtime" + assert EnumRuntimeErrorCategory.UNKNOWN == "unknown" + + def test_is_str_enum(self) -> None: + assert isinstance(EnumRuntimeErrorCategory.KAFKA_CONSUMER, str) + + +@pytest.mark.unit +class TestEnumRuntimeErrorSeverity: + """Tests for EnumRuntimeErrorSeverity.""" + + def test_all_values_exist(self) -> None: + assert EnumRuntimeErrorSeverity.WARNING == "warning" + assert EnumRuntimeErrorSeverity.ERROR == "error" + assert EnumRuntimeErrorSeverity.CRITICAL == "critical" + + +@pytest.mark.unit +class TestComputeErrorFingerprint: + """Tests for _compute_error_fingerprint.""" + + def test_deterministic(self) -> None: + fp1 = _compute_error_fingerprint( + "aiokafka.consumer", "heartbeat failed", "kafka_consumer" + ) + fp2 = _compute_error_fingerprint( + "aiokafka.consumer", "heartbeat failed", "kafka_consumer" + ) + assert fp1 == fp2 + + def test_different_inputs_different_fingerprints(self) -> None: + fp1 = _compute_error_fingerprint( + "aiokafka.consumer", "heartbeat failed", "kafka_consumer" + ) + fp2 = _compute_error_fingerprint("asyncpg", "connection refused", "database") + assert fp1 != fp2 + + def test_length(self) -> None: + fp = _compute_error_fingerprint("a", "b", "c") + assert len(fp) == 16 + + +@pytest.mark.unit +class TestModelRuntimeErrorEvent: + """Tests for ModelRuntimeErrorEvent.""" + + def test_create_factory(self) -> None: + event = ModelRuntimeErrorEvent.create( + logger_family="aiokafka.consumer", + log_level="ERROR", + message_template="Heartbeat failed for group {}", + raw_message="Heartbeat failed for group test-group", + error_category=EnumRuntimeErrorCategory.KAFKA_CONSUMER, + severity=EnumRuntimeErrorSeverity.ERROR, + ) + assert event.logger_family == "aiokafka.consumer" + assert event.log_level == "ERROR" + assert event.error_category == EnumRuntimeErrorCategory.KAFKA_CONSUMER + assert len(event.fingerprint) == 16 + assert isinstance(event.event_id, UUID) + assert isinstance(event.correlation_id, UUID) + assert event.occurrence_count_local == 1 + + def test_create_with_exception_info(self) -> None: + event = ModelRuntimeErrorEvent.create( + logger_family="asyncpg", + log_level="ERROR", + message_template="connection refused to {}:{}", + raw_message="connection refused to localhost:5432", + error_category=EnumRuntimeErrorCategory.DATABASE, + severity=EnumRuntimeErrorSeverity.ERROR, + exception_type="ConnectionRefusedError", + exception_message="Connection refused", + stack_trace="Traceback...", + occurrence_count_local=5, + ) + assert event.exception_type == "ConnectionRefusedError" + assert event.exception_message == "Connection refused" + assert event.stack_trace == "Traceback..." + assert event.occurrence_count_local == 5 + + def test_frozen_model(self) -> None: + event = ModelRuntimeErrorEvent.create( + logger_family="test", + log_level="ERROR", + message_template="test", + raw_message="test", + error_category=EnumRuntimeErrorCategory.UNKNOWN, + severity=EnumRuntimeErrorSeverity.ERROR, + ) + with pytest.raises(Exception): + event.logger_family = "other" # type: ignore[misc] + + def test_serialization_roundtrip(self) -> None: + event = ModelRuntimeErrorEvent.create( + logger_family="aiokafka.consumer", + log_level="WARNING", + message_template="slow poll detected", + raw_message="slow poll detected (2.5s)", + error_category=EnumRuntimeErrorCategory.KAFKA_CONSUMER, + severity=EnumRuntimeErrorSeverity.WARNING, + hostname="host-1", + service_label="my-service", + ) + data = event.model_dump(mode="json") + restored = ModelRuntimeErrorEvent.model_validate(data) + assert restored.logger_family == event.logger_family + assert restored.fingerprint == event.fingerprint + assert restored.hostname == "host-1" + + def test_fingerprint_ignores_raw_message(self) -> None: + """Fingerprint uses template, not raw message -- different raw = same fp.""" + e1 = ModelRuntimeErrorEvent.create( + logger_family="test", + log_level="ERROR", + message_template="connection refused to {}", + raw_message="connection refused to host-a", + error_category=EnumRuntimeErrorCategory.DATABASE, + severity=EnumRuntimeErrorSeverity.ERROR, + ) + e2 = ModelRuntimeErrorEvent.create( + logger_family="test", + log_level="ERROR", + message_template="connection refused to {}", + raw_message="connection refused to host-b", + error_category=EnumRuntimeErrorCategory.DATABASE, + severity=EnumRuntimeErrorSeverity.ERROR, + ) + assert e1.fingerprint == e2.fingerprint diff --git a/tests/unit/nodes/test_handler_consumer_health_triage.py b/tests/unit/nodes/test_handler_consumer_health_triage.py new file mode 100644 index 0000000000..416a0a4de1 --- /dev/null +++ b/tests/unit/nodes/test_handler_consumer_health_triage.py @@ -0,0 +1,191 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Unit tests for HandlerConsumerHealthTriage. + +Tests graduated response logic: Slack warning -> repeated -> restart -> Linear. + +Related Tickets: + - OMN-5520: Create NodeConsumerHealthTriageEffect + - OMN-5529: Runtime Health Event Pipeline (epic) +""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from omnibase_infra.models.health.enum_consumer_health_event_type import ( + EnumConsumerHealthEventType, +) +from omnibase_infra.models.health.enum_consumer_health_severity import ( + EnumConsumerHealthSeverity, +) +from omnibase_infra.models.health.model_consumer_health_event import ( + ModelConsumerHealthEvent, +) +from omnibase_infra.nodes.node_consumer_health_triage_effect.handlers.handler_consumer_health_triage import ( + HandlerConsumerHealthTriage, +) + + +def _make_event( + *, + fingerprint: str = "abc123", + event_type: EnumConsumerHealthEventType = EnumConsumerHealthEventType.HEARTBEAT_FAILURE, + severity: EnumConsumerHealthSeverity = EnumConsumerHealthSeverity.ERROR, +) -> ModelConsumerHealthEvent: + """Create a test health event.""" + return ModelConsumerHealthEvent.create( + consumer_identity="test-consumer", + consumer_group="test-group", + topic="test-topic", + event_type=event_type, + severity=severity, + ) + + +@pytest.mark.unit +class TestHandlerConsumerHealthTriage: + """Tests for graduated triage response.""" + + @patch.dict("os.environ", {"ENABLE_CONSUMER_HEALTH_TRIAGE": ""}) + async def test_suppressed_when_disabled(self) -> None: + """Handler returns suppressed when feature flag is off.""" + db_pool = MagicMock() + handler = HandlerConsumerHealthTriage(db_pool=db_pool) + event = _make_event() + + result = await handler.handle(event) + + assert result.action == "suppressed" + assert result.occurrence_count == 0 + + @patch.dict("os.environ", {"ENABLE_CONSUMER_HEALTH_TRIAGE": "true"}) + async def test_first_occurrence_slack_warning(self) -> None: + """First occurrence triggers Slack WARNING.""" + db_pool = MagicMock() + conn = AsyncMock() + conn.fetchrow = AsyncMock( + return_value={"occurrence_count": 1, "incident_state": "open"} + ) + db_pool.acquire = MagicMock( + return_value=AsyncMock( + __aenter__=AsyncMock(return_value=conn), __aexit__=AsyncMock() + ) + ) + + slack_handler = AsyncMock() + handler = HandlerConsumerHealthTriage( + db_pool=db_pool, slack_handler=slack_handler + ) + event = _make_event() + + result = await handler.handle(event) + + assert result.action == "slack_warning" + assert result.occurrence_count == 1 + slack_handler.assert_awaited_once() + + @patch.dict("os.environ", {"ENABLE_CONSUMER_HEALTH_TRIAGE": "true"}) + async def test_second_occurrence_slack_repeated(self) -> None: + """Second occurrence triggers Slack REPEATED.""" + db_pool = MagicMock() + conn = AsyncMock() + conn.fetchrow = AsyncMock( + return_value={"occurrence_count": 2, "incident_state": "open"} + ) + db_pool.acquire = MagicMock( + return_value=AsyncMock( + __aenter__=AsyncMock(return_value=conn), __aexit__=AsyncMock() + ) + ) + + slack_handler = AsyncMock() + handler = HandlerConsumerHealthTriage( + db_pool=db_pool, slack_handler=slack_handler + ) + event = _make_event() + + result = await handler.handle(event) + + assert result.action == "slack_repeated" + assert result.occurrence_count == 2 + + @patch.dict( + "os.environ", + { + "ENABLE_CONSUMER_HEALTH_TRIAGE": "true", + "ENABLE_CONSUMER_AUTO_RESTART": "true", + }, + ) + async def test_third_occurrence_restart_command(self) -> None: + """Third occurrence triggers restart command when auto-restart enabled.""" + db_pool = MagicMock() + conn = AsyncMock() + # First call: upsert returns count=3 + # Second call: rate limit check returns no prior restarts + conn.fetchrow = AsyncMock( + side_effect=[ + {"occurrence_count": 3, "incident_state": "open"}, + None, # No prior restarts + ] + ) + conn.execute = AsyncMock() + db_pool.acquire = MagicMock( + return_value=AsyncMock( + __aenter__=AsyncMock(return_value=conn), __aexit__=AsyncMock() + ) + ) + + producer = AsyncMock() + producer.send = AsyncMock() + handler = HandlerConsumerHealthTriage(db_pool=db_pool, producer=producer) + event = _make_event() + + result = await handler.handle(event) + + assert result.action == "restart_command" + assert result.incident_state == "restart_pending" + producer.send.assert_awaited_once() + + @patch.dict( + "os.environ", + { + "ENABLE_CONSUMER_HEALTH_TRIAGE": "true", + "ENABLE_CONSUMER_AUTO_RESTART": "", + }, + ) + async def test_third_occurrence_no_restart_when_disabled(self) -> None: + """Third occurrence sends Slack repeated when auto-restart is disabled.""" + db_pool = MagicMock() + conn = AsyncMock() + conn.fetchrow = AsyncMock( + return_value={"occurrence_count": 3, "incident_state": "open"} + ) + db_pool.acquire = MagicMock( + return_value=AsyncMock( + __aenter__=AsyncMock(return_value=conn), __aexit__=AsyncMock() + ) + ) + + slack_handler = AsyncMock() + handler = HandlerConsumerHealthTriage( + db_pool=db_pool, slack_handler=slack_handler + ) + event = _make_event() + + result = await handler.handle(event) + + assert result.action == "slack_repeated" + + def test_is_enabled_default_off(self) -> None: + """Feature flag defaults to off.""" + with patch.dict("os.environ", {}, clear=True): + assert not HandlerConsumerHealthTriage.is_enabled() + + def test_is_enabled_on(self) -> None: + """Feature flag responds to truthy values.""" + for val in ("1", "true", "yes", "on"): + with patch.dict("os.environ", {"ENABLE_CONSUMER_HEALTH_TRIAGE": val}): + assert HandlerConsumerHealthTriage.is_enabled() diff --git a/tests/unit/nodes/test_handler_runtime_error_triage.py b/tests/unit/nodes/test_handler_runtime_error_triage.py new file mode 100644 index 0000000000..27e86e8338 --- /dev/null +++ b/tests/unit/nodes/test_handler_runtime_error_triage.py @@ -0,0 +1,301 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Unit tests for HandlerRuntimeErrorTriage. + +Tests first-match-wins triage rule engine, cross-layer correlation, +and default rules for aiokafka/asyncpg/aiohttp. + +Related Tickets: + - OMN-5522: Create NodeRuntimeErrorTriageEffect + - OMN-5529: Runtime Health Event Pipeline (epic) +""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from omnibase_infra.models.health.enum_runtime_error_category import ( + EnumRuntimeErrorCategory, +) +from omnibase_infra.models.health.enum_runtime_error_severity import ( + EnumRuntimeErrorSeverity, +) +from omnibase_infra.models.health.model_runtime_error_event import ( + ModelRuntimeErrorEvent, +) +from omnibase_infra.nodes.node_runtime_error_triage_effect.handlers.handler_runtime_error_triage import ( + HandlerRuntimeErrorTriage, +) +from omnibase_infra.nodes.node_runtime_error_triage_effect.models.model_triage_rule import ( + DEFAULT_TRIAGE_RULES, + ModelTriageRule, +) + + +def _make_error_event( + *, + logger_family: str = "aiokafka.consumer", + error_category: EnumRuntimeErrorCategory = EnumRuntimeErrorCategory.KAFKA_CONSUMER, + raw_message: str = "Heartbeat session expired", + message_template: str = "Heartbeat session expired", +) -> ModelRuntimeErrorEvent: + """Create a test runtime error event.""" + return ModelRuntimeErrorEvent.create( + logger_family=logger_family, + log_level="ERROR", + message_template=message_template, + raw_message=raw_message, + error_category=error_category, + severity=EnumRuntimeErrorSeverity.ERROR, + ) + + +@pytest.mark.unit +class TestModelTriageRule: + """Tests for the triage rule matching logic.""" + + def test_matches_logger_prefix(self) -> None: + """Rule matches on logger prefix.""" + rule = ModelTriageRule(name="test", logger_prefix="aiokafka", action="alert") + assert rule.matches( + "aiokafka.consumer", EnumRuntimeErrorCategory.KAFKA_CONSUMER, "msg" + ) + assert not rule.matches("asyncpg", EnumRuntimeErrorCategory.DATABASE, "msg") + + def test_matches_error_category(self) -> None: + """Rule matches on error category.""" + rule = ModelTriageRule( + name="test", + error_category=EnumRuntimeErrorCategory.DATABASE, + action="alert", + ) + assert rule.matches("asyncpg", EnumRuntimeErrorCategory.DATABASE, "msg") + assert not rule.matches( + "asyncpg", EnumRuntimeErrorCategory.KAFKA_CONSUMER, "msg" + ) + + def test_matches_message_pattern(self) -> None: + """Rule matches on message regex pattern.""" + rule = ModelTriageRule( + name="test", + message_pattern=r"heartbeat|session.*timeout", + action="alert", + ) + assert rule.matches( + "logger", EnumRuntimeErrorCategory.UNKNOWN, "heartbeat failed" + ) + assert rule.matches( + "logger", EnumRuntimeErrorCategory.UNKNOWN, "session has timeout" + ) + assert not rule.matches( + "logger", EnumRuntimeErrorCategory.UNKNOWN, "connection refused" + ) + + def test_matches_all_conditions(self) -> None: + """Rule requires all conditions to match.""" + rule = ModelTriageRule( + name="test", + logger_prefix="aiokafka", + error_category=EnumRuntimeErrorCategory.KAFKA_CONSUMER, + message_pattern=r"heartbeat", + action="alert", + ) + assert rule.matches( + "aiokafka.consumer", + EnumRuntimeErrorCategory.KAFKA_CONSUMER, + "heartbeat failed", + ) + assert not rule.matches( + "aiokafka.consumer", + EnumRuntimeErrorCategory.KAFKA_CONSUMER, + "connection refused", + ) + + def test_empty_conditions_match_all(self) -> None: + """Rule with empty conditions matches everything.""" + rule = ModelTriageRule(name="catch_all", action="alert") + assert rule.matches( + "any.logger", EnumRuntimeErrorCategory.UNKNOWN, "any message" + ) + + def test_default_rules_have_catch_all(self) -> None: + """Default rules include a catch-all at the end.""" + sorted_rules = sorted(DEFAULT_TRIAGE_RULES, key=lambda r: r.priority) + last_rule = sorted_rules[-1] + assert last_rule.name == "catch_all" + assert last_rule.matches( + "anything", EnumRuntimeErrorCategory.UNKNOWN, "anything" + ) + + +@pytest.mark.unit +class TestHandlerRuntimeErrorTriage: + """Tests for the triage handler.""" + + async def test_alert_action(self) -> None: + """Alert action sends Slack notification.""" + db_pool = MagicMock() + conn = AsyncMock() + # Order: correlate_with_layer1 (KAFKA_CONSUMER) -> upsert_incident + conn.fetchrow = AsyncMock( + side_effect=[ + None, # correlation query: no active consumer incidents + {"occurrence_count": 1, "incident_state": "open"}, # upsert + ] + ) + db_pool.acquire = MagicMock( + return_value=AsyncMock( + __aenter__=AsyncMock(return_value=conn), __aexit__=AsyncMock() + ) + ) + + slack_handler = AsyncMock() + rules = [ModelTriageRule(name="alert_all", priority=1, action="alert")] + handler = HandlerRuntimeErrorTriage( + db_pool=db_pool, rules=rules, slack_handler=slack_handler + ) + event = _make_error_event() + + result = await handler.handle(event) + + assert result.action == "alert" + assert result.matched_rule == "alert_all" + slack_handler.assert_awaited_once() + + async def test_suppress_action(self) -> None: + """Suppress action suppresses the incident.""" + db_pool = MagicMock() + conn = AsyncMock() + conn.fetchrow = AsyncMock( + return_value={"occurrence_count": 1, "incident_state": "open"} + ) + conn.execute = AsyncMock() + db_pool.acquire = MagicMock( + return_value=AsyncMock( + __aenter__=AsyncMock(return_value=conn), __aexit__=AsyncMock() + ) + ) + + slack_handler = AsyncMock() + rules = [ModelTriageRule(name="suppress_all", priority=1, action="suppress")] + handler = HandlerRuntimeErrorTriage( + db_pool=db_pool, rules=rules, slack_handler=slack_handler + ) + event = _make_error_event() + + result = await handler.handle(event) + + assert result.action == "suppress" + assert result.incident_state == "suppressed" + + async def test_ticket_action(self) -> None: + """Ticket action creates Linear ticket.""" + db_pool = MagicMock() + conn = AsyncMock() + conn.fetchrow = AsyncMock( + return_value={"occurrence_count": 1, "incident_state": "open"} + ) + conn.execute = AsyncMock() + db_pool.acquire = MagicMock( + return_value=AsyncMock( + __aenter__=AsyncMock(return_value=conn), __aexit__=AsyncMock() + ) + ) + + linear_handler = AsyncMock() + rules = [ModelTriageRule(name="ticket_all", priority=1, action="ticket")] + handler = HandlerRuntimeErrorTriage( + db_pool=db_pool, rules=rules, linear_handler=linear_handler + ) + event = _make_error_event() + + result = await handler.handle(event) + + assert result.action == "ticket" + assert result.incident_state == "ticketed" + linear_handler.assert_awaited_once() + + async def test_cross_layer_correlation_kafka(self) -> None: + """Kafka-related errors trigger cross-layer correlation.""" + db_pool = MagicMock() + conn = AsyncMock() + # Order: correlate_with_layer1 -> upsert_incident + conn.fetchrow = AsyncMock( + side_effect=[ + {"fingerprint": "consumer-fp-123"}, # correlation query + {"occurrence_count": 1, "incident_state": "open"}, # upsert + ] + ) + db_pool.acquire = MagicMock( + return_value=AsyncMock( + __aenter__=AsyncMock(return_value=conn), __aexit__=AsyncMock() + ) + ) + + rules = [ModelTriageRule(name="alert_all", priority=1, action="alert")] + handler = HandlerRuntimeErrorTriage(db_pool=db_pool, rules=rules) + + event = _make_error_event( + error_category=EnumRuntimeErrorCategory.KAFKA_CONSUMER, + ) + result = await handler.handle(event) + + assert result.correlated_consumer_fingerprint == "consumer-fp-123" + + async def test_no_correlation_for_non_kafka(self) -> None: + """Non-Kafka errors do not trigger cross-layer correlation.""" + db_pool = MagicMock() + conn = AsyncMock() + conn.fetchrow = AsyncMock( + return_value={"occurrence_count": 1, "incident_state": "open"} + ) + db_pool.acquire = MagicMock( + return_value=AsyncMock( + __aenter__=AsyncMock(return_value=conn), __aexit__=AsyncMock() + ) + ) + + rules = [ModelTriageRule(name="alert_all", priority=1, action="alert")] + handler = HandlerRuntimeErrorTriage(db_pool=db_pool, rules=rules) + + event = _make_error_event( + logger_family="asyncpg", + error_category=EnumRuntimeErrorCategory.DATABASE, + raw_message="connection refused", + ) + result = await handler.handle(event) + + assert result.correlated_consumer_fingerprint is None + + async def test_first_match_wins_priority(self) -> None: + """Rules are evaluated in priority order; first match wins.""" + db_pool = MagicMock() + conn = AsyncMock() + # Uses DATABASE category to avoid correlation path + conn.fetchrow = AsyncMock( + return_value={"occurrence_count": 1, "incident_state": "open"} + ) + conn.execute = AsyncMock() + db_pool.acquire = MagicMock( + return_value=AsyncMock( + __aenter__=AsyncMock(return_value=conn), __aexit__=AsyncMock() + ) + ) + + rules = [ + ModelTriageRule(name="high_priority", priority=1, action="ticket"), + ModelTriageRule(name="low_priority", priority=100, action="alert"), + ] + handler = HandlerRuntimeErrorTriage(db_pool=db_pool, rules=rules) + event = _make_error_event( + logger_family="asyncpg", + error_category=EnumRuntimeErrorCategory.DATABASE, + raw_message="connection error", + ) + + result = await handler.handle(event) + + assert result.matched_rule == "high_priority" + assert result.action == "ticket" diff --git a/tests/unit/observability/test_runtime_log_event_bridge.py b/tests/unit/observability/test_runtime_log_event_bridge.py new file mode 100644 index 0000000000..9df44a42b4 --- /dev/null +++ b/tests/unit/observability/test_runtime_log_event_bridge.py @@ -0,0 +1,219 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Unit tests for RuntimeLogEventBridge. + +Ticket: OMN-5521 +""" + +from __future__ import annotations + +import logging +from unittest.mock import AsyncMock, patch + +import pytest + +from omnibase_infra.models.health.enum_runtime_error_category import ( + EnumRuntimeErrorCategory, +) +from omnibase_infra.models.health.enum_runtime_error_severity import ( + EnumRuntimeErrorSeverity, +) +from omnibase_infra.observability.runtime_log_event_bridge import ( + RuntimeLogEventBridge, + _categorize_logger, + _log_level_to_severity, + _templatize_message, +) + + +@pytest.mark.unit +class TestTemplatizeMessage: + """Tests for _templatize_message.""" + + def test_replaces_uuid(self) -> None: + msg = "Failed for id 123e4567-e89b-12d3-a456-426614174000" + result = _templatize_message(msg) + assert "123e4567" not in result + assert "{}" in result + + def test_replaces_ip(self) -> None: + msg = "Connection refused to 192.168.1.100:5432" + result = _templatize_message(msg) + assert "192.168.1.100" not in result + + def test_replaces_standalone_numbers(self) -> None: + msg = "Timeout after 30000 ms for partition 5" + result = _templatize_message(msg) + assert "30000" not in result + assert "partition {}" in result + + +@pytest.mark.unit +class TestCategorizeLogger: + """Tests for _categorize_logger.""" + + def test_kafka_consumer(self) -> None: + assert ( + _categorize_logger("aiokafka.consumer.group") + == EnumRuntimeErrorCategory.KAFKA_CONSUMER + ) + + def test_kafka_producer(self) -> None: + assert ( + _categorize_logger("aiokafka.producer") + == EnumRuntimeErrorCategory.KAFKA_PRODUCER + ) + + def test_database(self) -> None: + assert _categorize_logger("asyncpg.pool") == EnumRuntimeErrorCategory.DATABASE + + def test_http_client(self) -> None: + assert ( + _categorize_logger("aiohttp.client") == EnumRuntimeErrorCategory.HTTP_CLIENT + ) + + def test_unknown(self) -> None: + assert ( + _categorize_logger("my.custom.logger") == EnumRuntimeErrorCategory.UNKNOWN + ) + + +@pytest.mark.unit +class TestLogLevelToSeverity: + """Tests for _log_level_to_severity.""" + + def test_critical(self) -> None: + assert ( + _log_level_to_severity(logging.CRITICAL) + == EnumRuntimeErrorSeverity.CRITICAL + ) + + def test_error(self) -> None: + assert _log_level_to_severity(logging.ERROR) == EnumRuntimeErrorSeverity.ERROR + + def test_warning(self) -> None: + assert ( + _log_level_to_severity(logging.WARNING) == EnumRuntimeErrorSeverity.WARNING + ) + + +@pytest.mark.unit +class TestRuntimeLogEventBridge: + """Tests for RuntimeLogEventBridge.""" + + @patch.dict("os.environ", {"ENABLE_RUNTIME_LOG_BRIDGE": "true"}) + def test_emit_enqueues_event(self) -> None: + producer = AsyncMock() + bridge = RuntimeLogEventBridge(producer, hostname="test-host") + + record = logging.LogRecord( + name="aiokafka.consumer", + level=logging.ERROR, + pathname="", + lineno=0, + msg="Heartbeat failed for group test-group", + args=None, + exc_info=None, + ) + bridge.emit(record) + assert bridge._queue.qsize() == 1 + + @patch.dict("os.environ", {"ENABLE_RUNTIME_LOG_BRIDGE": ""}) + def test_emit_skipped_when_disabled(self) -> None: + producer = AsyncMock() + bridge = RuntimeLogEventBridge(producer) + + record = logging.LogRecord( + name="aiokafka.consumer", + level=logging.ERROR, + pathname="", + lineno=0, + msg="test", + args=None, + exc_info=None, + ) + bridge.emit(record) + assert bridge._queue.qsize() == 0 + + @patch.dict("os.environ", {"ENABLE_RUNTIME_LOG_BRIDGE": "true"}) + def test_rate_limiting(self) -> None: + producer = AsyncMock() + bridge = RuntimeLogEventBridge(producer) + + # Emit 5 events (max per window), then 6th should be rate-limited + for i in range(6): + record = logging.LogRecord( + name="aiokafka.consumer", + level=logging.ERROR, + pathname="", + lineno=0, + msg="same error message", + args=None, + exc_info=None, + ) + bridge.emit(record) + + assert bridge._queue.qsize() == 5 + assert bridge.events_rate_limited == 1 + + @patch.dict("os.environ", {"ENABLE_RUNTIME_LOG_BRIDGE": "true"}) + async def test_drain_loop_emits_to_kafka(self) -> None: + producer = AsyncMock() + producer.send = AsyncMock() + bridge = RuntimeLogEventBridge(producer, hostname="test-host") + + record = logging.LogRecord( + name="asyncpg", + level=logging.ERROR, + pathname="", + lineno=0, + msg="connection refused", + args=None, + exc_info=None, + ) + bridge.emit(record) + + await bridge.start() + # Give drain loop a chance to process + import asyncio + + await asyncio.sleep(0.1) + await bridge.stop() + + assert bridge.events_emitted == 1 + producer.send.assert_called_once() + + def test_is_enabled_default_false(self) -> None: + with patch.dict("os.environ", {}, clear=True): + assert not RuntimeLogEventBridge.is_enabled() + + def test_attach_and_detach(self) -> None: + producer = AsyncMock() + bridge = RuntimeLogEventBridge(producer) + test_logger = logging.getLogger("test.attach.target") + + bridge.attach_to_loggers(["test.attach.target"]) + assert bridge in test_logger.handlers + + bridge.detach_from_loggers(["test.attach.target"]) + assert bridge not in test_logger.handlers + + @patch.dict("os.environ", {"ENABLE_RUNTIME_LOG_BRIDGE": "true"}) + def test_circular_prevention(self) -> None: + """Bridge should NOT capture its own log records.""" + producer = AsyncMock() + bridge = RuntimeLogEventBridge(producer) + + # Create a record from the bridge's own module + record = logging.LogRecord( + name="omnibase_infra.observability.runtime_log_event_bridge._bridge", + level=logging.ERROR, + pathname="", + lineno=0, + msg="internal error", + args=None, + exc_info=None, + ) + bridge.emit(record) + # Should be filtered out + assert bridge._queue.qsize() == 0