diff --git a/CLAUDE.md b/CLAUDE.md index a97e85f4e5..788a234b98 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -59,6 +59,466 @@ This project follows a **ZERO BACKWARDS COMPATIBILITY** policy: - **Protocol Resolution** - Use duck typing through protocols, never isinstance - **OnexError Only** - All exceptions converted to OnexError with chaining: `raise OnexError(...) from e` +## 🚨 Infrastructure Error Usage Patterns + +### Error Class Selection Guide + +| Scenario | Error Class | Example | +|----------|-------------|---------| +| Service configuration invalid | `ProtocolConfigurationError` | Missing required config field | +| Secret/credential not found | `SecretResolutionError` | Vault secret missing | +| Cannot connect to service | `InfraConnectionError` | Database connection refused | +| Operation times out | `InfraTimeoutError` | Consul health check timeout | +| Authentication fails | `InfraAuthenticationError` | Invalid API key | +| Service unavailable | `InfraUnavailableError` | Kafka broker down | + +### Error Context Usage + +All infrastructure errors accept `ModelInfraErrorContext` for structured context: + +```python +from uuid import uuid4 +from omnibase_infra.errors import InfraConnectionError, ModelInfraErrorContext +from omnibase_infra.enums import EnumInfraTransportType + +# Create structured context +context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.DATABASE, + operation="execute_query", + target_name="postgresql-primary", + correlation_id=request.correlation_id, # Propagate from request +) + +# Raise with proper error chaining +try: + connection.execute(query) +except Exception as original_error: + raise InfraConnectionError( + "Failed to connect to database", + context=context, + host="db.example.com", # Additional context via kwargs + port=5432, + ) from original_error +``` + +Example: gRPC handler error + +```python +# Example: gRPC handler error +context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.GRPC, + operation="unary_call", + target_name="grpc-service", +) +raise InfraConnectionError("gRPC connection failed", context=context) +``` + +### Correlation ID Assignment Rules + +Correlation IDs enable distributed tracing across infrastructure components: + +1. **Always propagate**: Pass `correlation_id` from incoming requests to error context +2. **Auto-generation**: If no `correlation_id` exists, generate one using `uuid4()` +3. **UUID format**: Use UUID4 format for all new correlation IDs +4. **Include everywhere**: Add `correlation_id` in all error context for tracing + +```python +from uuid import UUID, uuid4 + +# Pattern 1: Propagate from request +correlation_id = request.correlation_id or uuid4() + +# Pattern 2: Generate if not available +context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.KAFKA, + operation="produce_message", + correlation_id=correlation_id, +) + +# Pattern 3: Extract from incoming event +correlation_id = event.metadata.get("correlation_id") +if isinstance(correlation_id, str): + correlation_id = UUID(correlation_id) +``` + +### Error Sanitization Guidelines + +**NEVER include in error messages or context**: +- Passwords, API keys, tokens, secrets +- Full connection strings with credentials +- PII (names, emails, SSNs, phone numbers) +- Internal IP addresses (in production logs) +- Private keys or certificates +- Session tokens or cookies + +**SAFE to include**: +- Service names (e.g., "postgresql", "kafka") +- Operation names (e.g., "connect", "query", "authenticate") +- Correlation IDs (always include for tracing) +- Error codes (e.g., `EnumCoreErrorCode.DATABASE_CONNECTION_ERROR`) +- Sanitized hostnames (e.g., "db.example.com") +- Port numbers +- Retry counts and timeout values +- Resource identifiers (non-sensitive) + +```python +# BAD - Exposes credentials +raise InfraConnectionError( + f"Failed to connect with password={password}", # NEVER DO THIS + context=context, +) + +# GOOD - Sanitized error message +raise InfraConnectionError( + "Failed to connect to database", + context=context, + host="db.example.com", + port=5432, + retry_count=3, +) + +# BAD - Full connection string +raise InfraConnectionError( + f"Connection failed: {connection_string}", # May contain credentials + context=context, +) + +# GOOD - Sanitized connection info +raise InfraConnectionError( + "Connection failed", + context=context, + host=parsed_host, + port=parsed_port, + database=database_name, +) +``` + +### Error Hierarchy Reference + +``` +ModelOnexError (from omnibase_core) +└── RuntimeHostError (base infrastructure error) + β”œβ”€β”€ ProtocolConfigurationError # Config validation failures + β”œβ”€β”€ SecretResolutionError # Secret/credential resolution + β”œβ”€β”€ InfraConnectionError # Connection failures + β”œβ”€β”€ InfraTimeoutError # Operation timeouts + β”œβ”€β”€ InfraAuthenticationError # Auth/authz failures + └── InfraUnavailableError # Resource unavailable +``` + +### Error Code Mapping Reference + +| Error Class | EnumCoreErrorCode | HTTP Equivalent | +|-------------|-------------------|-----------------| +| `ProtocolConfigurationError` | `INVALID_CONFIGURATION` | 400 Bad Request | +| `SecretResolutionError` | `RESOURCE_NOT_FOUND` | 404 Not Found | +| `InfraConnectionError` | **Transport-aware** (see below) | 503 Service Unavailable | +| `InfraTimeoutError` | `TIMEOUT_ERROR` | 504 Gateway Timeout | +| `InfraAuthenticationError` | `AUTHENTICATION_ERROR` | 401 Unauthorized | +| `InfraUnavailableError` | `SERVICE_UNAVAILABLE` | 503 Service Unavailable | + +#### InfraConnectionError Transport-Aware Error Codes + +`InfraConnectionError` automatically selects the appropriate error code based on `context.transport_type`: + +| Transport Type | EnumCoreErrorCode | Rationale | +|----------------|-------------------|-----------| +| `DATABASE` | `DATABASE_CONNECTION_ERROR` | Specific database connection error | +| `HTTP` | `NETWORK_ERROR` | Network-level transport failure | +| `GRPC` | `NETWORK_ERROR` | Network-level transport failure | +| `KAFKA` | `SERVICE_UNAVAILABLE` | Message broker service unavailable | +| `CONSUL` | `SERVICE_UNAVAILABLE` | Service discovery unavailable | +| `VAULT` | `SERVICE_UNAVAILABLE` | Secret management service unavailable | +| `REDIS` | `SERVICE_UNAVAILABLE` | Cache service unavailable | +| `None` (no context) | `SERVICE_UNAVAILABLE` | Generic fallback | + +```python +# Example: Transport-aware error code selection +from omnibase_infra.errors import InfraConnectionError, ModelInfraErrorContext +from omnibase_infra.enums import EnumInfraTransportType + +# Database connection -> DATABASE_CONNECTION_ERROR +db_context = ModelInfraErrorContext(transport_type=EnumInfraTransportType.DATABASE) +db_error = InfraConnectionError("DB failed", context=db_context) +assert db_error.model.error_code.name == "DATABASE_CONNECTION_ERROR" + +# HTTP connection -> NETWORK_ERROR +http_context = ModelInfraErrorContext(transport_type=EnumInfraTransportType.HTTP) +http_error = InfraConnectionError("API failed", context=http_context) +assert http_error.model.error_code.name == "NETWORK_ERROR" + +# Kafka connection -> SERVICE_UNAVAILABLE +kafka_context = ModelInfraErrorContext(transport_type=EnumInfraTransportType.KAFKA) +kafka_error = InfraConnectionError("Kafka failed", context=kafka_context) +assert kafka_error.model.error_code.name == "SERVICE_UNAVAILABLE" +``` + +### Error Recovery Patterns + +Infrastructure errors often require recovery strategies. Here are common patterns for handling infrastructure failures: + +#### Retry with Exponential Backoff (Connection Errors) + +Use exponential backoff for transient connection failures. This pattern is ideal for `InfraConnectionError` when services are temporarily unavailable: + +```python +import time +from uuid import uuid4 +from omnibase_infra.errors import InfraConnectionError, ModelInfraErrorContext +from omnibase_infra.enums import EnumInfraTransportType + +def connect_with_retry(host: str, port: int, max_retries: int = 3) -> Connection: + """Connect to database with exponential backoff retry strategy.""" + correlation_id = uuid4() + context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.DATABASE, + operation="connect", + target_name="postgresql-primary", + correlation_id=correlation_id, + ) + + for attempt in range(max_retries): + try: + return create_connection(host, port) + except ConnectionError as e: + if attempt == max_retries - 1: + raise InfraConnectionError( + f"Failed to connect after {max_retries} attempts", + context=context, + host=host, + port=port, + retry_count=attempt + 1, + ) from e + + # Exponential backoff: 1s, 2s, 4s + wait_time = 2 ** attempt + time.sleep(wait_time) +``` + +#### Circuit Breaker Pattern (Unavailable Services) + +Use the circuit breaker pattern for `InfraUnavailableError` to prevent cascading failures and give services time to recover: + +```python +import time +from enum import Enum +from omnibase_infra.errors import InfraUnavailableError, ModelInfraErrorContext +from omnibase_infra.enums import EnumInfraTransportType + +class CircuitState(str, Enum): + """Circuit breaker state machine.""" + CLOSED = "closed" # Normal operation + OPEN = "open" # Blocking requests + HALF_OPEN = "half_open" # Testing recovery + +class CircuitBreaker: + """Prevents cascading failures with configurable circuit breaker.""" + + def __init__( + self, + failure_threshold: int = 5, + reset_timeout: float = 30.0, + context: ModelInfraErrorContext = None, + ): + self.failure_count = 0 + self.threshold = failure_threshold + self.reset_timeout = reset_timeout + self.last_failure_time = 0.0 + self.state = CircuitState.CLOSED + self.context = context or ModelInfraErrorContext( + transport_type=EnumInfraTransportType.HTTP, + operation="circuit_breaker", + target_name="service", + ) + + def call(self, func, *args, **kwargs): + """Execute function through circuit breaker protection.""" + if self.state == CircuitState.OPEN: + # Check if reset timeout has passed + if time.time() - self.last_failure_time > self.reset_timeout: + self.state = CircuitState.HALF_OPEN + self.failure_count = 0 + else: + raise InfraUnavailableError( + "Circuit breaker is open - service temporarily unavailable", + context=self.context, + circuit_state=self.state.value, + retry_after_seconds=int( + self.reset_timeout - (time.time() - self.last_failure_time) + ), + ) + + try: + result = func(*args, **kwargs) + + # Success - reset circuit + if self.state == CircuitState.HALF_OPEN: + self.state = CircuitState.CLOSED + self.failure_count = 0 + return result + + except Exception as e: + self.failure_count += 1 + self.last_failure_time = time.time() + + # Open circuit if threshold exceeded + if self.failure_count >= self.threshold: + self.state = CircuitState.OPEN + + raise +``` + +#### Graceful Degradation (Timeout Errors) + +Use graceful degradation for `InfraTimeoutError` to maintain service availability with reduced functionality: + +```python +from omnibase_infra.errors import InfraTimeoutError, ModelInfraErrorContext +from omnibase_infra.enums import EnumInfraTransportType + +def fetch_with_timeout_fallback( + primary_func, + fallback_func, + timeout_seconds: float = 5.0, + correlation_id = None, +) -> dict: + """Fetch from primary source with graceful degradation to fallback.""" + import signal + + context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.DATABASE, + operation="fetch", + target_name="primary-source", + correlation_id=correlation_id, + ) + + def timeout_handler(signum, frame): + raise TimeoutError("Operation exceeded timeout") + + # Set timeout handler + signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(int(timeout_seconds)) + + try: + # Try primary data source + return {"data": primary_func(), "source": "primary", "degraded": False} + + except TimeoutError as e: + # Log timeout but continue with fallback + context_with_fallback = ModelInfraErrorContext( + transport_type=context.transport_type, + operation=context.operation, + target_name=context.target_name, + correlation_id=context.correlation_id, + ) + + try: + # Use fallback source (cache, secondary database, etc.) + return { + "data": fallback_func(), + "source": "fallback", + "degraded": True, + "warning": f"Primary source timed out, using fallback data", + } + + except Exception as fallback_error: + raise InfraTimeoutError( + "Primary timeout and fallback failed", + context=context_with_fallback, + timeout_seconds=timeout_seconds, + ) from fallback_error + + finally: + signal.alarm(0) # Cancel alarm +``` + +#### Credential Refresh (Authentication Errors) + +Use credential refresh for `InfraAuthenticationError` to handle token expiration gracefully: + +```python +from omnibase_infra.errors import InfraAuthenticationError, ModelInfraErrorContext +from omnibase_infra.enums import EnumInfraTransportType +import time + +class CredentialRefreshManager: + """Manages credential refresh with automatic token renewal.""" + + def __init__( + self, + credential_provider, + refresh_threshold_seconds: float = 300.0, + ): + self.provider = credential_provider + self.refresh_threshold = refresh_threshold_seconds + self.current_credential = None + self.credential_expires_at = 0.0 + self.context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.VAULT, + operation="credential_refresh", + target_name="vault-server", + ) + + def get_valid_credential(self): + """Get credential, refreshing if near expiration.""" + current_time = time.time() + + # Check if credential exists and is still valid + if ( + self.current_credential is not None + and current_time < self.credential_expires_at - self.refresh_threshold + ): + return self.current_credential + + # Credential missing, expired, or approaching expiration - refresh + try: + credential = self.provider.refresh_credential() + self.current_credential = credential + self.credential_expires_at = ( + current_time + credential.get("ttl_seconds", 3600) + ) + return credential + + except Exception as e: + raise InfraAuthenticationError( + "Failed to refresh authentication credentials", + context=self.context, + provider="vault", + ) from e + + def call_with_auth(self, func, *args, **kwargs): + """Execute function with automatic credential refresh on auth failure.""" + max_retries = 2 + + for attempt in range(max_retries): + try: + credential = self.get_valid_credential() + return func(*args, credential=credential, **kwargs) + + except InfraAuthenticationError as e: + if attempt == max_retries - 1: + # Last attempt failed - propagate error + raise + + # Force refresh and retry + self.current_credential = None + self.credential_expires_at = 0.0 +``` + +### Transport Type Reference + +Use `EnumInfraTransportType` for transport identification in error context: + +| Transport Type | Value | Usage | +|---------------|-------|-------| +| `HTTP` | `"http"` | REST API transport | +| `DATABASE` | `"db"` | PostgreSQL, etc. | +| `KAFKA` | `"kafka"` | Kafka message broker | +| `CONSUL` | `"consul"` | Service discovery | +| `VAULT` | `"vault"` | Secret management | +| `REDIS` | `"redis"` | Cache/message transport | +| `GRPC` | `"grpc"` | gRPC protocol | + ## πŸ—οΈ Infrastructure-Specific Patterns ### Service Integration Architecture diff --git a/docs/CURRENT_NODE_ARCHITECTURE.md b/docs/CURRENT_NODE_ARCHITECTURE.md deleted file mode 100644 index b744c793ea..0000000000 --- a/docs/CURRENT_NODE_ARCHITECTURE.md +++ /dev/null @@ -1,927 +0,0 @@ -# ONEX Current Node Architecture (Pre-Runtime Host Migration) - -This document describes the current ONEX node architecture that uses a **1-container-per-node** deployment model. This is the "before" state that will be migrated to the new Runtime Host model. - ---- - -## 1. Overview - -### Current Architecture: 1 Container Per Node - -In the current ONEX architecture, each node runs as an **independent container** with its own: - -- Python runtime environment -- Entry point (`node.py` with `if __name__ == "__main__"`) -- Container injection setup -- Kafka consumer/producer connections -- Health check endpoint - -**Deployment Model:** -``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ Container 1 β”‚ β”‚ Container 2 β”‚ β”‚ Container 3 β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ VaultNode β”‚ β”‚ β”‚ β”‚ConsulNode β”‚ β”‚ β”‚ β”‚ KafkaNode β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β”‚ asyncio.run β”‚ β”‚ asyncio.run β”‚ β”‚ asyncio.run β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ β”‚ β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β”Œβ”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β” - β”‚ Kafka β”‚ - β”‚ Event Bus β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -### The 4 Node Types - -ONEX follows a strict **4-node architecture** pattern: - -| Node Type | Base Class | Purpose | I/O Operations | -|-----------|------------|---------|----------------| -| **EFFECT** | `NodeEffectService` | External I/O (APIs, DB, files) | Yes (`io_operations`) | -| **COMPUTE** | `NodeComputeService` | Pure transforms/algorithms | No | -| **REDUCER** | `NodeReducerService` | Aggregation/persistence | No (DB via adapters) | -| **ORCHESTRATOR** | `NodeOrchestratorService` | Workflow coordination | No | - -**Communication Pattern:** -``` -Adapters (EFFECT) β†’ Events β†’ Reducer β†’ Intents β†’ Orchestrator β†’ Workflows β†’ Adapters -``` - ---- - -## 2. Node Directory Structure - -### Standard Node Structure - -``` -nodes//v1_0_0/ -β”œβ”€β”€ __init__.py # Package initialization -β”œβ”€β”€ node.py # Main node implementation with entry point -β”œβ”€β”€ contract.yaml # Node contract definition -β”œβ”€β”€ models/ # Node-specific models -β”‚ β”œβ”€β”€ __init__.py -β”‚ β”œβ”€β”€ model__input.py -β”‚ └── model__output.py -└── registry/ # Dependency injection registry (optional) - └── __init__.py -``` - -### Naming Conventions - -- **Directory Name**: `node__` (e.g., `node_vault_adapter_effect`) -- **Node Class**: `Node` in CamelCase (e.g., `NodeVaultAdapterEffect`) -- **Models**: `ModelInput`, `ModelOutput` -- **Files**: All snake_case (`model_vault_adapter_input.py`) - ---- - -## 3. Full Example: Effect Node (Vault Adapter) - -### File Tree - -``` -nodes/node_vault_adapter_effect/v1_0_0/ -β”œβ”€β”€ __init__.py -β”œβ”€β”€ node.py # 706 lines - Main implementation -β”œβ”€β”€ contract.yaml # 177 lines - Contract definition -β”œβ”€β”€ models/ -β”‚ β”œβ”€β”€ __init__.py -β”‚ β”œβ”€β”€ model_vault_adapter_input.py # Input model for envelope payloads -β”‚ └── model_vault_adapter_output.py # Output model for results -└── registry/ - └── __init__.py -``` - -### node.py - Key Sections - -**Imports and Base Class:** -```python -#!/usr/bin/env python3 - -import asyncio -import logging -import os -from typing import Any - -from omnibase_core.core.errors.onex_error import CoreErrorCode, OnexError -from omnibase_core.core.node_effect_service import NodeEffectService -from omnibase_core.core.onex_container import ModelONEXContainer -from omnibase_core.enums.enum_health_status import EnumHealthStatus -from omnibase_core.models.core.model_health_status import ModelHealthStatus - -from omnibase_infra.models.vault import ( - ModelVaultSecretRequest, - ModelVaultSecretResponse, - ModelVaultTokenRequest, -) -``` - -**Node Class Definition:** -```python -class NodeVaultAdapterEffect(NodeEffectService): - """ - Vault Adapter - Event-Driven Secret Management Effect - - NodeEffect that processes event envelopes to perform Vault operations. - Integrates with event bus for secret management, token lifecycle, - and encryption services. Provides health check HTTP endpoint for monitoring. - """ - - def __init__(self, container: ModelONEXContainer): - # Use proper base class - no more boilerplate! - super().__init__(container) - - self.node_type = "effect" - self.domain = "infrastructure" - - # ONEX logger initialization with fallback - try: - self.logger = getattr(container, "get_tool", lambda x: None)( - "LOGGER", - ) or logging.getLogger(__name__) - except (AttributeError, Exception): - self.logger = logging.getLogger(__name__) - - # Vault client configuration - all environment variables required - vault_addr = os.getenv("VAULT_ADDR") - vault_token = os.getenv("VAULT_TOKEN") - vault_namespace = os.getenv("VAULT_NAMESPACE", "") - - if not vault_addr: - raise OnexError( - message="VAULT_ADDR environment variable is required but not set", - error_code=CoreErrorCode.MISSING_REQUIRED_PARAMETER, - ) - # ... configuration continues -``` - -**Resource Lifecycle Methods:** -```python - async def _initialize_node_resources(self) -> None: - """Override to initialize vault client.""" - await super()._initialize_node_resources() - await self.initialize_vault_client() - - async def _cleanup_node_resources(self) -> None: - """Override to cleanup vault connection pool resources.""" - if self.vault_connection_pool: - await self.vault_connection_pool.close_all() - await super()._cleanup_node_resources() -``` - -**Health Check Method:** -```python - def health_check(self) -> ModelHealthStatus: - """Check Vault service health and connectivity.""" - try: - client = self._get_vault_client() - - if client is None: - return ModelHealthStatus( - status=EnumHealthStatus.UNREACHABLE, - message="Vault client is not initialized", - ) - - health = client.sys.read_health_status(method="GET") - - if not health.get("initialized", False): - return ModelHealthStatus( - status=EnumHealthStatus.UNHEALTHY, - message="Vault is not initialized", - details=health, - ) - - if health.get("sealed", True): - return ModelHealthStatus( - status=EnumHealthStatus.UNHEALTHY, - message="Vault is sealed", - details=health, - ) - - return ModelHealthStatus( - status=EnumHealthStatus.HEALTHY, - message=f"Vault is healthy (version: {health.get('version', 'unknown')})", - details={...}, - ) - except Exception as e: - return ModelHealthStatus( - status=EnumHealthStatus.UNREACHABLE, - message=f"Vault health check failed: {str(e)}", - ) -``` - -**Entry Point (1-Container-Per-Node Pattern):** -```python -# Entry point for running the node -if __name__ == "__main__": - import sys - - # Create container (simplified for standalone operation) - container = ModelONEXContainer() - - # Create and run the node - node = NodeVaultAdapterEffect(container) - - # Run the node with asyncio - try: - asyncio.run(node.run()) - except KeyboardInterrupt: - print("\nVault adapter shutting down...") - sys.exit(0) -``` - -### contract.yaml - Full Structure - -```yaml -name: "vault_adapter" -contract_name: "vault_adapter" -node_name: "vault_adapter" -version: - major: 1 - minor: 0 - patch: 0 -contract_version: "1.0.0" -node_version: "1.0.0" - -node_type: "EFFECT" - -description: > - HashiCorp Vault secret management adapter for secure credential storage and retrieval. - Message bus bridge pattern for Vault operations including secret management, - token operations, and encryption services. - -capabilities: - - name: "secret_management" - description: "Read, write, delete, and list secrets in Vault" - - name: "token_management" - description: "Create, renew, revoke, and lookup Vault tokens" - - name: "encryption_services" - description: "Encrypt and decrypt data using Vault transit engine" - - name: "lease_management" - description: "Manage secret leases and renewals" - - name: "health_monitoring" - description: "Monitor Vault health and seal status" - -input_model: "ModelVaultAdapterInput" -output_model: "ModelVaultAdapterOutput" - -io_operations: - - operation: "get_secret" - description: "Retrieve secret from Vault" - input_fields: - - path - - version - - mount_path - output_fields: - - data - - metadata - - version - - - operation: "set_secret" - description: "Store secret in Vault" - input_fields: - - path - - data - - mount_path - output_fields: - - version - - created_time - - - operation: "delete_secret" - description: "Delete secret from Vault" - input_fields: - - path - - mount_path - output_fields: - - success - - - operation: "health_check" - description: "Check Vault health and seal status" - input_fields: [] - output_fields: - - initialized - - sealed - - standby - - version - -dependencies: - - name: "protocol_event_bus" - type: "protocol" - class_name: "ProtocolEventBus" - module: "omnibase_spi.protocols.event_bus" - - - name: "model_vault_secret_request" - type: "model" - class_name: "ModelVaultSecretRequest" - module: "omnibase_infra.models.vault.model_vault_secret_request" - - - name: "model_vault_secret_response" - type: "model" - class_name: "ModelVaultSecretResponse" - module: "omnibase_infra.models.vault.model_vault_secret_response" - -definitions: - ModelVaultAdapterInput: - type: object - description: "Input model for Vault adapter operations" - properties: - operation: - type: string - description: "Operation to perform (get_secret, set_secret, etc.)" - path: - type: string - description: "Vault secret path" - data: - type: object - description: "Secret data for write operations" - correlation_id: - type: string - description: "Request correlation ID" - required: - - operation - - correlation_id - - ModelVaultAdapterOutput: - type: object - description: "Output model for Vault adapter operations" - properties: - success: - type: boolean - description: "Whether operation succeeded" - data: - type: object - description: "Response data" - error: - type: string - description: "Error message if failed" - correlation_id: - type: string - description: "Request correlation ID" - required: - - success - - correlation_id - -metadata: - author: "ONEX Infrastructure Team" - created: "2025-11-14" - tags: - - vault - - secrets - - security - - adapter - - effect -``` - -### Model Files - -**model_vault_adapter_input.py:** -```python -#!/usr/bin/env python3 - -from typing import Literal -from pydantic import BaseModel, Field - - -class ModelVaultAdapterInput(BaseModel): - """Input model for Vault adapter operations from event envelopes. - - Node-specific model for processing event envelope payloads into Vault operations. - """ - - action: Literal[ - "vault_get_secret", - "vault_set_secret", - "vault_delete_secret", - "vault_list_secrets", - "vault_create_token", - "vault_renew_token", - "vault_revoke_token", - "vault_health_check", - ] = Field(description="Vault operation to perform") - - # Secret operation parameters - path: str | None = Field(default=None, description="Secret path in Vault") - mount_path: str = Field(default="secret", description="Vault mount path") - secret_data: dict | None = Field(default=None, description="Secret data for write operations") - version: int | None = Field(default=None, description="Secret version to retrieve") - - # Token operation parameters - token: str | None = Field(default=None, description="Token for renew/revoke operations") - policies: list[str] | None = Field(default=None, description="Policies for token creation") - ttl: str | None = Field(default=None, description="Token TTL (e.g., '768h')") - renewable: bool = Field(default=True, description="Whether token is renewable") - - # Common fields - correlation_id: str = Field(description="Correlation ID for request tracking") -``` - -**model_vault_adapter_output.py:** -```python -#!/usr/bin/env python3 - -from pydantic import BaseModel, Field -from omnibase_infra.models.vault.model_vault_secret_response import ( - ModelVaultSecretResponse, -) - - -class ModelVaultAdapterOutput(BaseModel): - """Output model for Vault adapter operation results. - - Node-specific model for returning Vault operation results through effect outputs. - """ - - vault_operation_result: ( - ModelVaultSecretResponse - | dict[str, str | int | bool | list | None] - | str - | bool - ) = Field(description="Result of Vault operation") - - success: bool = Field(description="Whether the operation succeeded") - operation_type: str = Field(description="Type of Vault operation performed") - correlation_id: str = Field(description="Correlation ID from request") -``` - ---- - -## 4. Full Example: Effect Node (Consul Projector) - -### File Tree - -``` -nodes/node_consul_projector_effect/v1_0_0/ -β”œβ”€β”€ __init__.py -β”œβ”€β”€ node.py # Main implementation -β”œβ”€β”€ contract.yaml # 342 lines - Contract definition -β”œβ”€β”€ models/ -β”‚ β”œβ”€β”€ __init__.py -β”‚ β”œβ”€β”€ model_consul_cache_entry.py -β”‚ β”œβ”€β”€ model_consul_health_projection.py -β”‚ β”œβ”€β”€ model_consul_kv_details.py -β”‚ β”œβ”€β”€ model_consul_kv_projection.py -β”‚ β”œβ”€β”€ model_consul_kv_summary.py -β”‚ β”œβ”€β”€ model_consul_projection_type.py -β”‚ β”œβ”€β”€ model_consul_projections.py -β”‚ β”œβ”€β”€ model_consul_projector_input.py -β”‚ β”œβ”€β”€ model_consul_projector_output.py -β”‚ β”œβ”€β”€ model_consul_service_projection.py -β”‚ β”œβ”€β”€ model_consul_topology_graph.py -β”‚ β”œβ”€β”€ model_consul_topology_metrics.py -β”‚ └── model_consul_topology_projection.py -└── registry/ - └── __init__.py -``` - -### node.py - Key Sections - -```python -#!/usr/bin/env python3 - -import asyncio -import logging -from datetime import UTC, datetime - -from omnibase_core.core.errors.onex_error import CoreErrorCode, OnexError -from omnibase_core.core.node_effect_service import NodeEffectService -from omnibase_core.core.onex_container import ModelONEXContainer -from omnibase_core.enums.enum_health_status import EnumHealthStatus -from omnibase_core.models.core.model_health_status import ModelHealthStatus - -# Import node-specific models -from .models import ( - ModelConsulHealthCacheEntry, - ModelConsulHealthProjection, - ModelConsulKVCacheEntry, - ModelConsulKVProjection, - ModelConsulProjectorInput, - ModelConsulProjectorOutput, - ModelConsulServiceCacheEntry, - ModelConsulServiceProjection, - ModelConsulTopologyProjection, -) - - -class NodeConsulProjectorEffect(NodeEffectService): - """ - Consul Projector - Event-Driven Infrastructure State Projector - - NodeEffect that processes Consul state data to create projected views and aggregations. - Integrates with event bus for event-driven state projection and monitoring. - Provides comprehensive state views for service discovery, health monitoring, and topology analysis. - """ - - def __init__(self, container: ModelONEXContainer): - super().__init__(container) - - self.node_type = "effect" - self.domain = "infrastructure" - - # ONEX logger initialization with fallback - try: - self.logger = getattr(container, "get_tool", lambda x: None)( - "LOGGER", - ) or logging.getLogger(__name__) - except (AttributeError, Exception): - self.logger = logging.getLogger(__name__) - - # State cache for projection optimization with strong typing - self._service_cache: dict[str, ModelConsulServiceCacheEntry] = {} - self._health_cache: dict[str, ModelConsulHealthCacheEntry] = {} - self._kv_cache: dict[str, ModelConsulKVCacheEntry] = {} - self._cache_ttl: int = 300 # 5 minutes - - self._initialized = False - - async def project_service_state(self, input_data: ModelConsulProjectorInput) -> ModelConsulServiceProjection: - """Project current service state from Consul data.""" - # ... projection logic - - async def project_health_state(self, input_data: ModelConsulProjectorInput) -> ModelConsulHealthProjection: - """Project health state aggregation from Consul data.""" - # ... projection logic - - async def project_kv_state(self, input_data: ModelConsulProjectorInput) -> ModelConsulKVProjection: - """Project KV store state changes from Consul data.""" - # ... projection logic - - async def project_topology(self, input_data: ModelConsulProjectorInput) -> ModelConsulTopologyProjection: - """Project service topology view from Consul data.""" - # ... topology generation logic - - def health_check(self) -> ModelHealthStatus: - """Single comprehensive health check for Consul projector.""" - try: - if not self._initialized: - return ModelHealthStatus( - status=EnumHealthStatus.UNHEALTHY, - message="Consul projector not initialized", - ) - - cache_health = len(self._service_cache) + len(self._health_cache) + len(self._kv_cache) - - if cache_health == 0: - return ModelHealthStatus( - status=EnumHealthStatus.DEGRADED, - message="Consul projector operational but caches empty", - ) - - return ModelHealthStatus( - status=EnumHealthStatus.HEALTHY, - message=f"Consul projector healthy - cache entries: {cache_health}", - ) - except Exception as e: - return ModelHealthStatus( - status=EnumHealthStatus.UNREACHABLE, - message=f"Consul projector health check failed: {e!s}", - ) - - -# Entry point for running the node -if __name__ == "__main__": - import sys - - container = ModelONEXContainer() - node = NodeConsulProjectorEffect(container) - - try: - asyncio.run(node.run()) - except KeyboardInterrupt: - print("\nConsul projector shutting down...") - sys.exit(0) -``` - ---- - -## 5. Contract YAML Structure - -### Required Fields - -| Field | Type | Description | -|-------|------|-------------| -| `name` | string | Short identifier for the node | -| `contract_name` | string | Full contract identifier | -| `node_name` | string | Node identifier | -| `version` | object | Semantic version `{major, minor, patch}` | -| `contract_version` | string | Contract version string | -| `node_version` | string | Node implementation version | -| `node_type` | enum | One of: `EFFECT`, `COMPUTE`, `REDUCER`, `ORCHESTRATOR` | -| `description` | string | Human-readable description | -| `input_model` | string | Name of input model class | -| `output_model` | string | Name of output model class | -| `dependencies` | array | List of required protocols and models | -| `definitions` | object | Model definitions for input/output | - -### Effect Node Specific Fields - -```yaml -io_operations: - - operation: "operation_name" - description: "What this operation does" - input_fields: - - field1 - - field2 - output_fields: - - result_field -``` - -### Orchestrator Specific Fields - -```yaml -workflows: - workflow_name: - name: "WorkflowClassName" - description: "What this workflow does" - trigger: "event_type_that_triggers" - steps: - - step: "step_name" - action: "action_to_perform" - depends_on: ["previous_step"] - output: "output_variable" - -intent_consumption: - subscribed_intents: - - "intent_type_1" - - "intent_type_2" - intent_routing_table: - intent_type_1: "workflow_to_trigger" -``` - -### Reducer Specific Fields - -```yaml -event_consumption: - subscribed_topics: - - "topic-name-1" - - "topic-name-2" - consumer_group: "reducer_consumer_group" - consumed_event_types: - - "EVENT_TYPE_1" - - "EVENT_TYPE_2" - -intent_emission: - published_intents: - - "intent_type_1" - - "intent_type_2" - intent_routing: "orchestrator_name" - -state_schema: - tables: - - name: "table_name" - columns: - - name: "column_name" - type: "uuid" - primary_key: true -``` - ---- - -## 6. Node Base Classes - -### Import Patterns - -All nodes import from `omnibase_core`: - -```python -# Base classes (pick one based on node type) -from omnibase_core.core.node_effect_service import NodeEffectService -from omnibase_core.base.node_compute_service import NodeComputeService -from omnibase_core.core.node_reducer_service import NodeReducerService -from omnibase_core.core.node_orchestrator_service import NodeOrchestratorService - -# Common imports -from omnibase_core.core.errors.onex_error import CoreErrorCode, OnexError -from omnibase_core.core.onex_container import ModelONEXContainer -from omnibase_core.enums.enum_health_status import EnumHealthStatus -from omnibase_core.models.core.model_health_status import ModelHealthStatus -``` - -### Base Class Methods - -**NodeEffectService:** -```python -class NodeEffectService: - def __init__(self, container: ModelONEXContainer): ... - async def _initialize_node_resources(self) -> None: ... - async def _cleanup_node_resources(self) -> None: ... - async def run(self) -> None: ... # Main event loop - def health_check(self) -> ModelHealthStatus: ... -``` - -**NodeComputeService[TInput, TOutput]:** -```python -class NodeComputeService(Generic[TInput, TOutput]): - def __init__(self, container: ModelONEXContainer): ... - async def initialize(self) -> None: ... - async def compute(self, input_data: TInput) -> TOutput: ... # Pure transformation -``` - -**NodeReducerService:** -```python -class NodeReducerService: - def __init__(self, container: ModelONEXContainer): ... - async def reduce(self, input_data: TInput) -> TOutput: ... # Aggregation - async def initialize(self) -> None: ... - async def cleanup(self) -> None: ... -``` - -**NodeOrchestratorService:** -```python -class NodeOrchestratorService: - def __init__(self, container: ModelONEXContainer): ... - async def orchestrate(self, input_data: TInput) -> TOutput: ... # Workflow coordination - async def initialize(self) -> None: ... - async def cleanup(self) -> None: ... - async def health_check(self) -> dict: ... -``` - ---- - -## 7. Deployment Model - -### Current: 1 Container Per Node - -Each node is deployed as an independent Docker container: - -```dockerfile -# Example Dockerfile for a node -FROM python:3.11-slim - -WORKDIR /app -COPY . . -RUN pip install -e . - -# Each node has its own entry point -CMD ["python", "-m", "omnibase_infra.nodes.node_vault_adapter_effect.v1_0_0.node"] -``` - -**Docker Compose Example:** -```yaml -services: - vault-adapter: - build: . - command: python -m omnibase_infra.nodes.node_vault_adapter_effect.v1_0_0.node - environment: - - VAULT_ADDR=http://vault:8200 - - VAULT_TOKEN=${VAULT_TOKEN} - - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 - depends_on: - - kafka - - vault - - consul-projector: - build: . - command: python -m omnibase_infra.nodes.node_consul_projector_effect.v1_0_0.node - environment: - - CONSUL_ADDR=http://consul:8500 - - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 - depends_on: - - kafka - - consul - - kafka-adapter: - build: . - command: python -m omnibase_infra.nodes.kafka_adapter.v1_0_0.node - environment: - - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 - depends_on: - - kafka -``` - -### Node Communication - -Nodes communicate exclusively via **Kafka topics**: - -``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” publish β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ Effect Node β”‚ ─────────────────► β”‚ Kafka β”‚ -β”‚ (vault_adapter) β”‚ β”‚ Event Bus β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β”‚ subscribe - β–Ό - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ Reducer Node β”‚ - β”‚ (omni_reducer) β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β”‚ emit intent - β–Ό - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ Orchestrator β”‚ - β”‚ (omni_orchestr) β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -### Entry Point Pattern - -Every node has the same entry point pattern: - -```python -# Entry point for running the node -if __name__ == "__main__": - import sys - - # Create container (simplified for standalone operation) - container = ModelONEXContainer() - - # Create and run the node - node = NodeClassName(container) - - # Run the node with asyncio - try: - asyncio.run(node.run()) - except KeyboardInterrupt: - print("\nNode shutting down...") - sys.exit(0) -``` - ---- - -## 8. Limitations (Why We're Migrating) - -### Resource Overhead - -**Problem:** Each node requires its own container with: -- Full Python runtime (~150MB base image) -- Separate Kafka consumer connections -- Independent health check endpoints -- Duplicate dependency installations - -**Impact:** With 15+ nodes, this results in significant resource waste: -``` -15 nodes Γ— 150MB = 2.25GB+ for container images alone -15 nodes Γ— 1 Kafka connection = 15 Kafka connections -``` - -### Complex Deployment - -**Problem:** Each node needs separate: -- Docker image builds -- Kubernetes deployments/pods -- Service definitions -- ConfigMaps/Secrets -- Health check probes - -**Impact:** Managing 15+ separate deployments becomes unwieldy: -- Difficult to coordinate rolling updates -- Complex dependency management -- Increased Kubernetes resource definitions - -### No Shared Handler Infrastructure - -**Problem:** Common functionality is duplicated: -- Kafka consumer setup in every node -- Health check endpoints repeated -- Error handling patterns duplicated -- Logging configuration repeated - -**Impact:** -- Code duplication across nodes -- Inconsistent error handling -- Difficult to add cross-cutting concerns - -### Scaling Limitations - -**Problem:** Cannot scale node types independently: -- Must scale entire container for one node -- Cannot collocate related nodes efficiently -- Memory-intensive nodes affect all - -### Migration Path: Runtime Host Model - -The new **Runtime Host** model addresses these limitations by: -- Running multiple nodes in a single container -- Sharing Kafka connections and infrastructure -- Providing unified health check endpoints -- Enabling efficient resource sharing -- Simplifying deployment to a single host - -``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ Runtime Host β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚VaultNode β”‚ β”‚ConsulNode β”‚ β”‚OrchestratorNd β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β”‚ β”‚ -β”‚ Shared: Kafka, Health, Logging, Error Handling β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - ---- - -## Summary - -This document captures the current ONEX node architecture with: - -1. **1-container-per-node deployment** - Each node runs independently -2. **4 node types** - EFFECT, COMPUTE, REDUCER, ORCHESTRATOR -3. **Standard directory structure** - `nodes//v1_0_0/` -4. **Contract-driven configuration** - `contract.yaml` defines everything -5. **Base classes from omnibase_core** - Consistent inheritance patterns -6. **Kafka-based communication** - Event bus for all inter-node messaging - -The migration to Runtime Host will preserve the node contracts and logic while fundamentally changing the deployment model for improved efficiency. diff --git a/docs/DECLARATIVE_EFFECT_NODES_PLAN.md b/docs/DECLARATIVE_EFFECT_NODES_PLAN.md deleted file mode 100644 index 64583bed3f..0000000000 --- a/docs/DECLARATIVE_EFFECT_NODES_PLAN.md +++ /dev/null @@ -1,1623 +0,0 @@ -# Effect Nodes Plan for omnibase_infra - -**Created**: December 2, 2025 -**Status**: Planning -**Dependencies**: MVP_EXECUTION_PLAN.md, EFFECT_NODES_SPEC.md (omniintelligence) -**Target**: omnibase-core ^0.4.0 (post contract-driven effect release) - ---- - -## Executive Summary - -This document outlines the integration of contract-driven effect nodes into `omnibase_infra`, building on the existing MVP execution plan. The contract-driven pattern will eliminate boilerplate code, standardize resilience patterns, and enable runtime configuration changes without code deployment. - -**Naming Convention Change (December 2025)**: Contract-driven nodes are now the DEFAULT implementation and use standard names (no suffix). Previous imperative implementations are renamed with a `Legacy` suffix during the migration period. - -**Key Benefits**: -- Reduce legacy imperative effect node code by ~60% -- Standardize retry, circuit breaker, and DLQ patterns -- Enable infrastructure-as-code for all effect nodes -- Simplify testing through protocol handler mocking - ---- - -## Naming Convention - -### Standard Names (Contract-Driven - DEFAULT) - -Contract-driven effect nodes use standard names without any suffix. These are the default implementation: - -| Class Name | Description | -|------------|-------------| -| `NodeEffect` | Base class for contract-driven effect nodes | -| `NodePostgresAdapter` | PostgreSQL adapter (contract-driven) | -| `NodeKafkaAdapter` | Kafka adapter (contract-driven) | -| `NodeConsulAdapter` | Consul adapter (contract-driven) | -| `NodeVaultAdapterEffect` | Vault adapter (contract-driven) | -| `NodeKeycloakAdapterEffect` | Keycloak adapter (contract-driven) | - -### Legacy Names (Imperative - Deprecated) - -Previous imperative implementations are renamed with a `Legacy` suffix during the migration period: - -| Legacy Class Name | Original Name | Deprecation Timeline | -|-------------------|---------------|----------------------| -| `NodeEffectLegacy` | `NodeEffect` | Remove in v0.5.0 | -| `NodePostgresAdapterLegacy` | `NodePostgresAdapter` | Remove in v0.5.0 | -| `NodeKafkaAdapterLegacy` | `NodeKafkaAdapter` | Remove in v0.5.0 | -| `NodeConsulAdapterLegacy` | `NodeConsulAdapter` | Remove in v0.5.0 | -| `NodeVaultAdapterEffectLegacy` | `NodeVaultAdapterEffect` | Remove in v0.5.0 | -| `NodeKeycloakAdapterEffectLegacy` | `NodeKeycloakAdapterEffect` | Remove in v0.5.0 | - -### Migration Path - -1. **Phase 1 (v0.4.0)**: Legacy suffix added to imperative nodes, contract-driven nodes take standard names -2. **Phase 2 (v0.4.x)**: Deprecation warnings on Legacy imports -3. **Phase 3 (v0.5.0)**: Legacy classes removed entirely - -### Import Examples - -```python -# NEW (v0.4.0+): Contract-driven effect nodes (DEFAULT) -from omnibase_core.nodes import NodeEffect # Base class -from omnibase_infra.nodes import NodePostgresAdapter # Contract-driven - -# DEPRECATED (v0.4.0-v0.4.x): Legacy imperative nodes -from omnibase_core.nodes import NodeEffectLegacy # Will be removed in v0.5.0 -from omnibase_infra.nodes import NodePostgresAdapterLegacy # Will be removed in v0.5.0 -``` - ---- - -## Core Design Invariants - -These invariants apply to all contract-driven effect nodes: - -1. **All behavior is contract-driven** - YAML contracts define everything -2. **NodeRuntime is the only executable event loop** - Effect nodes don't run their own loops -3. **Node logic is pure: no I/O, no mixins, no inheritance** -4. **Core never depends on SPI or infra** - Dependency: infra -> spi -> core -5. **SPI only defines protocols, never implementations** -6. **Infra owns all I/O and real system integrations** - Handlers live in infra - -**Critical Invariant**: -``` -No code in omnibase_core may initiate network I/O, database I/O, -file I/O, or external process execution. -``` - -This means: -- Handler implementations (HTTP, Postgres, Kafka, Bolt) MUST be in omnibase_infra -- Protocol interfaces are defined in omnibase_spi -- NodeRuntime and NodeInstance are in omnibase_core - ---- - -## Handler Placement Rule - -**Rule**: Any code that touches the network, filesystem, sockets, or external systems -MUST live in `omnibase_infra`. Core and SPI must remain pure and dependency-free. - -**Correct Structure**: -``` -omnibase_infra/ - handlers/ - http/ - http_rest_handler.py - http_retry_policy.py - http_circuit_breaker.py - db/ - postgres_handler.py - connection_pool.py - graph/ - bolt_handler.py - event/ - kafka_handler.py - resilience/ - retry_policy.py - circuit_breaker.py - rate_limiter.py - runtime_host/ - entrypoint.py - wiring.py -``` - ---- - -## Integration with Runtime Host Model - -Contract-driven effect nodes are designed to work with the Runtime Host model: - -``` -RuntimeHostProcess (omnibase_infra) - +-- NodeRuntime (omnibase_core) - |-- NodeInstance(contract-driven effect 1) - |-- NodeInstance(contract-driven effect 2) - +-- handlers: - postgres_handler (infra) - kafka_handler (infra) - http_rest_handler (infra) -``` - -**Key Points**: -- Contract-driven effects are loaded as NodeInstances -- Handlers are registered with the runtime at startup -- Effect contracts declare which handler types they need -- Runtime routes operations to appropriate handlers - ---- - -## 1. Integration with Existing MVP Plan - -### 1.1 Timeline Integration - -The contract-driven effect work slots into the MVP as a **Phase 2.5** enhancement: - -| Phase | Description | Status | Contract-Driven Impact | -|-------|-------------|--------|------------------------| -| Phase 0 | Pre-flight | Existing | No change | -| Phase 1 | Foundation | Existing | No change | -| Phase 2 | Effect Nodes (Legacy) | Existing | **Baseline, renamed to Legacy** | -| **Phase 2.5** | **Contract-Driven Effect Migration** | **NEW** | **New nodes take standard names** | -| Phase 3 | Stamping Service | Existing | Can use contract-driven Postgres | -| Phase 4 | Contract-Driven Nodes (Reducer/Orchestrator) | Existing | Foundation for effect contracts | -| Phase 5 | Compute Nodes | Existing | No change | -| Phase 6 | Testing & Validation | Existing | Extended for contracts | - -### 1.2 Dependency Chain - -``` -omnibase-spi ^0.2.0 (protocol definitions) - | - +-- ProtocolHandler protocol (abstract, no I/O) - +-- Protocol contracts and interfaces - | - v -omnibase-core ^0.4.0 (contract-driven effects release) - | - +-- NodeEffect base class (contract-driven, DEFAULT) - +-- NodeEffectLegacy base class (imperative, DEPRECATED) - +-- NodeRuntime (event loop, pure orchestration) - +-- NodeInstance (pure node logic) - +-- Effect contract JSON schema - | - v -omnibase_infra ^0.2.0 (post-migration) - | - +-- YAML effect contracts - +-- Protocol handlers (HTTP, Bolt, Postgres, Kafka) - ALL I/O HERE - +-- ProtocolHandlerRegistry (manages handler lifecycle) - +-- Resilience utilities (RetryPolicy, CircuitBreaker) - +-- RuntimeHostProcess (wiring and entrypoint) - +-- NodePostgresAdapter, NodeKafkaAdapter, etc. (contract-driven) - +-- NodePostgresAdapterLegacy, etc. (deprecated, for fallback) - +-- Infrastructure-specific operations -``` - -### 1.3 Parallel vs Sequential Work - -**Can Run in Parallel with MVP**: -- Contract YAML design (no code dependency) -- Protocol handler implementation in omnibase_infra -- Documentation and examples - -**Must Wait for omnibase-core 0.4.0**: -- Actual node conversion -- Integration testing -- Runtime validation - ---- - -## 2. Effect Nodes Inventory - -### 2.1 Current Effect Nodes in omnibase_infra - -| Node | Location | Protocol | Priority | Effort | -|------|----------|----------|----------|--------| -| `postgres_adapter` | `nodes/postgres_adapter/v1_0_0/` | postgres | P0 | Medium | -| `kafka_adapter` | `nodes/kafka_adapter/v1_0_0/` | kafka | P0 | Medium | -| `consul_adapter` | `nodes/consul_adapter/v1_0_0/` | http_rest | P1 | Low | -| `node_vault_adapter_effect` | `nodes/node_vault_adapter_effect/v1_0_0/` | http_rest | P1 | Low | -| `node_keycloak_adapter_effect` | `nodes/node_keycloak_adapter_effect/v1_0_0/` | http_rest | P2 | Medium | -| `hook_node` (webhook) | `nodes/hook_node/v1_0_0/` | http_rest | P2 | Low | -| `node_consul_projector_effect` | `nodes/node_consul_projector_effect/v1_0_0/` | http_rest | P3 | Low | - -### 2.2 Priority Justification - -**P0 - Foundation (Week 1)**: -- **Postgres**: Workflow state storage, FSM transitions - core to all operations -- **Kafka**: Event publishing - enables loose coupling between services - -**P1 - Service Integration (Week 2)**: -- **Consul**: Service discovery, health checks - required for deployment -- **Vault**: Secret management - security critical but can use env vars initially - -**P2 - Extended Features (Week 3)**: -- **Keycloak**: Authentication flows - can defer to later if using API keys -- **Webhook**: Notification delivery - enhancement, not critical path - -**P3 - Projections (Week 4)**: -- **Consul Projector**: Read-model projection - optimization, not core - -### 2.3 Effort Estimation - -| Effort Level | Hours | Description | -|--------------|-------|-------------| -| Low | 2-4h | Simple HTTP REST, 3-5 operations | -| Medium | 4-8h | Complex protocol, many operations, custom validation | -| High | 8-16h | New protocol handler, extensive testing | - ---- - -## 3. YAML Contracts to Create - -### 3.1 Contract Directory Structure - -``` -omnibase_infra/ -β”œβ”€β”€ contracts/ -β”‚ └── effects/ -β”‚ β”œβ”€β”€ _schema/ -β”‚ β”‚ └── effect_contract_schema.json # JSON Schema for validation -β”‚ β”œβ”€β”€ postgres_workflow.yaml # Workflow state persistence -β”‚ β”œβ”€β”€ postgres_fsm.yaml # FSM transition storage -β”‚ β”œβ”€β”€ kafka_event.yaml # Event publishing -β”‚ β”œβ”€β”€ consul_registry.yaml # Service discovery -β”‚ β”œβ”€β”€ consul_kv.yaml # KV store operations -β”‚ β”œβ”€β”€ vault_secret.yaml # Secret management -β”‚ β”œβ”€β”€ keycloak_auth.yaml # Authentication -β”‚ β”œβ”€β”€ webhook_delivery.yaml # Webhook notifications -β”‚ └── valkey_cache.yaml # Caching (future) -└── src/omnibase_infra/ - β”œβ”€β”€ handlers/ # At package root, NOT under nodes - β”‚ β”œβ”€β”€ __init__.py - β”‚ β”œβ”€β”€ http/ - β”‚ β”‚ β”œβ”€β”€ http_rest_handler.py # HTTP REST protocol handler - β”‚ β”‚ β”œβ”€β”€ http_retry_policy.py # HTTP-specific retry logic - β”‚ β”‚ └── http_circuit_breaker.py # HTTP circuit breaker - β”‚ β”œβ”€β”€ db/ - β”‚ β”‚ β”œβ”€β”€ postgres_handler.py # PostgreSQL protocol handler - β”‚ β”‚ └── connection_pool.py # Connection pool management - β”‚ β”œβ”€β”€ graph/ - β”‚ β”‚ └── bolt_handler.py # Neo4j Bolt protocol handler - β”‚ β”œβ”€β”€ event/ - β”‚ β”‚ └── kafka_handler.py # Kafka protocol handler - β”‚ └── cache/ - β”‚ └── valkey_handler.py # Valkey (Redis-compatible) handler - β”œβ”€β”€ resilience/ # Shared resilience patterns - β”‚ β”œβ”€β”€ __init__.py - β”‚ β”œβ”€β”€ retry_policy.py # Generic retry policy - β”‚ β”œβ”€β”€ circuit_breaker.py # Generic circuit breaker - β”‚ └── rate_limiter.py # Rate limiting utilities - β”œβ”€β”€ runtime_host/ # Runtime host wiring - β”‚ β”œβ”€β”€ entrypoint.py # Process entrypoint - β”‚ └── wiring.py # Handler registration - └── nodes/ # Nodes are separate from handlers - β”œβ”€β”€ effect_nodes/ # Contract-driven effect nodes (DEFAULT) - β”‚ β”œβ”€β”€ __init__.py - β”‚ └── loader.py # Contract loader utility - └── effect_nodes_legacy/ # Legacy imperative nodes (DEPRECATED) - β”œβ”€β”€ __init__.py - └── ... # To be removed in v0.5.0 -``` - -**Important**: Handlers live at the package root (`src/omnibase_infra/handlers/`), NOT under -`nodes/`. This follows the architectural invariant that handlers contain I/O code which must -remain separate from pure node logic. - -### 3.2 postgres_workflow.yaml - -```yaml -# Workflow state persistence for infrastructure orchestration -name: postgres_workflow_effect -version: - major: 1 - minor: 0 - patch: 0 - -description: | - PostgreSQL effect for workflow state persistence. - Stores workflow execution states, checkpoints, and audit logs. - -protocol: - type: postgres - version: "14" - -connection: - host: ${POSTGRES_HOST} - port: ${POSTGRES_PORT} - database: ${POSTGRES_DB} - pool: - min_size: 2 - max_size: 10 - max_idle_time_ms: 300000 - timeout_ms: 30000 - tls: - enabled: ${POSTGRES_TLS_ENABLED} - verify: true - -authentication: - type: basic - basic: - username: ${POSTGRES_USER} - password: ${POSTGRES_PASSWORD} - -operations: - save_workflow_state: - description: "Persist workflow execution state" - request: - sql: | - INSERT INTO workflow_states ( - workflow_id, execution_id, state, payload, checkpoint, created_at - ) VALUES ($1, $2, $3, $4, $5, NOW()) - ON CONFLICT (workflow_id, execution_id) - DO UPDATE SET state = $3, payload = $4, checkpoint = $5, updated_at = NOW() - RETURNING id, workflow_id, execution_id, state - sql_params: - - ${input.workflow_id} - - ${input.execution_id} - - ${input.state} - - ${input.payload} - - ${input.checkpoint} - response: - mapping: - record_id: "$.rows[0].id" - workflow_id: "$.rows[0].workflow_id" - validation: - required_fields: - - workflow_id - - execution_id - - state - - get_workflow_state: - description: "Retrieve workflow state by execution ID" - request: - sql: | - SELECT id, workflow_id, execution_id, state, payload, checkpoint, created_at, updated_at - FROM workflow_states - WHERE workflow_id = $1 AND execution_id = $2 - sql_params: - - ${input.workflow_id} - - ${input.execution_id} - response: - mapping: - state: "$.rows[0].state" - payload: "$.rows[0].payload" - checkpoint: "$.rows[0].checkpoint" - - list_workflow_history: - description: "List workflow execution history" - request: - sql: | - SELECT id, workflow_id, execution_id, state, created_at, updated_at - FROM workflow_states - WHERE workflow_id = $1 - ORDER BY created_at DESC - LIMIT $2 OFFSET $3 - sql_params: - - ${input.workflow_id} - - ${input.limit} - - ${input.offset} - response: - mapping: - executions: "$.rows" - count: "$.row_count" - - delete_workflow_state: - description: "Delete workflow state (for cleanup)" - request: - sql: | - DELETE FROM workflow_states - WHERE workflow_id = $1 AND execution_id = $2 - RETURNING id - sql_params: - - ${input.workflow_id} - - ${input.execution_id} - response: - mapping: - deleted: "$.affected_rows" - -resilience: - retry: - enabled: true - max_attempts: 3 - initial_delay_ms: 100 - max_delay_ms: 2000 - backoff_multiplier: 2.0 - jitter: true - circuit_breaker: - enabled: true - failure_threshold: 5 - success_threshold: 2 - timeout_ms: 30000 - timeout: - request_ms: 5000 - operation_ms: 30000 - -events: - consume: - topic: dev.omnibase-infra.effect.postgres-workflow.request.v1 - group_id: postgres-workflow-effect-consumer - produce: - success_topic: dev.omnibase-infra.effect.postgres-workflow.response.v1 - failure_topic: dev.omnibase-infra.effect.postgres-workflow.failure.v1 - dlq_topic: dev.omnibase-infra.effect.postgres-workflow.request.v1.dlq - -observability: - metrics: - enabled: true - prefix: omnibase_infra_postgres_workflow - labels: - service: omnibase_infra - component: postgres_workflow_effect - logging: - level: INFO - sanitize_secrets: true - secret_patterns: - - password - - secret - - token - -metadata: - author: OmniInfra Team - created_at: "2025-12-02" - tags: - - infrastructure - - persistence - - workflow - documentation: https://docs.omninode.ai/infra/effects/postgres-workflow -``` - -### 3.3 postgres_fsm.yaml - -```yaml -# FSM state transition storage -name: postgres_fsm_effect -version: - major: 1 - minor: 0 - patch: 0 - -description: | - PostgreSQL effect for FSM state transition persistence. - Stores state machine states, transitions, and audit history. - -protocol: - type: postgres - version: "14" - -connection: - host: ${POSTGRES_HOST} - port: ${POSTGRES_PORT} - database: ${POSTGRES_DB} - pool: - min_size: 2 - max_size: 10 - timeout_ms: 30000 - -authentication: - type: basic - basic: - username: ${POSTGRES_USER} - password: ${POSTGRES_PASSWORD} - -operations: - record_transition: - description: "Record an FSM state transition" - request: - sql: | - INSERT INTO fsm_transitions ( - entity_id, entity_type, fsm_type, from_state, to_state, - event, context, correlation_id, created_at - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW()) - RETURNING id, entity_id, from_state, to_state - sql_params: - - ${input.entity_id} - - ${input.entity_type} - - ${input.fsm_type} - - ${input.from_state} - - ${input.to_state} - - ${input.event} - - ${input.context} - - ${context.correlation_id} - response: - mapping: - transition_id: "$.rows[0].id" - entity_id: "$.rows[0].entity_id" - validation: - required_fields: - - entity_id - - entity_type - - fsm_type - - from_state - - to_state - - event - - get_current_state: - description: "Get current FSM state for an entity" - request: - sql: | - SELECT to_state as current_state, event as last_event, created_at as last_transition - FROM fsm_transitions - WHERE entity_id = $1 AND entity_type = $2 AND fsm_type = $3 - ORDER BY created_at DESC - LIMIT 1 - sql_params: - - ${input.entity_id} - - ${input.entity_type} - - ${input.fsm_type} - response: - mapping: - current_state: "$.rows[0].current_state" - last_event: "$.rows[0].last_event" - last_transition: "$.rows[0].last_transition" - - get_transition_history: - description: "Get FSM transition history for an entity" - request: - sql: | - SELECT id, from_state, to_state, event, context, correlation_id, created_at - FROM fsm_transitions - WHERE entity_id = $1 AND entity_type = $2 AND fsm_type = $3 - ORDER BY created_at DESC - LIMIT $4 OFFSET $5 - sql_params: - - ${input.entity_id} - - ${input.entity_type} - - ${input.fsm_type} - - ${input.limit} - - ${input.offset} - response: - mapping: - transitions: "$.rows" - count: "$.row_count" - -resilience: - retry: - enabled: true - max_attempts: 3 - initial_delay_ms: 100 - max_delay_ms: 2000 - circuit_breaker: - enabled: true - failure_threshold: 5 - timeout_ms: 30000 - -events: - consume: - topic: dev.omnibase-infra.effect.postgres-fsm.request.v1 - group_id: postgres-fsm-effect-consumer - produce: - success_topic: dev.omnibase-infra.effect.postgres-fsm.response.v1 - dlq_topic: dev.omnibase-infra.effect.postgres-fsm.request.v1.dlq - -observability: - metrics: - enabled: true - prefix: omnibase_infra_postgres_fsm - logging: - level: INFO - sanitize_secrets: true - -metadata: - author: OmniInfra Team - created_at: "2025-12-02" - tags: - - infrastructure - - fsm - - state-management -``` - -### 3.4 kafka_event.yaml - -```yaml -# Kafka event publishing for infrastructure events -name: kafka_event_effect -version: - major: 1 - minor: 0 - patch: 0 - -description: | - Kafka effect for event publishing. - Publishes infrastructure events with delivery confirmation. - -protocol: - type: kafka - version: "3.0" - -connection: - url: ${KAFKA_BOOTSTRAP_SERVERS} - timeout_ms: 30000 - pool: - max_size: 5 - -authentication: - type: none # SASL config can be added if needed - -operations: - publish_event: - description: "Publish an event to a Kafka topic" - request: - topic: ${input.topic} - key: ${input.key} - payload: ${input.payload} - response: - mapping: - partition: "$.partition" - offset: "$.offset" - topic: "$.topic" - validation: - required_fields: - - topic - - payload - - publish_batch: - description: "Publish multiple events to a topic" - request: - topic: ${input.topic} - messages: ${input.messages} - response: - mapping: - published_count: "$.count" - partitions: "$.partitions" - validation: - required_fields: - - topic - - messages - - publish_workflow_event: - description: "Publish a workflow lifecycle event" - request: - topic: dev.omnibase-infra.workflow.${input.event_type}.v1 - key: ${input.workflow_id} - payload: - event_id: ${context.event_id} - event_type: ${input.event_type} - workflow_id: ${input.workflow_id} - execution_id: ${input.execution_id} - state: ${input.state} - timestamp: ${context.timestamp} - correlation_id: ${context.correlation_id} - response: - mapping: - partition: "$.partition" - offset: "$.offset" - - publish_fsm_transition: - description: "Publish an FSM transition event" - request: - topic: dev.omnibase-infra.fsm.transition.v1 - key: ${input.entity_id} - payload: - event_id: ${context.event_id} - entity_id: ${input.entity_id} - entity_type: ${input.entity_type} - fsm_type: ${input.fsm_type} - from_state: ${input.from_state} - to_state: ${input.to_state} - event: ${input.event} - timestamp: ${context.timestamp} - correlation_id: ${context.correlation_id} - response: - mapping: - partition: "$.partition" - offset: "$.offset" - -resilience: - retry: - enabled: true - max_attempts: 5 - initial_delay_ms: 500 - max_delay_ms: 10000 - backoff_multiplier: 2.0 - circuit_breaker: - enabled: true - failure_threshold: 10 - success_threshold: 3 - timeout_ms: 60000 - timeout: - request_ms: 10000 - operation_ms: 60000 - -observability: - metrics: - enabled: true - prefix: omnibase_infra_kafka_event - labels: - service: omnibase_infra - logging: - level: INFO - include_request_body: false - -metadata: - author: OmniInfra Team - created_at: "2025-12-02" - tags: - - infrastructure - - events - - kafka -``` - -### 3.5 consul_registry.yaml - -```yaml -# Consul service registry operations -name: consul_registry_effect -version: - major: 1 - minor: 0 - patch: 0 - -description: | - Consul effect for service registration and discovery. - Manages service lifecycle in the Consul catalog. - -protocol: - type: http_rest - version: "HTTP/1.1" - content_type: application/json - -connection: - url: http://${CONSUL_HOST}:${CONSUL_PORT} - timeout_ms: 10000 - pool: - max_size: 5 - -authentication: - type: api_key - api_key: - header: X-Consul-Token - prefix: "" - value: ${CONSUL_ACL_TOKEN} - -operations: - register_service: - description: "Register a service with Consul" - request: - method: PUT - path: /v1/agent/service/register - body: - ID: ${input.service_id} - Name: ${input.service_name} - Tags: ${input.tags} - Address: ${input.address} - Port: ${input.port} - Check: - HTTP: http://${input.address}:${input.port}${input.health_path} - Interval: ${input.check_interval} - Timeout: ${input.check_timeout} - response: - success_codes: [200] - mapping: - registered: "true" - validation: - required_fields: - - service_id - - service_name - - address - - port - - deregister_service: - description: "Deregister a service from Consul" - request: - method: PUT - path: /v1/agent/service/deregister/${input.service_id} - response: - success_codes: [200] - mapping: - deregistered: "true" - validation: - required_fields: - - service_id - - get_service: - description: "Get service details by ID" - request: - method: GET - path: /v1/catalog/service/${input.service_name} - response: - success_codes: [200] - mapping: - services: "$[*]" - count: "$.length" - - list_services: - description: "List all registered services" - request: - method: GET - path: /v1/catalog/services - response: - success_codes: [200] - mapping: - services: "$" - - health_check: - description: "Get health status of a service" - request: - method: GET - path: /v1/health/service/${input.service_name} - query: - passing: ${input.passing_only} - response: - success_codes: [200] - mapping: - instances: "$[*]" - healthy_count: "$.length" - -resilience: - retry: - enabled: true - max_attempts: 3 - initial_delay_ms: 500 - circuit_breaker: - enabled: true - failure_threshold: 5 - timeout_ms: 30000 - -events: - consume: - topic: dev.omnibase-infra.effect.consul-registry.request.v1 - group_id: consul-registry-effect-consumer - produce: - success_topic: dev.omnibase-infra.effect.consul-registry.response.v1 - dlq_topic: dev.omnibase-infra.effect.consul-registry.request.v1.dlq - -observability: - metrics: - enabled: true - prefix: omnibase_infra_consul_registry - -metadata: - author: OmniInfra Team - created_at: "2025-12-02" - tags: - - infrastructure - - service-discovery - - consul -``` - -### 3.6 valkey_cache.yaml - -```yaml -# Valkey (Redis-compatible) caching operations -name: valkey_cache_effect -version: - major: 1 - minor: 0 - patch: 0 - -description: | - Valkey effect for caching operations. - Provides high-performance key-value caching with TTL support. - -protocol: - type: valkey # Custom handler required - version: "7.0" - -connection: - host: ${VALKEY_HOST} - port: ${VALKEY_PORT} - database: ${VALKEY_DB} - pool: - min_size: 2 - max_size: 20 - timeout_ms: 5000 - tls: - enabled: ${VALKEY_TLS_ENABLED} - -authentication: - type: basic - basic: - password: ${VALKEY_PASSWORD} - -operations: - get: - description: "Get a value by key" - request: - command: GET - key: ${input.key} - response: - mapping: - value: "$.value" - exists: "$.exists" - validation: - required_fields: - - key - - set: - description: "Set a value with optional TTL" - request: - command: SET - key: ${input.key} - value: ${input.value} - ttl_seconds: ${input.ttl} - response: - mapping: - success: "$.ok" - validation: - required_fields: - - key - - value - - delete: - description: "Delete a key" - request: - command: DEL - key: ${input.key} - response: - mapping: - deleted: "$.count" - - get_many: - description: "Get multiple values by keys" - request: - command: MGET - keys: ${input.keys} - response: - mapping: - values: "$.values" - - set_many: - description: "Set multiple key-value pairs" - request: - command: MSET - pairs: ${input.pairs} - response: - mapping: - success: "$.ok" - - increment: - description: "Increment a numeric value" - request: - command: INCR - key: ${input.key} - amount: ${input.amount} - response: - mapping: - new_value: "$.value" - - expire: - description: "Set TTL on an existing key" - request: - command: EXPIRE - key: ${input.key} - ttl_seconds: ${input.ttl} - response: - mapping: - success: "$.ok" - -resilience: - retry: - enabled: true - max_attempts: 2 - initial_delay_ms: 50 - max_delay_ms: 500 - circuit_breaker: - enabled: true - failure_threshold: 10 - timeout_ms: 10000 - timeout: - request_ms: 1000 - operation_ms: 5000 - -events: - consume: - topic: dev.omnibase-infra.effect.valkey-cache.request.v1 - group_id: valkey-cache-effect-consumer - produce: - success_topic: dev.omnibase-infra.effect.valkey-cache.response.v1 - dlq_topic: dev.omnibase-infra.effect.valkey-cache.request.v1.dlq - -observability: - metrics: - enabled: true - prefix: omnibase_infra_valkey_cache - histograms: - buckets: [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0] - logging: - level: DEBUG - include_request_body: false - -metadata: - author: OmniInfra Team - created_at: "2025-12-02" - tags: - - infrastructure - - caching - - valkey - dependencies: - - valkey_handler # Custom protocol handler -``` - ---- - -## 4. Integration with omnibase_core - -### 4.1 Importing NodeEffect (Contract-Driven) - -```python -"""Example of using contract-driven effect nodes in omnibase_infra.""" - -from pathlib import Path -from omnibase_core.nodes import NodeEffect # Contract-driven base class (DEFAULT) -from omnibase_core.models import ModelEffectInput - -# Path to contracts directory -CONTRACTS_DIR = Path(__file__).parent.parent / "contracts" / "effects" - -async def create_postgres_workflow_effect() -> NodeEffect: - """Create and initialize Postgres workflow effect node.""" - node = NodeEffect( - contract_path=CONTRACTS_DIR / "postgres_workflow.yaml", - config_overrides={ - # Override env vars if needed for testing - "POSTGRES_HOST": "localhost", - "POSTGRES_PORT": "5432", - } - ) - await node.initialize() - return node - -async def save_workflow_state( - node: NodeEffect, - workflow_id: str, - execution_id: str, - state: str, - payload: dict -) -> dict: - """Save workflow state using contract-driven effect.""" - result = await node.execute_effect( - ModelEffectInput( - operation="save_workflow_state", - params={ - "workflow_id": workflow_id, - "execution_id": execution_id, - "state": state, - "payload": payload, - "checkpoint": None, - } - ) - ) - - if not result.success: - raise RuntimeError(f"Failed to save workflow state: {result.error}") - - return result.data -``` - -### 4.1.1 Legacy Fallback (Deprecated) - -For migration purposes, you can still use the legacy imperative nodes: - -```python -"""Legacy imperative nodes - DEPRECATED, will be removed in v0.5.0.""" - -import warnings -from omnibase_core.nodes import NodeEffectLegacy # DEPRECATED - -# This will emit a deprecation warning -warnings.warn( - "NodeEffectLegacy is deprecated and will be removed in v0.5.0. " - "Use NodeEffect with YAML contracts instead.", - DeprecationWarning, - stacklevel=2 -) -``` - -### 4.2 Registering Custom Protocol Handlers - -**Architectural Note**: All handler implementations live in `omnibase_infra`, never in -`omnibase_core`. Core only contains pure logic (NodeRuntime, NodeInstance). Protocol -interfaces (abstract base classes) are defined in `omnibase_spi`. - -For custom protocols (e.g., Valkey): - -```python -"""Custom Valkey protocol handler for omnibase_infra. - -Location: src/omnibase_infra/handlers/cache/valkey_handler.py - -This handler implements the ProtocolHandler protocol from omnibase_spi -and provides the actual I/O operations for Valkey/Redis. -""" - -from typing import Any -# Protocol interface from SPI (abstract, no I/O) -from omnibase_spi.protocols import ProtocolHandler -# Models can be shared or defined in infra -from omnibase_infra.handlers.models import ModelProtocolRequest, ModelProtocolResponse -# Handler registry lives in infra (manages I/O components) -from omnibase_infra.handlers.registry import ProtocolHandlerRegistry -import valkey # or redis-py with valkey compatibility - -class ValkeyHandler(ProtocolHandler): - """Protocol handler for Valkey (Redis-compatible) operations.""" - - def __init__(self): - self.client: valkey.Redis | None = None - - async def initialize(self, config: dict[str, Any]) -> None: - """Initialize Valkey connection pool.""" - self.client = valkey.Redis( - host=config.get("host", "localhost"), - port=config.get("port", 6379), - db=config.get("database", 0), - password=config.get("authentication", {}).get("basic", {}).get("password"), - decode_responses=True, - max_connections=config.get("pool", {}).get("max_size", 10), - ) - - async def shutdown(self) -> None: - """Close Valkey connection.""" - if self.client: - await self.client.close() - - async def execute( - self, - request: ModelProtocolRequest, - operation_config: dict[str, Any] - ) -> ModelProtocolResponse: - """Execute Valkey command.""" - import time - start_time = time.perf_counter() - - try: - req_config = operation_config.get("request", {}) - command = req_config.get("command", "").upper() - - # Route to appropriate Valkey command - if command == "GET": - value = await self.client.get(request.params.get("input", {}).get("key")) - data = {"value": value, "exists": value is not None} - elif command == "SET": - params = request.params.get("input", {}) - ttl = params.get("ttl") - if ttl: - result = await self.client.setex(params["key"], ttl, params["value"]) - else: - result = await self.client.set(params["key"], params["value"]) - data = {"ok": result} - elif command == "DEL": - count = await self.client.delete(request.params.get("input", {}).get("key")) - data = {"count": count} - elif command == "MGET": - keys = request.params.get("input", {}).get("keys", []) - values = await self.client.mget(keys) - data = {"values": dict(zip(keys, values))} - elif command == "INCR": - params = request.params.get("input", {}) - amount = params.get("amount", 1) - if amount == 1: - value = await self.client.incr(params["key"]) - else: - value = await self.client.incrby(params["key"], amount) - data = {"value": value} - else: - raise ValueError(f"Unsupported Valkey command: {command}") - - duration_ms = (time.perf_counter() - start_time) * 1000 - - return ModelProtocolResponse( - success=True, - data=data, - duration_ms=duration_ms - ) - - except Exception as e: - duration_ms = (time.perf_counter() - start_time) * 1000 - return ModelProtocolResponse( - success=False, - error=str(e), - duration_ms=duration_ms - ) - - async def health_check(self) -> bool: - """Check Valkey connectivity.""" - try: - return await self.client.ping() - except Exception: - return False - -# Register custom handler at module load -def register_custom_handlers(): - """Register omnibase_infra custom protocol handlers.""" - ProtocolHandlerRegistry.register("valkey", ValkeyHandler) -``` - -### 4.3 Version Pinning Strategy - -In `pyproject.toml`: - -```toml -[tool.poetry.dependencies] -# Pin to specific minor version for stability -omnibase-core = ">=0.4.0,<0.5.0" - -# Allow patch updates -omnibase-spi = "^0.2.0" -``` - -**Upgrade Policy**: -1. **Patch versions** (0.4.x): Auto-update, run tests -2. **Minor versions** (0.5.0): Review changelog, test in staging -3. **Major versions** (1.0.0): Full compatibility review, migration plan - ---- - -## 5. Migration Path - -### 5.1 Phased Migration Strategy - -``` -Phase A: Naming Transition (Week 1) -β”œβ”€β”€ Rename current imperative nodes to *Legacy suffix -β”œβ”€β”€ Create YAML contracts for new contract-driven nodes -β”œβ”€β”€ New contract-driven nodes take standard names (no suffix) -β”œβ”€β”€ Feature flag: USE_CONTRACT_DRIVEN_EFFECTS=false -└── No production impact, import aliases maintained - -Phase B: Parallel Implementation (Week 2) -β”œβ”€β”€ Implement contract-driven nodes alongside Legacy code -β”œβ”€β”€ Implement wrapper that can use either -β”œβ”€β”€ Deprecation warnings on Legacy imports -└── Feature flag: USE_CONTRACT_DRIVEN_EFFECTS=false - -Phase C: Shadow Mode (Week 3) -β”œβ”€β”€ Run both implementations in parallel -β”œβ”€β”€ Compare results for consistency -β”œβ”€β”€ Log discrepancies -└── Feature flag: USE_CONTRACT_DRIVEN_EFFECTS=shadow - -Phase D: Gradual Rollout (Week 4) -β”œβ”€β”€ Enable contract-driven for non-critical paths -β”œβ”€β”€ Monitor metrics and error rates -β”œβ”€β”€ Quick rollback to Legacy capability -└── Feature flag: USE_CONTRACT_DRIVEN_EFFECTS=true (per-node) - -Phase E: Full Migration (Week 5-6) -β”œβ”€β”€ All nodes using contract-driven implementation -β”œβ”€β”€ Legacy nodes deprecated but available -β”œβ”€β”€ Plan removal for v0.5.0 -└── Feature flag: USE_CONTRACT_DRIVEN_EFFECTS=true (default) - -Phase F: Legacy Removal (v0.5.0) -β”œβ”€β”€ Remove all *Legacy classes -β”œβ”€β”€ Remove feature flags -β”œβ”€β”€ Contract-driven is the only implementation -└── Clean up deprecated imports -``` - -### 5.1.1 Naming Transition Details - -| Current Name | v0.4.0 Name | v0.5.0 Name | -|--------------|-------------|-------------| -| `NodeEffect` (imperative) | `NodeEffectLegacy` | REMOVED | -| N/A | `NodeEffect` (contract-driven) | `NodeEffect` (contract-driven) | -| `NodePostgresAdapter` (imperative) | `NodePostgresAdapterLegacy` | REMOVED | -| N/A | `NodePostgresAdapter` (contract-driven) | `NodePostgresAdapter` (contract-driven) | -| `NodeKafkaAdapter` (imperative) | `NodeKafkaAdapterLegacy` | REMOVED | -| N/A | `NodeKafkaAdapter` (contract-driven) | `NodeKafkaAdapter` (contract-driven) | - -### 5.2 Feature Flag Implementation - -```python -"""Feature flags for contract-driven effect migration.""" - -import os -from enum import Enum -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from omnibase_core.nodes import NodeEffect - -class EffectNodeMode(str, Enum): - """Migration mode for effect nodes.""" - LEGACY = "legacy" # Use Legacy imperative only (deprecated) - SHADOW = "shadow" # Run both, compare, use Legacy - CONTRACT = "contract" # Use contract-driven only (DEFAULT in v0.4.0+) - PER_NODE = "per_node" # Check per-node flags - -def get_effect_mode() -> EffectNodeMode: - """Get current effect node mode from environment.""" - mode = os.getenv("USE_CONTRACT_DRIVEN_EFFECTS", "contract").lower() - return EffectNodeMode(mode) - -def should_use_contract_driven(node_name: str) -> bool: - """Check if a specific node should use contract-driven implementation.""" - mode = get_effect_mode() - - if mode == EffectNodeMode.LEGACY: - return False - elif mode == EffectNodeMode.CONTRACT: - return True - elif mode == EffectNodeMode.PER_NODE: - # Check node-specific flag - flag = os.getenv(f"USE_CONTRACT_DRIVEN_{node_name.upper()}", "true") - return flag.lower() == "true" - elif mode == EffectNodeMode.SHADOW: - # Shadow mode handled separately - return False - - return True # Default to contract-driven -``` - -### 5.3 Rollback Strategy - -```python -"""Rollback utilities for contract-driven effect migration.""" - -import logging -from typing import TypeVar, Generic -from omnibase_core.nodes import NodeEffect, NodeEffectLegacy -from omnibase_core.models import ModelEffectInput - -logger = logging.getLogger(__name__) - -T = TypeVar("T", bound=NodeEffectLegacy) - -class EffectNodeFallback(Generic[T]): - """ - Fallback wrapper that tries contract-driven first, - falls back to Legacy imperative on failure. - - Note: This is a temporary migration utility. Once Legacy - nodes are removed in v0.5.0, this class will be deprecated. - """ - - def __init__( - self, - contract_node: NodeEffect, - legacy_fallback: T, - fallback_threshold: int = 3, - ): - self.contract = contract_node - self.legacy = legacy_fallback # Deprecated, for fallback only - self.threshold = fallback_threshold - self.consecutive_failures = 0 - self._using_legacy = False - - async def execute(self, operation: str, params: dict) -> dict: - """Execute with automatic fallback to Legacy.""" - if self._using_legacy: - return await self._execute_legacy(operation, params) - - try: - result = await self.contract.execute_effect( - ModelEffectInput(operation=operation, params=params) - ) - - if result.success: - self.consecutive_failures = 0 - return result.data - else: - raise RuntimeError(result.error) - - except Exception as e: - self.consecutive_failures += 1 - logger.warning( - f"Contract-driven effect failed | " - f"operation={operation} | " - f"failures={self.consecutive_failures} | " - f"error={e}" - ) - - if self.consecutive_failures >= self.threshold: - logger.error( - f"Switching to Legacy fallback | " - f"node={self.contract.contract.get('name')}" - ) - self._using_legacy = True - - return await self._execute_legacy(operation, params) - - async def _execute_legacy(self, operation: str, params: dict) -> dict: - """Execute using Legacy imperative implementation (deprecated).""" - # Call the appropriate method on the Legacy node - method = getattr(self.legacy, f"execute_{operation}", None) - if method: - return await method(**params) - raise ValueError(f"Unknown operation: {operation}") - - def reset_fallback(self) -> None: - """Reset to try contract-driven again.""" - self._using_legacy = False - self.consecutive_failures = 0 -``` - ---- - -## 6. Timeline Integration - -### 6.1 Detailed Timeline - -| Week | Phase | Tasks | Deliverables | -|------|-------|-------|--------------| -| W1 | MVP Phase 2 | Complete Legacy effect nodes, rename with suffix | Working *Legacy adapters | -| W2 | MVP Phase 3-4 | Stamping service, contract-driven reducer | Stamping node, FSM contracts | -| W3 | **Phase 2.5a** | Design YAML contracts | 5 contract files, schema validation | -| W4 | **Phase 2.5b** | Wait for omnibase-core 0.4.0 | Dependency available (NodeEffect, NodeEffectLegacy) | -| W5 | **Phase 2.5c** | Implement contract-driven nodes | New nodes with standard names, feature flags | -| W6 | **Phase 2.5d** | Shadow mode testing | Comparison logs, discrepancy fixes | -| W7 | MVP Phase 6 | Full testing, validation | Test coverage, quality gates | -| W8 | Release | Production deployment | omnibase-infra 0.2.0 | -| Future | v0.5.0 | Remove Legacy nodes | Clean codebase, only contract-driven | - -### 6.2 Dependencies and Blockers - -**Hard Dependencies**: -1. omnibase-core 0.4.0 release with `NodeEffect` (contract-driven) and `NodeEffectLegacy` -2. Protocol handlers (HTTP, Bolt, Postgres, Kafka) implemented in omnibase_infra -3. `ProtocolHandler` protocol defined in omnibase_spi -4. `ProtocolHandlerRegistry` API stable (in omnibase_infra) - -**Soft Dependencies**: -1. MVP Phase 2 Legacy effect nodes (for fallback baseline) -2. MVP Phase 4 FSM contracts (shared patterns) -3. Integration test infrastructure - -**Potential Blockers**: -1. **omnibase-core 0.4.0 delay**: Proceed with contract design only -2. **Protocol handler gaps**: Implement custom handlers in infra -3. **Schema incompatibility**: Version contracts, maintain backward compat -4. **Legacy removal timing**: Ensure sufficient deprecation period before v0.5.0 - -### 6.3 Risk Mitigation - -| Risk | Likelihood | Impact | Mitigation | -|------|------------|--------|------------| -| Core release delay | Medium | High | Parallel contract design, fallback to Legacy | -| Contract schema changes | Medium | Medium | Version contracts, maintain v1 compatibility | -| Performance regression | Low | High | Shadow mode comparison, benchmarking | -| Protocol handler bugs | Medium | Medium | Custom handler overrides, rapid patching | -| Premature Legacy removal | Low | High | Clear deprecation timeline, warning logs | - ---- - -## 7. Success Criteria - -### 7.1 Technical Criteria - -- [ ] All 5 core contracts (postgres x2, kafka, consul, valkey) validated -- [ ] Custom Valkey handler working -- [ ] Feature flag system operational -- [ ] Shadow mode comparison passing (>99% consistency) -- [ ] Performance within 5% of Legacy baseline -- [ ] All existing tests passing with contract-driven nodes -- [ ] Legacy nodes renamed and deprecation warnings active - -### 7.2 Quality Criteria - -- [ ] Contract schema validation passing -- [ ] mypy --strict on new code -- [ ] Test coverage >85% for contract-driven wrappers -- [ ] Documentation for contract authoring -- [ ] Runbook for rollback to Legacy procedures -- [ ] Clear deprecation timeline documented - -### 7.3 Operational Criteria - -- [ ] Metrics dashboards for contract-driven vs Legacy comparison -- [ ] Alerting for circuit breaker opens -- [ ] DLQ monitoring configured -- [ ] Health check endpoints including contract validation -- [ ] Deprecation warning logs for Legacy usage - ---- - -## Appendix A: Contract Validation - -```python -"""Utility for validating effect contracts.""" - -import json -import yaml -from pathlib import Path -from jsonschema import validate, ValidationError - -SCHEMA_PATH = Path(__file__).parent / "_schema" / "effect_contract_schema.json" - -def load_schema() -> dict: - """Load the effect contract JSON schema.""" - with open(SCHEMA_PATH) as f: - return json.load(f) - -def validate_contract(contract_path: Path) -> list[str]: - """ - Validate an effect contract against the schema. - - Returns: - List of validation errors (empty if valid) - """ - errors = [] - - try: - with open(contract_path) as f: - contract = yaml.safe_load(f) - - schema = load_schema() - validate(instance=contract, schema=schema) - - except ValidationError as e: - errors.append(f"Schema validation failed: {e.message}") - errors.append(f" Path: {'.'.join(str(p) for p in e.path)}") - - except yaml.YAMLError as e: - errors.append(f"YAML parsing failed: {e}") - - except Exception as e: - errors.append(f"Unexpected error: {e}") - - return errors - -def validate_all_contracts(contracts_dir: Path) -> dict[str, list[str]]: - """Validate all contracts in a directory.""" - results = {} - - for contract_file in contracts_dir.glob("*.yaml"): - if contract_file.name.startswith("_"): - continue # Skip schema/internal files - - errors = validate_contract(contract_file) - results[contract_file.name] = errors - - return results -``` - ---- - -## Appendix B: Quick Reference - -### Environment Variables - -| Variable | Purpose | Default | -|----------|---------|---------| -| `USE_CONTRACT_DRIVEN_EFFECTS` | Migration mode | `contract` | -| `USE_CONTRACT_DRIVEN_{NODE_NAME}` | Per-node override | `true` | -| `POSTGRES_HOST` | PostgreSQL host | `localhost` | -| `POSTGRES_PORT` | PostgreSQL port | `5432` | -| `POSTGRES_DB` | Database name | `omnibase_infra` | -| `POSTGRES_USER` | Database user | `omni` | -| `POSTGRES_PASSWORD` | Database password | (required) | -| `KAFKA_BOOTSTRAP_SERVERS` | Kafka brokers | `localhost:9092` | -| `CONSUL_HOST` | Consul host | `localhost` | -| `CONSUL_PORT` | Consul port | `8500` | -| `CONSUL_ACL_TOKEN` | Consul ACL token | (optional) | -| `VALKEY_HOST` | Valkey host | `localhost` | -| `VALKEY_PORT` | Valkey port | `6379` | - -### CLI Commands - -```bash -# Validate all contracts -poetry run python -m omnibase_infra.contracts.validate - -# Run with contract-driven effects (DEFAULT in v0.4.0+) -USE_CONTRACT_DRIVEN_EFFECTS=contract poetry run pytest - -# Run with Legacy effects (deprecated) -USE_CONTRACT_DRIVEN_EFFECTS=legacy poetry run pytest - -# Run shadow mode comparison -USE_CONTRACT_DRIVEN_EFFECTS=shadow poetry run pytest tests/integration/ - -# Enable specific node for contract-driven -USE_CONTRACT_DRIVEN_POSTGRES_WORKFLOW=true poetry run pytest - -# Force Legacy for specific node (deprecated) -USE_CONTRACT_DRIVEN_POSTGRES_WORKFLOW=false poetry run pytest -``` - -### Class Names Quick Reference - -| Standard Name (v0.4.0+) | Legacy Name (Deprecated) | Removal | -|-------------------------|--------------------------|---------| -| `NodeEffect` | `NodeEffectLegacy` | v0.5.0 | -| `NodePostgresAdapter` | `NodePostgresAdapterLegacy` | v0.5.0 | -| `NodeKafkaAdapter` | `NodeKafkaAdapterLegacy` | v0.5.0 | -| `NodeConsulAdapter` | `NodeConsulAdapterLegacy` | v0.5.0 | -| `NodeVaultAdapterEffect` | `NodeVaultAdapterEffectLegacy` | v0.5.0 | - ---- - -*This plan was created on December 2, 2025 and should be reviewed when omnibase-core 0.4.0 is released. Naming convention updated to make contract-driven the default (December 2025).* diff --git a/docs/HANDOFF_MVP_PLANNING_2025_12_03.md b/docs/HANDOFF_MVP_PLANNING_2025_12_03.md deleted file mode 100644 index bda7e48a7d..0000000000 --- a/docs/HANDOFF_MVP_PLANNING_2025_12_03.md +++ /dev/null @@ -1,228 +0,0 @@ -# Handoff: MVP Planning Session - omnibase_infra - -**Date**: 2025-12-03 -**Session Focus**: MVP Proposed Work Issues document enhancement and Linear setup -**Status**: Document complete, Linear ticket creation pending - ---- - -## Session Summary - -This session focused on transforming the MVP Proposed Work Issues document from a dense technical spec into a comprehensive, navigable, and actionable planning document. - -### What Was Accomplished - -#### 1. Document Enhancements (41 Total Improvements) - -**Round 1 (10 suggestions)**: -- Core/Infra boundary clarifications -- Contract semantics (stability warnings, handler uniqueness) -- Event bus requirements (ordering guarantees, response patterns) -- RuntimeHostProcess clarifications (error handling, concurrency model) -- Testing improvements (determinism, performance requirements) -- Handler API sharpness (async requirements, lifecycle) -- Deployment design (resource targets, production warnings) -- Topic naming schema (validation rules, examples) -- Scope boundary notes (Do Not Implement sections) -- Minor micro-fixes (fail-fast rules, threading prohibition) - -**Round 2 (18 suggestions)**: -- Executive Summary (5 bullet points) -- Glossary (9 terms defined) -- FAQ section (5 common questions) -- 8 new architectural invariants -- Prohibited fields section -- Cross-version compatibility rules -- Error taxonomy improvements (InvalidOperationError, ContractValidationError) -- Handler contracts (operations, fields, response shapes) -- SQL injection protection details -- Handler implementation guidelines -- NodeRuntime registration order and envelope immutability -- InMemoryEventBus memory bounds and test features -- KafkaEventBus slow consumer handling -- Contract evolution roadmap -- Testing infrastructure (deterministic helpers) -- Dockerfile ENTRYPOINT documentation -- K8s HPA template with multi-instance warning -- Naming conventions table -- Risk items table (7 risks with mitigations) -- Maintainability guidelines (LOC limits, docstrings) -- Future-proofing section - -**Round 3 (13 suggestions)**: -- "Why This Document Exists" statement -- Error envelope requirements (9-field table + JSON example) -- Correlation ID ownership rules -- Contract loader fail-fast rules (9 conditions) -- Blocking I/O rules table (12 operations) -- Handler operation naming convention -- Expected first PRs (12-entry onboarding guide) -- Handler lifecycle state machine (ASCII diagram) -- Missing handler behavior documentation -- Testing without Docker guide -- Example node contracts (effect + compute) -- Topic naming examples (12 valid/invalid examples) - -**Round 4 (4 high-impact additions)**: -- Map of Abstractions (ASCII diagram showing Coreβ†’SPIβ†’Infra) -- MVP Constraints Checklist (12 things = MVP works) -- Failure Examples (6 concrete code snippets of what NOT to do) -- Minimum Reference Contract (smallest working YAML) -- How to Read This Document (role-based navigation) - -#### 2. Linear Setup - -**Labels Created**: -| Label | Color | ID | -|-------|-------|-----| -| `mvp` | Blue (#0ea5e9) | `afae1c79-1608-4150-aa9a-a21f97fc31e8` | -| `beta` | Purple (#8b5cf6) | `3a95919f-97a6-42ad-9083-0f51e9968375` | -| `production` | Green (#22c55e) | `6908456d-63fd-406d-a6e2-deef4ba25c3d` | - -**Existing Labels Available**: -- `omnibase_infra` (red) -- `omnibase_core` (green) -- `omnibase_spi` (pink) -- `testing`, `docs`, `security`, `event bus`, `db`, `infra`, etc. - -**Team**: Omninode (`9bdff6a3-f4ef-4ff7-b29a-6c4cf44371e6`) -**Project**: MVP - OmniNode Platform Foundation (`e44ddbf4-b4c7-40dc-84fa-f402ec27b38e`) - ---- - -## Pending Work - -### 1. Linear Issue Templates - -User requested standardized issue templates in Linear for cross-project consistency. The template format (based on omnibase_core tickets) is: - -```markdown -## Description - -{description} - -## TDD Approach - -**TDD: {Required|Optional|None}** - {reason} - -## Acceptance Criteria - -- [ ] {criterion 1} -- [ ] {criterion 2} -- [ ] mypy --strict passes -- [ ] pyright passes - -## Phase - -Phase {N}: {Phase Name} - -## Dependencies - -* {dependency or "None"} - -## Reference - -See `docs/MVP_PROPOSED_WORK_ISSUES.md` for full context. -``` - -**Action Required**: Linear doesn't have native issue templates. Options: -1. Create a Linear document with template text for copy/paste -2. Use automation (Linear API) to enforce template structure -3. Create GitHub issue templates that sync to Linear - -### 2. Linear Ticket Creation - -54 issues ready to be created in Linear: - -| Milestone | Count | Labels | -|-----------|-------|--------| -| MVP (v0.1.0) | 24 | `mvp`, `omnibase_infra` | -| Beta (v0.2.0) | 22 | `beta`, `omnibase_infra` | -| Production (v0.3.0) | 8 | `production`, `omnibase_infra` | - -**Issue Breakdown by Phase**: -- Phase 0: CI Guardrails (2 MVP, 2 Beta) -- Phase 1: Core Types (9 MVP, 2 Beta) - in omnibase_core -- Phase 2: SPI Updates (3 MVP) - in omnibase_spi -- Phase 3: Infrastructure (8 MVP, 10 Beta) -- Phase 4: Testing (2 MVP, 5 Beta, 3 Prod) -- Phase 5: Deployment (2 MVP, 3 Beta, 2 Prod) - ---- - -## Key Files Modified - -| File | Status | Description | -|------|--------|-------------| -| `docs/MVP_PROPOSED_WORK_ISSUES.md` | Updated | ~2800 lines, comprehensive MVP spec | -| `docs/MVP_EXECUTION_PLAN.md` | Unchanged | Original execution plan | - ---- - -## Document Structure (MVP_PROPOSED_WORK_ISSUES.md) - -``` -1. Why This Document Exists -2. Executive Summary -3. Glossary (9 terms) -4. Naming Conventions -5. FAQ (5 questions) -6. Map of Abstractions (ASCII diagram) -7. Milestone Overview (MVP/Beta/Production) -8. MVP Constraints Checklist (12 items) -9. Simplified Contract Format -10. Topic Naming Schema -11. Contract Evolution Roadmap -12. Non-Goals -13. MVP Scope Boundaries (Do Not Implement) -14. Beta Scope Boundaries -15. Architectural Invariants Checklist -16. Correlation ID Ownership Rules -17. Blocking I/O Rules -18. Failure Examples (6 code snippets) -19. Phase 0-5 Issues (54 total) -20. Handler Lifecycle State Machine -21. Testing Infrastructure -22. Expected First PRs (12-entry onboarding) -23. Execution Order (dependency graphs) -24. Naming Conventions -25. MVP Risk Items (7 risks) -26. Maintainability Guidelines -27. Future-Proofing -28. Success Metrics -29. Issue Creation Guidelines -30. Example Node Contracts -31. Minimum Reference Contract -32. How to Read This Document -33. Document Organization (Future) -``` - ---- - -## Next Steps for Continuation - -1. **Decide on Linear template approach** (document vs automation vs GitHub sync) -2. **Create 54 Linear tickets** using the MCP tool -3. **Review the updated MVP document** for any final adjustments -4. **Consider splitting document** into multiple files as suggested in "Document Organization" - ---- - -## Commands for Quick Context - -```bash -# View the updated MVP document -cat /Users/jonah/Code/omnibase_infra/docs/MVP_PROPOSED_WORK_ISSUES.md - -# Check Linear labels -# Use mcp__linear-server__list_issue_labels - -# Create issues (when ready) -# Use mcp__linear-server__create_issue with project "MVP - OmniNode Platform Foundation" -``` - ---- - -**Session Duration**: Extended planning session -**Polymorphic Agents Used**: 14 (across 4 rounds of parallel execution) -**Total Suggestions Integrated**: 41 diff --git a/docs/HANDOFF_OMNIBASE_INFRA_MVP.md b/docs/HANDOFF_OMNIBASE_INFRA_MVP.md deleted file mode 100644 index 275e1b8e8d..0000000000 --- a/docs/HANDOFF_OMNIBASE_INFRA_MVP.md +++ /dev/null @@ -1,495 +0,0 @@ -# omnibase_infra MVP Handoff Document - -**Date**: December 2, 2025 -**Status**: Planning Complete, Ready for Execution -**Estimated MVP Timeline**: 2-3 weeks (realistic) -**Session ID**: da3112e3-8682-428f-81ae-39616e41bd57 - ---- - -## Executive Summary - -This document captures the complete analysis and planning for rebuilding `omnibase_infra` as a fresh, ONEX-compliant repository using `omnibase_core 0.3.5` and `omnibase_spi 0.2.0` (PyPI releases). - -### Key Decision - -**Approach**: Fresh start with reference repositories - -1. Move current `omnibase_infra` to `omnibase_infra_bak` -2. Create new `omnibase_infra` repository -3. Use backup AND `omninode_bridge` as references -4. Extract only valuable business logic, contracts, and models -5. Follow `omniintelligence` patterns exactly - -### Why Fresh Start vs Refactor - -| Factor | Refactor Existing | Fresh Start | -|--------|-------------------|-------------| -| Import Path Fixes | 300+ files, complex | Built correct from start | -| Scaffolding Rewrite | 100% needed anyway | New scaffolding | -| Technical Debt | Carried forward | Clean slate | -| Time Estimate | 3-4 weeks | 2-3 weeks | -| Risk | High (unknown breaks) | Low (known patterns) | - ---- - -## Current State Analysis - -### omnibase_infra (Backup) - -| Metric | Value | -|--------|-------| -| Total Python Files | 304 | -| Total YAML Contracts | 23 | -| Active Nodes | 11 (1 stub-only) | -| Total Node Code Lines | ~11,610 | -| Shared Models | 196 files, ~17,336 LOC | -| Infrastructure Utils | ~2,743 LOC | -| Active Tests | 0 (68K archived) | -| ONEX Compliance | 0% (wrong import paths) | - -**Problem**: All nodes use import paths from `omnibase_core v0.1.0` which don't exist in v0.3.5: -```python -# WRONG (v0.1.0 paths - don't exist) -from omnibase_core.core.node_effect_service import NodeEffectService -from omnibase_core.core.errors.onex_error import CoreErrorCode, OnexError -from omnibase_core.base.node_compute_service import NodeComputeService - -# CORRECT (v0.3.5 paths) -from omnibase_core.nodes import NodeEffect, NodeCompute, NodeReducer, NodeOrchestrator -from omnibase_core.models.errors.model_onex_error import ModelOnexError -from omnibase_core.enums.enum_onex_error_code import EnumOnexErrorCode -``` - -### omninode_bridge (Reference) - -| Metric | Value | -|--------|-------| -| Total LOC | ~260,000 | -| MVP Status | 85% complete | -| Infra Components | ~95,000 LOC designated | -| SQL Migrations | 49 files (40+ for infra) | -| Status | Being fully decomposed (will be deleted) | - -**Valuable for omnibase_infra**: -- SQL migrations (40+ files, 25+ tables) -- Security validators (SQL injection protection) -- Metadata stamping service (21K LOC) - if needed - -### omniintelligence (Pattern Reference) - -| Metric | Value | -|--------|-------| -| Total LOC | 211,146 | -| Nodes | 21 (7 effect, 7 compute, 1 reducer, 1 orchestrator) | -| omnibase-core Version | 0.3.4 (PyPI) | -| Test Files | 12 | -| Pattern Compliance | 100% | - -**Follow this repository's patterns exactly.** - ---- - -## Target Architecture - -### Repository Structure - -``` -omnibase_infra/ # NEW - Fresh repository -β”œβ”€β”€ pyproject.toml # PyPI deps: core ^0.3.5, spi ^0.2.0 -β”œβ”€β”€ src/omnibase_infra/ -β”‚ β”œβ”€β”€ __init__.py -β”‚ β”œβ”€β”€ adapters/ # Thin external service wrappers -β”‚ β”œβ”€β”€ clients/ # Service clients -β”‚ β”œβ”€β”€ contracts/ # Top-level contracts -β”‚ β”œβ”€β”€ enums/ # Centralized enums -β”‚ β”‚ β”œβ”€β”€ __init__.py -β”‚ β”‚ β”œβ”€β”€ enum_infra_fsm_type.py -β”‚ β”‚ β”œβ”€β”€ enum_operation_type.py -β”‚ β”‚ └── enum_service_status.py -β”‚ β”œβ”€β”€ events/ # Event infrastructure -β”‚ β”‚ β”œβ”€β”€ publisher/ -β”‚ β”‚ └── models/ -β”‚ β”œβ”€β”€ models/ # Centralized Pydantic models (CRITICAL) -β”‚ β”‚ β”œβ”€β”€ __init__.py # Comprehensive exports -β”‚ β”‚ β”œβ”€β”€ model_postgres_*.py -β”‚ β”‚ β”œβ”€β”€ model_vault_*.py -β”‚ β”‚ β”œβ”€β”€ model_consul_*.py -β”‚ β”‚ β”œβ”€β”€ model_kafka_*.py -β”‚ β”‚ └── model_infrastructure_*.py -β”‚ β”œβ”€β”€ nodes/ # ONEX nodes -β”‚ β”‚ β”œβ”€β”€ node_postgres_adapter_effect/v1_0_0/ -β”‚ β”‚ β”‚ β”œβ”€β”€ contracts/ -β”‚ β”‚ β”‚ β”‚ └── effect_contract.yaml -β”‚ β”‚ β”‚ β”œβ”€β”€ effect.py -β”‚ β”‚ β”‚ β”œβ”€β”€ __init__.py -β”‚ β”‚ β”‚ └── __main__.py -β”‚ β”‚ β”œβ”€β”€ node_vault_adapter_effect/v1_0_0/ -β”‚ β”‚ β”œβ”€β”€ node_consul_adapter_effect/v1_0_0/ -β”‚ β”‚ β”œβ”€β”€ node_keycloak_adapter_effect/v1_0_0/ -β”‚ β”‚ β”œβ”€β”€ node_webhook_effect/v1_0_0/ -β”‚ β”‚ β”œβ”€β”€ node_circuit_breaker_compute/v1_0_0/ -β”‚ β”‚ β”œβ”€β”€ node_tracing_compute/v1_0_0/ -β”‚ β”‚ β”œβ”€β”€ infra_reducer/v1_0_0/ -β”‚ β”‚ β”‚ β”œβ”€β”€ contracts/ -β”‚ β”‚ β”‚ β”‚ └── fsm_infrastructure_state.yaml -β”‚ β”‚ β”‚ β”œβ”€β”€ reducer.py # Uses MixinFSMExecution -β”‚ β”‚ β”‚ └── __init__.py -β”‚ β”‚ └── infra_orchestrator/v1_0_0/ -β”‚ β”‚ β”œβ”€β”€ contracts/ -β”‚ β”‚ β”‚ └── workflow_deployment.yaml -β”‚ β”‚ β”œβ”€β”€ orchestrator.py # Inherits NodeOrchestratorDeclarative -β”‚ β”‚ └── __init__.py -β”‚ β”œβ”€β”€ infrastructure/ # Utilities (from backup) -β”‚ β”‚ └── postgres_connection_manager.py -β”‚ β”œβ”€β”€ shared/ # Shared utilities -β”‚ └── utils/ -β”œβ”€β”€ tests/ -β”‚ β”œβ”€β”€ conftest.py -β”‚ β”œβ”€β”€ unit/ -β”‚ β”œβ”€β”€ integration/ -β”‚ └── nodes/ -└── docs/ -``` - -### Node Inventory (Target) - -| Node | Type | Suffix | Base Class | Priority | -|------|------|--------|------------|----------| -| `node_postgres_adapter_effect` | EFFECT | `_effect` | `NodeEffect` | P0 | -| `node_vault_adapter_effect` | EFFECT | `_effect` | `NodeEffect` | P1 | -| `node_consul_adapter_effect` | EFFECT | `_effect` | `NodeEffect` | P1 | -| `node_keycloak_adapter_effect` | EFFECT | `_effect` | `NodeEffect` | P2 | -| `node_webhook_effect` | EFFECT | `_effect` | `NodeEffect` | P2 | -| `node_circuit_breaker_compute` | COMPUTE | `_compute` | `NodeCompute` | P2 | -| `node_tracing_compute` | COMPUTE | `_compute` | `NodeCompute` | P3 | -| `infra_reducer` | REDUCER | (none) | `MixinFSMExecution` | P1 | -| `infra_orchestrator` | ORCHESTRATOR | (none) | `NodeOrchestratorDeclarative` | P1 | - -### Nodes NOT to Include - -| Node | Reason | -|------|--------| -| `kafka_adapter` | Use event bus mixins from core | -| `consul_projector` | Stub implementation, projectors go in reducer | -| `omni_infra_reducer` | Replace with declarative version | -| `omni_infra_orchestrator` | Replace with declarative version | - ---- - -## Dependencies - -### pyproject.toml Configuration - -```toml -[tool.poetry.dependencies] -python = "^3.12" - -# ONEX dependencies - PyPI releases (CRITICAL) -omnibase-core = "^0.3.5" -omnibase-spi = "^0.2.0" - -# Core dependencies -pydantic = "^2.11.7" -fastapi = "^0.115.0" -uvicorn = "^0.32.0" - -# Database -asyncpg = "^0.29.0" -psycopg2-binary = "^2.9.10" - -# Service integration -python-consul = "^1.1.0" -hvac = "^2.1.0" # Vault - -# Observability -structlog = "^23.2.0" -prometheus-client = "^0.19.0" -opentelemetry-api = "^1.27.0" -opentelemetry-sdk = "^1.27.0" - -# Resilience -tenacity = "^9.0.0" -circuitbreaker = "^2.0.0" -``` - -**REMOVED** (not needed): -- `aiokafka` - Use event bus mixins -- `confluent-kafka` - Use event bus mixins -- `redis` - Not needed for infra - ---- - -## Declarative Node Patterns - -### Reducer Pattern (FSM-Driven) - -```python -# infra_reducer/v1_0_0/reducer.py -from omnibase_core.mixins.mixin_fsm_execution import MixinFSMExecution -from omnibase_core.models.contracts.subcontracts.model_fsm_subcontract import ModelFSMSubcontract -from omnibase_core.utils.util_safe_yaml_loader import load_and_validate_yaml_model - -class InfraReducer(MixinFSMExecution): - """Infrastructure reducer using declarative FSM pattern.""" - - def __init__(self, config: ModelReducerConfig) -> None: - super().__init__() - self._fsm_contracts = self._load_fsm_contracts() - - def _load_fsm_contracts(self) -> dict[EnumInfraFSMType, ModelFSMSubcontract]: - return { - EnumInfraFSMType.INFRASTRUCTURE_STATE: load_and_validate_yaml_model( - _CONTRACTS_DIR / "fsm_infrastructure_state.yaml", - ModelFSMSubcontract, - ), - } -``` - -### Orchestrator Pattern (Workflow-Driven) - -```python -# infra_orchestrator/v1_0_0/orchestrator.py -from omnibase_core.nodes.node_orchestrator_declarative import NodeOrchestratorDeclarative -from omnibase_core.models.container.model_onex_container import ModelONEXContainer - -class InfraOrchestrator(NodeOrchestratorDeclarative): - """Infrastructure orchestrator using declarative workflow pattern.""" - - def __init__(self, container: ModelONEXContainer) -> None: - super().__init__(container) - self._workflow_cache: dict[EnumOperationType, ModelWorkflowDefinition] = {} -``` - ---- - -## Reusable Assets from Backup - -### High Value (Copy with Minor Fixes) - -| Asset | Location | LOC | Changes Needed | -|-------|----------|-----|----------------| -| PostgresConnectionManager | `infrastructure/postgres/` | 634 | Import fixes | -| Shared Models (Pydantic) | `models/*/` | 17,336 | Consolidate to central dir | -| Contracts (YAML) | `nodes/*/contract.yaml` | 5,663 | Restructure format | -| SQL Sanitizer | `nodes/node_distributed_tracing_compute/utils/` | ~200 | Copy as utility | -| CircuitBreakerFactory | `infrastructure/resilience/` | 550 | Import fixes | - -### Business Logic to Extract - -| Node | Logic Location | Approx LOC | Extract | -|------|----------------|------------|---------| -| postgres_adapter | `nodes/postgres_adapter/v1_0_0/node.py` | 1,689 | SQL validation, circuit breaker integration | -| vault_adapter | `nodes/node_vault_adapter_effect/v1_0_0/node.py` | 705 | Secret lifecycle, mock client | -| consul_adapter | `nodes/consul_adapter/v1_0_0/node.py` | 983 | Service discovery, KV operations | -| keycloak_adapter | `nodes/node_keycloak_adapter_effect/v1_0_0/node.py` | 799 | JWT handling, role management | -| hook_node | `nodes/hook_node/v1_0_0/node.py` | 1,450 | Webhook delivery, retry logic | -| circuit_breaker | `nodes/node_event_bus_circuit_breaker_compute/v1_0_0/node.py` | 524 | State machine logic | - -### DO NOT Copy - -| Asset | Reason | -|-------|--------| -| Import statements | All wrong | -| Base class inheritance | Different in 0.3.5 | -| Node scaffolding | Rebuild with correct patterns | -| Archived tests | Recreate fresh | -| kafka_adapter | Use event bus mixins | - ---- - -## Development Velocity Reference - -### Historical Data (from omninode_bridge) - -| Period | Commits | Rate | -|--------|---------|------| -| Initial Sprint (Sep 21-30) | 117 | 11.7/day | -| Stabilization (Oct) | 12 | 0.4/day | -| Feature Dev (Nov) | 19 | 0.6/day | -| **Sustainable Average** | - | **1-2/day** | - -### Planning Velocity - -| Scenario | Commits/Day | PRs/Week | -|----------|-------------|----------| -| Optimistic | 4-5 | 5-6 | -| Realistic | 1-2 | 3-4 | -| Conservative | 0.5 | 1-2 | - ---- - -## MVP Timeline - -### Week 1: Foundation (Days 1-5) - -#### Day 1-2: Repository Setup -- [ ] Backup current repo: `mv omnibase_infra omnibase_infra_bak` -- [ ] Create new repo with correct structure -- [ ] Initialize pyproject.toml with PyPI dependencies -- [ ] Create directory structure following omniintelligence - -#### Day 3-4: Core Infrastructure -- [ ] Create centralized `models/` directory -- [ ] Migrate Pydantic models from backup (fix imports) -- [ ] Create centralized `enums/` directory -- [ ] Set up `events/` infrastructure skeleton -- [ ] Copy PostgresConnectionManager (fix imports) - -#### Day 5: Validation -- [ ] Verify all imports resolve with `python -c "import omnibase_infra"` -- [ ] Run mypy on entire package -- [ ] Run ruff linting -- [ ] Basic smoke tests - -### Week 2: Node Implementation (Days 6-10) - -#### Day 6-7: Priority Effect Nodes -- [ ] `node_postgres_adapter_effect` (P0) - - New scaffolding with NodeEffect - - Extract business logic from backup - - Create contract YAML -- [ ] `node_vault_adapter_effect` (P1) -- [ ] `node_consul_adapter_effect` (P1) - -#### Day 8: Additional Effect Nodes -- [ ] `node_keycloak_adapter_effect` (P2) -- [ ] `node_webhook_effect` (P2) - -#### Day 9-10: Declarative Nodes -- [ ] `infra_reducer` with FSM contracts - - Follow omniintelligence IntelligenceReducer pattern - - Create FSM YAML contracts -- [ ] `infra_orchestrator` with workflow contracts - - Inherit from NodeOrchestratorDeclarative - - Create workflow YAML contracts - -### Week 3: Testing & Polish (Days 11-15) - -#### Day 11-12: Compute Nodes -- [ ] `node_circuit_breaker_compute` -- [ ] `node_tracing_compute` (if needed) - -#### Day 13-14: Test Suite -- [ ] Create test infrastructure (conftest.py, fixtures) -- [ ] Unit tests for each node -- [ ] Integration tests for critical paths -- [ ] Migrate useful patterns from archived tests - -#### Day 15: Final Validation -- [ ] Full test run with coverage report -- [ ] Target: >80% coverage -- [ ] Documentation review -- [ ] MVP sign-off - ---- - -## Success Criteria - -### MVP Must-Haves - -| Requirement | Target | -|-------------|--------| -| PyPI dependencies (core 0.3.5, spi 0.2.0) | Working | -| 5 Effect nodes operational | All passing tests | -| 1 Declarative Reducer | FSM-driven, YAML contracts | -| 1 Declarative Orchestrator | Workflow-driven, YAML contracts | -| Test coverage | >80% | -| Type checking | mypy --strict passes | -| Linting | ruff clean | - -### MVP Nice-to-Haves - -| Requirement | Priority | -|-------------|----------| -| 2 Compute nodes | P2 | -| Full integration test suite | P2 | -| Performance benchmarks | P3 | -| CI/CD pipeline | P3 | - ---- - -## Risk Mitigation - -| Risk | Mitigation | -|------|------------| -| omnibase_core 0.3.5 incompatible | Validate on Day 1 with simple import test | -| Business logic extraction breaks | Keep backup, extract incrementally | -| Declarative patterns unfamiliar | Follow omniintelligence exactly | -| Single developer bottleneck | Focus on P0/P1, defer P2/P3 | -| Test recreation takes too long | Start with critical path tests only | - ---- - -## Reference Documents - -### In omninode_bridge -- `/docs/migration/OMNIBASE_INFRA_MAP.md` - Complete component inventory -- `/docs/migration/SQL_CATEGORIZATION.md` - SQL migration distribution -- `/docs/planning/MVP_REGISTRY_SELF_REGISTRATION.md` - MVP Phase 1a -- `/docs/planning/IMPLEMENTATION_ROADMAP.md` - Complete roadmap - -### In omniintelligence -- Pattern reference for all node types -- Declarative reducer: `nodes/intelligence_reducer/v1_0_0/reducer.py` -- Declarative orchestrator: `nodes/intelligence_orchestrator/v1_0_0/orchestrator.py` - -### In omnibase_core -- `nodes/node_reducer_declarative.py` - Base declarative reducer -- `nodes/node_orchestrator_declarative.py` - Base declarative orchestrator -- `mixins/mixin_fsm_execution.py` - FSM execution mixin -- `mixins/mixin_workflow_execution.py` - Workflow execution mixin - ---- - -## Quick Start Commands - -```bash -# 1. Backup current repo -cd /Users/jonah/Code -mv omnibase_infra omnibase_infra_bak - -# 2. Create new repo -mkdir omnibase_infra -cd omnibase_infra -git init - -# 3. Initialize with poetry -poetry init --name omnibase_infra --python "^3.12" -poetry add omnibase-core@^0.3.5 omnibase-spi@^0.2.0 - -# 4. Create structure -mkdir -p src/omnibase_infra/{adapters,clients,contracts,enums,events,models,nodes,shared,utils} -mkdir -p tests/{unit,integration,nodes} - -# 5. Validate dependencies -python -c "from omnibase_core.nodes import NodeEffect; print('Core OK')" -python -c "from omnibase_spi import ProtocolEventBus; print('SPI OK')" -``` - ---- - -## Open Questions - -1. **SQL Migrations**: Do we need to copy the 40+ SQL migrations from omninode_bridge, or will infrastructure tables be managed elsewhere? - -2. **Metadata Stamping**: Is the metadata stamping service (21K LOC) needed in omnibase_infra, or is it going to a different repo? - -3. **Event Infrastructure**: How much of the event publisher infrastructure do we need if we're using core event bus mixins? - -4. **Keycloak Priority**: Is Keycloak adapter P1 or P2? Can it be deferred past MVP? - -5. **Test Coverage Target**: Is 80% sufficient for MVP, or do we need higher? - ---- - -## Next Steps - -1. **Review this document** and answer open questions -2. **Validate omnibase_core 0.3.5** - ensure it's published to PyPI -3. **Create the backup** and new repository -4. **Begin Week 1 execution** - ---- - -*This handoff document was generated during session da3112e3-8682-428f-81ae-39616e41bd57 on December 2, 2025.* diff --git a/docs/HANDOFF_SESSION_2025_12_03.md b/docs/HANDOFF_SESSION_2025_12_03.md deleted file mode 100644 index 85dfdd39ec..0000000000 --- a/docs/HANDOFF_SESSION_2025_12_03.md +++ /dev/null @@ -1,518 +0,0 @@ -# Session Handoff: ONEX Infrastructure Migration - -**Date**: December 3, 2025 -**Session Type**: Extended architectural planning and repository setup -**Working Directory**: `/Users/jonah/Code/omnibase_infra.bak` (with operations on new repo) -**Key Tools Used**: Linear MCP, parallel-solve, polymorphic agents - ---- - -## Executive Summary - -This session accomplished significant foundational work for the ONEX infrastructure migration. The primary outcome is a **major architectural pivot** from a 1-container-per-node model to a **Runtime Host model** that will dramatically improve resource efficiency and deployment simplicity. - -Key accomplishments: -1. Fresh repository setup with proper directory structure -2. Linear ticket cleanup (11 tickets canceled due to migration) -3. Comprehensive architecture documentation -4. Critical architectural decision: Runtime Host model adoption - ---- - -## Work Completed This Session - -### 1. Repository Setup - -Created fresh `/Users/jonah/Code/omnibase_infra/` directory (sibling to `omnibase_infra.bak`). - -**Seeded with planning documents from backup**: -- `docs/MVP_EXECUTION_PLAN.md` -- `docs/HANDOFF_OMNIBASE_INFRA_MVP.md` -- `docs/DECLARATIVE_EFFECT_NODES_PLAN.md` - -**Copied configuration files**: -- `CLAUDE.md` - Claude Code instructions and ONEX patterns -- `.claude/settings.local.json` - Local Claude settings -- `pyproject.toml` - Reference template for dependencies - -**Created initial directory structure**: -``` -src/omnibase_infra/ -β”œβ”€β”€ __init__.py -β”œβ”€β”€ clients/ -β”‚ └── __init__.py -β”œβ”€β”€ enums/ -β”‚ └── __init__.py -β”œβ”€β”€ infrastructure/ -β”‚ └── __init__.py -β”œβ”€β”€ models/ -β”‚ └── __init__.py -β”œβ”€β”€ nodes/ -β”‚ └── __init__.py -β”œβ”€β”€ shared/ -β”‚ └── __init__.py -└── utils/ - └── __init__.py -``` - -**Created tests/ structure**: -``` -tests/ -β”œβ”€β”€ __init__.py -β”œβ”€β”€ conftest.py -β”œβ”€β”€ integration/ -β”‚ └── __init__.py -β”œβ”€β”€ nodes/ -β”‚ └── __init__.py -└── unit/ - └── __init__.py -``` - -**Created documentation**: -- `README.md` - Repository overview - -**Architectural Note**: Removed `adapters/` directory - adapters ARE effect nodes, they belong in `nodes/`. This enforces the ONEX principle that all external integrations are effect nodes with proper contracts. - ---- - -### 2. Linear Ticket Cleanup - -Canceled 11 tickets related to omnibase_infra, omniarchon, and omninode_bridge due to repository migration: - -| Ticket | Title | Status | -|--------|-------|--------| -| OMN-145 | Fix Pydantic v2 PrivateAttr initialization | Canceled | -| OMN-132 | Promote declarative transformation models | Canceled | -| OMN-133 | Enforce node_ prefix naming convention | Canceled | -| OMN-131 | ONEX 4-Node Ingestion Pipeline | Canceled | -| OMN-119 | Agent Observability Dashboard Integration | Canceled | -| OMN-69 | Fix omnibase_core import structure | Canceled | -| OMN-53 | Phase 5: Intelligence Effect Adapter | Canceled | -| OMN-44 | Document Event Architecture | Canceled | -| OMN-38 | Add DLQ Routing to Publishers | Canceled | -| OMN-41 | Create DLQ Monitoring Service | Canceled | -| OMN-39 | Implement Secret Sanitization | Canceled | - -**Cancellation Reason**: "Canceled due to repository migration and consolidation. This work is being superseded by the fresh ONEX-compliant repository rebuild." - ---- - -### 3. Architecture Documentation - -Created `docs/CURRENT_NODE_ARCHITECTURE.md` (927 lines) documenting: - -- **Current 1-container-per-node architecture** - How nodes work today -- **Full file trees** for Vault Adapter and Consul Projector nodes -- **Contract YAML structure** - All required fields and patterns -- **Node base classes** - `NodeEffectService`, `NodeComputeService`, etc. -- **Import patterns** - From `omnibase_core` dependencies -- **Deployment model limitations** - Why the current model doesn't scale - -This document serves as the authoritative reference for the pre-migration architecture. - ---- - -### 4. Major Architectural Decision: Runtime Host Model - -**THIS IS THE KEY CHANGE FROM THIS SESSION** - -#### Problem Statement - -The current architecture has significant limitations: -- **Resource waste**: Each node = 1 Docker container (~150MB memory each) -- **Connection explosion**: Each container maintains its own Kafka connections -- **Deployment complexity**: N nodes = N containers to deploy and manage -- **Startup overhead**: Each container goes through full initialization - -#### Old Architecture (1-Container-Per-Node) - -``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ Container 1 β”‚ β”‚ Container 2 β”‚ β”‚ Container 3 β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ vault β”‚ β”‚ β”‚ β”‚ consul β”‚ β”‚ β”‚ β”‚ postgres β”‚ β”‚ -β”‚ β”‚ adapter β”‚ β”‚ β”‚ β”‚ projector β”‚ β”‚ β”‚ β”‚ adapter β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β”‚ [Kafka conn] β”‚ β”‚ [Kafka conn] β”‚ β”‚ [Kafka conn] β”‚ -β”‚ [Entry point] β”‚ β”‚ [Entry point] β”‚ β”‚ [Entry point] β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -#### New Architecture (Runtime Host) - -``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ NodeRuntime Host β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ Shared Kafka Connection β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β”‚ β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ NodeInstanceβ”‚ β”‚ NodeInstanceβ”‚ β”‚ NodeInstanceβ”‚ β”‚ -β”‚ β”‚ vault β”‚ β”‚ consul β”‚ β”‚ postgres β”‚ β”‚ -β”‚ β”‚ adapter β”‚ β”‚ projector β”‚ β”‚ adapter β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β”‚ β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ Handlers: [local] [http] [db] [llm] [kafka] β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -#### Key Concepts - -| Concept | Description | -|---------|-------------| -| `NodeRuntime` | Single process that hosts multiple node instances | -| `NodeInstance` | Lightweight wrapper around a node's business logic | -| `Handler` | Shared infrastructure (http, db, llm, kafka) | -| `FileRegistry` | Loads contracts from filesystem | -| `OnexEnvelope` | Unified message format for all node communication | - -#### Benefits - -1. **Resource efficiency**: Single Kafka connection shared by all nodes -2. **Memory reduction**: ~150MB total vs ~150MB per node -3. **Simpler deployment**: 1 container instead of N containers -4. **Faster startup**: Nodes are just instances, not full processes -5. **Easier testing**: Spin up runtime with multiple nodes in-process - ---- - -## Repository Responsibilities - -The Runtime Host model requires changes across multiple repositories: - -| Component | Repository | Notes | -|-----------|------------|-------| -| `NodeRuntime` class | `omnibase_core` | Core runtime host implementation | -| `NodeInstance` class | `omnibase_core` | Node instance wrapper | -| `RuntimeHostContract` | `omnibase_core` or `omnibase_spi` | Contract schema for hosts | -| `OnexEnvelope` | `omnibase_core` | Unified message envelope | -| `NodeKind.RUNTIME_HOST` | `omnibase_core` | New enum value | -| `FileRegistry` | `omnibase_core` | Contract file loading | -| CLI entry point | `omnibase_core` | `omninode-runtime-host` command | -| Handler: `local` | `omnibase_core` | Echo/test handler (no external deps) | -| Handler: `http` | `omnibase_infra` | httpx-based HTTP handler | -| Handler: `db` | `omnibase_infra` | PostgreSQL handler | -| Handler: `llm` | `omnibase_infra` | LLM API handler | -| Handler contracts | `omnibase_infra` | Per-handler contracts | -| Infrastructure config | `omnibase_infra` | Kafka, Postgres config | - -**Dependency Flow**: `core <- spi <- infra` (infra depends on spi, spi depends on core, core depends on nothing) - ---- - -## Core Design Invariants - -These invariants MUST be maintained throughout the ONEX architecture: - -1. **All behavior is contract-driven** - No implicit behavior, everything defined in YAML contracts -2. **NodeRuntime is the only executable event loop** - Nodes don't run their own loops -3. **Node logic is pure: no I/O, no mixins, no inheritance** - Pure functions only -4. **Core never depends on SPI or infra** - Dependency flows one way: infra -> spi -> core -5. **SPI only defines protocols, never implementations** - Pure interfaces only -6. **Infra owns all I/O and real system integrations** - All external interactions in infra - -**Critical Invariant**: -``` -No code in omnibase_core may initiate network I/O, database I/O, -file I/O, or external process execution. -``` - ---- - -## Runtime Host Wiring - -The runtime host consists of components from both core and infra: - -``` -RuntimeHostProcess (omnibase_infra) - |-- NodeRuntime (omnibase_core) - |-- NodeInstance(vault_adapter) - |-- NodeInstance(user_query) - |-- NodeInstance(echo) - |-- handlers: - vault_handler: VaultHandler (infra) - db_handler: PostgresHandler (infra) - http_handler: HttpRestHandler (infra) -``` - -**Key Separation**: -- `NodeRuntime` -> Core logic (what the runtime **is**) -- `RuntimeHostProcess` -> Infra-level process container (where the runtime **runs**) -- Handler registry -> Infra's injection layer (how handlers are **wired**) - ---- - -## Topic Naming Schema - -Standardized Kafka topic naming for the Runtime Host model: - -``` -# Command topics (inbound to nodes) -onex.app.local.global.cmd.node..v1 - -# Event topics (outbound from nodes) -onex.app.local.global.evt.node..v1 - -# System log topics (runtime logs) -onex.sys.local.global.log.runtime..v1 - -# Examples: -onex.app.local.global.cmd.node.vault-adapter.v1 -onex.app.local.global.evt.node.vault-adapter.v1 -onex.sys.local.global.log.runtime.infra-host-01.v1 -``` - ---- - -## Phased Implementation Plan - -### Phase 0: Core Types (omnibase_core) -- Add `NodeKind.RUNTIME_HOST` to enums -- Implement `OnexEnvelope` model -- Implement `RuntimeHostContract` model -- Define handler protocol interface - -### Phase 1: Minimal Local Runtime -- Implement `NodeRuntime` class -- Implement `NodeInstance` wrapper -- Implement `FileRegistry` for contract loading -- Create `local_handler` (echo/test) -- Create `http_handler` (httpx-based) -- Add CLI: `omninode-runtime-host` -- **Validation**: End-to-end echo flow through Kafka - -### Phase 2: First Real Integration -- Implement ONE of: `db_handler` OR `llm_handler` -- Full integration test with real external service -- Performance benchmarking vs old model - -### Phase 3: Cloud Data Plane -- Local runtime connects to AWS infrastructure -- S3, DynamoDB, or other cloud handlers -- Production deployment patterns - -### Phase 4: Multi-Host Scaling -- Multiple runtime hosts -- Load balancing across hosts -- Node affinity and routing - ---- - -## Current State of New Repository - -``` -/Users/jonah/Code/omnibase_infra/ -β”œβ”€β”€ CLAUDE.md # Claude Code instructions -β”œβ”€β”€ README.md # Repository overview -β”œβ”€β”€ pyproject.toml # Python project config -β”œβ”€β”€ .claude/ -β”‚ └── settings.local.json # Local Claude settings -β”œβ”€β”€ docs/ -β”‚ β”œβ”€β”€ CURRENT_NODE_ARCHITECTURE.md # NEW - Pre-migration reference -β”‚ β”œβ”€β”€ DECLARATIVE_EFFECT_NODES_PLAN.md # Contract-driven nodes -β”‚ β”œβ”€β”€ HANDOFF_OMNIBASE_INFRA_MVP.md # Previous MVP handoff -β”‚ β”œβ”€β”€ HANDOFF_SESSION_2025_12_03.md # THIS DOCUMENT -β”‚ └── MVP_EXECUTION_PLAN.md # Original MVP plan -β”œβ”€β”€ src/ -β”‚ β”œβ”€β”€ __init__.py -β”‚ └── omnibase_infra/ -β”‚ β”œβ”€β”€ __init__.py -β”‚ β”œβ”€β”€ clients/ # External service clients -β”‚ β”‚ └── __init__.py -β”‚ β”œβ”€β”€ enums/ # Infrastructure enums -β”‚ β”‚ └── __init__.py -β”‚ β”œβ”€β”€ infrastructure/ # Core infrastructure -β”‚ β”‚ └── __init__.py -β”‚ β”œβ”€β”€ models/ # Pydantic models -β”‚ β”‚ └── __init__.py -β”‚ β”œβ”€β”€ nodes/ # ONEX nodes (including adapters) -β”‚ β”‚ └── __init__.py -β”‚ β”œβ”€β”€ shared/ # Shared utilities -β”‚ β”‚ └── __init__.py -β”‚ └── utils/ # Helper utilities -β”‚ └── __init__.py -└── tests/ - β”œβ”€β”€ __init__.py - β”œβ”€β”€ conftest.py # Pytest configuration - β”œβ”€β”€ integration/ # Integration tests - β”‚ └── __init__.py - β”œβ”€β”€ nodes/ # Node-specific tests - β”‚ └── __init__.py - └── unit/ # Unit tests - └── __init__.py -``` - ---- - -## Next Steps (Immediate Priority) - -### 1. In omnibase_core (MUST BE DONE FIRST) - -```python -# New enum value -class NodeKind(str, Enum): - EFFECT = "effect" - COMPUTE = "compute" - REDUCER = "reducer" - ORCHESTRATOR = "orchestrator" - RUNTIME_HOST = "runtime_host" # NEW - -# OnexEnvelope model -class OnexEnvelope(BaseModel): - envelope_id: UUID - correlation_id: UUID - node_slug: str - handler_type: str # "local", "http", "db", "llm" - payload: dict[str, Any] - metadata: dict[str, Any] - timestamp: datetime - -# RuntimeHostContract model -class RuntimeHostContract(BaseModel): - name: str - version: str - nodes: list[str] # List of node contracts to load - handlers: list[str] # Available handlers - -# NodeRuntime class -class NodeRuntime: - def __init__(self, config: RuntimeHostConfig): ... - def register_handler(self, handler: Handler): ... - def load_nodes(self, contracts_dir: Path): ... - async def start(self): ... - async def stop(self): ... - -# NodeInstance class -class NodeInstance: - def __init__(self, contract: NodeContract, runtime: NodeRuntime): ... - async def handle(self, envelope: OnexEnvelope) -> OnexEnvelope: ... - -# FileRegistry class -class FileRegistry: - def __init__(self, contracts_dir: Path): ... - def load_all(self) -> list[NodeContract]: ... - def get(self, slug: str) -> NodeContract: ... -``` - -### 2. In omnibase_infra - -``` -src/omnibase_infra/ -β”œβ”€β”€ handlers/ # NEW - Handler implementations -β”‚ β”œβ”€β”€ __init__.py -β”‚ β”œβ”€β”€ local_handler.py # Echo/test handler -β”‚ β”œβ”€β”€ http_handler.py # httpx-based HTTP handler -β”‚ β”œβ”€β”€ db_handler.py # PostgreSQL handler (Phase 2) -β”‚ └── llm_handler.py # LLM API handler (Phase 2) -β”œβ”€β”€ contracts/ # NEW - Example contracts -β”‚ β”œβ”€β”€ runtime_host.yaml # Host contract example -β”‚ β”œβ”€β”€ vault_adapter.yaml # Vault node contract -β”‚ └── consul_projector.yaml # Consul node contract -└── ... (existing structure) -``` - -### 3. Validation Checklist - -- [ ] Echo flow: CLI -> Kafka -> Runtime -> local_handler -> Kafka -> response -- [ ] HTTP flow: CLI -> Kafka -> Runtime -> http_handler -> external API -> response -- [ ] Multi-node: Single runtime hosting 3+ node instances -- [ ] Performance: Memory usage < 200MB for 10 nodes -- [ ] Contract loading: FileRegistry loads all contracts from directory - ---- - -## Open Questions - -### 1. RuntimeHostContract Location -**Question**: Where does `RuntimeHostContract` live - `omnibase_core` or `omnibase_spi`? -**Recommendation**: Start in `omnibase_core` since it's tightly coupled to `NodeRuntime`. Move to SPI later if needed. - -### 2. Local Handler Location -**Decision**: `local_handler` (echo) belongs in `omnibase_core`. -**Rationale**: -- Essential for testing and development workflows -- Has zero external dependencies -- Core is the right place for pure testing utilities -- Aligns with the invariant that core contains no I/O (echo is in-memory only) - -### 3. Handler Registration -**Question**: How do handlers register themselves with the runtime? -**Options**: -- **Explicit registration**: Runtime config lists handlers to load -- **Auto-discovery**: Scan package for Handler implementations -- **Plugin system**: Entry points in pyproject.toml - -**Recommendation**: Start with explicit registration for simplicity. Add auto-discovery later. - -### 4. Envelope Versioning -**Question**: How do we version `OnexEnvelope` for backwards compatibility? -**Recommendation**: Add `envelope_version` field. Runtime rejects unknown versions. - ---- - -## Reference Documents - -| Document | Path | Description | -|----------|------|-------------| -| MVP Execution Plan | `/Users/jonah/Code/omnibase_infra/docs/MVP_EXECUTION_PLAN.md` | Original MVP plan (needs update for Runtime Host) | -| Current Architecture | `/Users/jonah/Code/omnibase_infra/docs/CURRENT_NODE_ARCHITECTURE.md` | Pre-migration reference | -| Declarative Effects | `/Users/jonah/Code/omnibase_infra/docs/DECLARATIVE_EFFECT_NODES_PLAN.md` | Contract-driven effect nodes | -| Previous Handoff | `/Users/jonah/Code/omnibase_infra/docs/HANDOFF_OMNIBASE_INFRA_MVP.md` | Prior session handoff | -| Backup Repository | `/Users/jonah/Code/omnibase_infra.bak/` | Old repository for reference | - ---- - -## Session Metadata - -| Field | Value | -|-------|-------| -| Date | December 3, 2025 | -| Duration | Extended session | -| Primary Working Directory | `/Users/jonah/Code/omnibase_infra.bak` | -| Target Repository | `/Users/jonah/Code/omnibase_infra` | -| Linear Tickets Modified | 11 canceled | -| Documents Created | 2 (CURRENT_NODE_ARCHITECTURE.md, this handoff) | -| Architectural Decisions | 1 major (Runtime Host model) | - ---- - -## Glossary - -| Term | Definition | -|------|------------| -| **NodeRuntime** | Single process that hosts multiple NodeInstances | -| **NodeInstance** | Lightweight wrapper around node business logic | -| **Handler** | Shared infrastructure component (http, db, llm, kafka) | -| **OnexEnvelope** | Unified message format for all node communication | -| **FileRegistry** | Loads node contracts from filesystem | -| **Effect Node** | Node that performs external I/O (adapters are effect nodes) | -| **Contract** | YAML specification defining a node's interface and behavior | - ---- - -## Appendix: Why Runtime Host? - -### Before (10 nodes) -- 10 Docker containers -- 10 Kafka connections -- ~1.5GB memory -- 10 separate deployments -- 10 health checks -- 10 log streams - -### After (10 nodes) -- 1 Docker container -- 1 Kafka connection -- ~200MB memory -- 1 deployment -- 1 health check (with per-node status) -- 1 log stream (with node context) - -**Bottom line**: The Runtime Host model is essential for scaling ONEX to production workloads with hundreds of nodes. - ---- - -*End of Handoff Document* diff --git a/docs/MVP_PLAN.md b/docs/MVP_PLAN.md new file mode 100644 index 0000000000..5fce1f5ad7 --- /dev/null +++ b/docs/MVP_PLAN.md @@ -0,0 +1,494 @@ +# MVP Proposed Work Issues - omnibase_infra + +> **Why This Document Exists**: This is the canonical infrastructure plan for shipping ONEX Runtime Host. If a feature, requirement, or constraint is not in this document, it is not happening in MVP. This document is the single source of truth for scope, architecture, and acceptance criteria. When in doubt, check this document. + +**Repository**: omnibase_infra +**Generated**: 2025-12-03 +**Updated**: 2025-12-04 (Split into milestone-specific documents) +**Linear Project**: MVP - ONEX Runtime Host Infrastructure + +--- + +## Document Structure + +This document has been split into milestone-specific files for easier navigation: + +| Milestone | Version | Document | Issue Count | +|-----------|---------|----------|-------------| +| **MVP Core** | v0.1.0 | [MVP_v0.1.0_CORE.md](./milestones/MVP_v0.1.0_CORE.md) | 24 | +| **Beta Hardening** | v0.2.0 | [BETA_v0.2.0_HARDENING.md](./milestones/BETA_v0.2.0_HARDENING.md) | 22 | +| **Production** | v0.3.0 | [PRODUCTION_v0.3.0.md](./milestones/PRODUCTION_v0.3.0.md) | 8 | + +--- + +## Milestone Overview + +This document organizes issues into three milestones with progressive scope: + +> **Core Principle**: Core is **pure orchestrator**; infra is **pure transport**. This boundary is inviolable. + +| Milestone | Version | Focus | Issue Count | Timeline | +|-----------|---------|-------|-------------|----------| +| **MVP Core** | v0.1.0 | Minimal working runtime with InMemoryEventBus | 24 | Sprint 1-2 | +| **Beta Hardening** | v0.2.0 | Production handlers, Kafka, observability | 22 | Sprint 3-4 | +| **Production** | v0.3.0 | Full deployment, chaos testing, advanced features | 8 | Sprint 5 | + +### MVP vs Beta vs Production Philosophy + +**MVP (v0.1.0)**: Prove the architecture works end-to-end with minimal scope +- InMemoryEventBus only (no Kafka complexity) +- HTTP + DB handlers only (no Vault, no Consul) +- Simplified contract format +- Basic error handling +- Unit tests with mocks + +**Beta (v0.2.0)**: Harden for production use +- KafkaEventBus with backpressure +- Vault + Consul handlers +- Full retry/rate-limit policies +- Integration tests with real services +- Observability layer + +**Production (v0.3.0)**: Deploy and validate at scale +- Kubernetes manifests +- Chaos testing +- Performance benchmarks +- Complete documentation + +--- + +## Summary Statistics + +| Phase | MVP (v0.1.0) | Beta (v0.2.0) | Production (v0.3.0) | +|-------|--------------|---------------|---------------------| +| Phase 0: CI Guardrails | 2 | 2 | 0 | +| Phase 1: Core Types | 9 | 2 | 0 | +| Phase 2: SPI Updates | 3 | 0 | 0 | +| Phase 3: Infrastructure | 8 | 10 | 0 | +| Phase 4: Testing | 2 | 5 | 3 | +| Phase 5: Deployment | 2 | 3 | 2 | +| **Total** | **24** | **22** | **8** | + +--- + +## Map of Abstractions + +Understanding which component lives where is critical. This diagram shows the dependency flow: + +``` ++-------------------------------------------------------------------------+ +| ONEX ARCHITECTURE | ++-------------------------------------------------------------------------+ +| | +| +------------------------------------------------------------------+ | +| | omnibase_infra (YOU ARE HERE) | | +| | +----------------+ +----------------+ +------------------+ | | +| | | RuntimeHost | | Handlers | | EventBus | | | +| | | Process | | - HttpHandler | | - InMemoryBus | | | +| | | | | - DbHandler | | - KafkaBus (Beta)| | | +| | | Owns event | | - VaultHandler | | | | | +| | | loop + wiring | | (Beta) | | Transport layer | | | +| | +----------------+ +----------------+ +------------------+ | | +| +------------------------------------------------------------------+ | +| | | +| v implements | +| +------------------------------------------------------------------+ | +| | omnibase_spi | | +| | +--------------------+ +------------------------------------+ | | +| | | ProtocolHandler | | ProtocolEventBus | | | +| | | (abstract) | | (abstract) | | | +| | +--------------------+ +------------------------------------+ | | +| | Defines interfaces only. Zero implementations. | | +| +------------------------------------------------------------------+ | +| | | +| v depends on | +| +------------------------------------------------------------------+ | +| | omnibase_core | | +| | +----------------+ +----------------+ +------------------+ | | +| | | NodeRuntime | | NodeInstance | | ModelOnex | | | +| | | | | | | Envelope | | | +| | | Routes | | Wraps node | | | | | +| | | envelopes to | | contracts | | Unified message | | | +| | | handlers | | | | format | | | +| | +----------------+ +----------------+ +------------------+ | | +| | Pure orchestration. ZERO I/O. ZERO transport imports. | | +| +------------------------------------------------------------------+ | +| | ++-------------------------------------------------------------------------+ + +DEPENDENCY RULE: infra -> spi -> core (never reverse) +``` + +**Repository Ownership Summary**: + +| Component | Repository | Responsibility | +|-----------|------------|----------------| +| `NodeRuntime` | `omnibase_core` | Routes envelopes to handlers. NO I/O. | +| `NodeInstance` | `omnibase_core` | Wraps node contracts. Delegates to runtime. | +| `ModelOnexEnvelope` | `omnibase_core` | Unified message format. JSON serializable. | +| `ProtocolHandler` | `omnibase_spi` | Abstract interface for handlers. | +| `ProtocolEventBus` | `omnibase_spi` | Abstract interface for event buses. | +| `BaseBaseRuntimeHostProcess` | `omnibase_infra` | Owns event loop. Wires handlers. Drives runtime. Base class for app-specific hosts. | +| `HttpHandler`, `DbHandler` | `omnibase_infra` | Concrete handler implementations. | +| `InMemoryEventBus` | `omnibase_infra` | Concrete event bus for MVP. | + +**The Golden Rule**: +``` +CORE defines models and orchestration. +SPI defines abstract protocols. +INFRA implements handlers + event bus + BaseBaseRuntimeHostProcess bootstrap. +``` + +--- + +## Simplified Contract Format (MVP) + +MVP uses a minimal contract format: + +> **Stability Warning**: The MVP contract schema is intentionally incomplete and NOT considered stable. Breaking changes expected through v0.2. + +```yaml +runtime: + name: "my_runtime_host" + version: "1.0.0" + + event_bus: + kind: "inmemory" # "kafka" in Beta + + handlers: + - type: "http" + - type: "db" + + nodes: + - slug: "node_a" + - slug: "node_b" +``` + +See [MVP_v0.1.0_CORE.md](./milestones/MVP_v0.1.0_CORE.md) for full contract format details and constraints. + +See [BETA_v0.2.0_HARDENING.md](./milestones/BETA_v0.2.0_HARDENING.md) for Beta contract additions including Kafka, retry policies, and topic naming. + +--- + +## Non-Goals (All Milestones) + +To prevent scope creep, the following are explicitly **NOT** in scope: + +- **No multi-region failover** - Single cluster only +- **No automatic topic creation** - Assumes infra bootstrap handled separately +- **No dynamic handler discovery** - Static wiring from contract only +- **No auto-migration of legacy nodes** - Handled by higher-level repos +- **No LLM handler** - Deferred to future milestone +- **No Consul-based service mesh** - Basic discovery only (Beta) + +--- + +## Architectural Invariants Checklist + +Before starting any issue, review these invariants: + +- [ ] **Core is transport-agnostic**: `omnibase_core` has NO Kafka/HTTP/DB/Vault imports +- [ ] **NodeRuntime is pure in-memory**: No event loop, no bus consumer +- [ ] **Event bus is infra concern**: `BaseRuntimeHostProcess` + `ProtocolEventBus` drives runtime +- [ ] **Handlers use strong typing**: `handler_type` returns `EnumHandlerType`, not `str` +- [ ] **LocalHandler is dev/test only**: Never in production contracts +- [ ] **Single source of truth**: `wiring.py` for handler registration +- [ ] **No secrets in contracts**: All secrets via `secret_ref` or resolver +- [ ] **Single BaseRuntimeHostProcess per OS process**: No nested runtimes +- [ ] **Error boundary**: Infra MUST map all raw exceptions into core RuntimeHostError types before returning to NodeRuntime +- [ ] **No threads in handlers**: No handler may create threads or executors. All concurrency must be async + event bus +- [ ] **Payload opacity**: Infra never inspects payload interpretation; all semantics live in core +- [ ] **Correlation tracking**: If an incoming envelope has no correlation_id, the runtime MUST assign one +- [ ] **Correlation ID immutability**: Once assigned, correlation_id MUST NOT be modified by any component +- [ ] **No dynamic imports**: Handlers MUST NOT use `importlib` or dynamic module loading +- [ ] **No blocking I/O**: All I/O operations MUST be async. No `time.sleep()`, no synchronous HTTP/DB calls +- [ ] **No top-level awaits**: Module-level code MUST NOT await; all async work happens in lifecycle methods +- [ ] **No task creation in handlers**: Handlers MUST NOT call `asyncio.create_task()`. All concurrency is managed by BaseRuntimeHostProcess +- [ ] **Single-threaded MVP**: BaseRuntimeHostProcess MUST be single-threaded in MVP. No thread pools, no multiprocessing +- [ ] **Envelope immutability in core**: NodeRuntime MUST NOT mutate envelopes. Only handlers or event bus may enrich envelopes +- [ ] **Fail-fast contracts**: BaseRuntimeHostProcess MUST crash fast on malformed contracts. Fail early, fail loudly +- [ ] **No raw exceptions on bus**: BaseRuntimeHostProcess MUST NOT emit raw exceptions to the bus. All errors wrapped in response envelopes + +--- + +## Blocking I/O Rules + +**Blocking I/O Rules** (Binary Yes/No): + +| Operation | Allowed in Handlers? | Allowed in Core? | Notes | +|-----------|---------------------|------------------|-------| +| `await asyncio.sleep()` | YES | NO | Use for backoff, rate limiting | +| `time.sleep()` | NO | NO | Blocks event loop - FORBIDDEN | +| `asyncpg` queries | YES | NO | Infra handler only | +| `httpx` async requests | YES | NO | Infra handler only | +| `requests.get()` | NO | NO | Synchronous HTTP - FORBIDDEN | +| `open()` file read | LIMITED | NO | Only during initialization | +| `os.environ.get()` | LIMITED | YES | Only during initialization | +| `logging.info()` | YES | YES | Sync logging is acceptable | +| `print()` | NO | NO | Use structured logging | +| `subprocess.run()` | NO | NO | Blocks - FORBIDDEN | +| `json.dumps()` | YES | YES | CPU-bound, not I/O | +| DNS resolution | ASYNC ONLY | NO | Use `aiodns` or let `httpx` handle | + +**Initialization Exception**: +During `initialize()` lifecycle method, synchronous operations ARE allowed: +- Reading config files +- Environment variable access +- Initial connection pool creation (async preferred) +- One-time setup operations + +After `initialize()` completes, ALL I/O must be async. + +--- + +## Correlation ID Ownership Rules + +| Scenario | Behavior | +|----------|----------| +| Request envelope has `correlation_id` | MUST preserve exactly as-is | +| Request envelope missing `correlation_id` | BaseRuntimeHostProcess MUST assign new UUID | +| Handler receives envelope | MUST NOT modify `correlation_id` | +| Event bus receives envelope | MUST NOT modify `correlation_id` | +| Response envelope creation | MUST echo `correlation_id` from request | +| Error envelope creation | MUST echo `correlation_id` from request | + +**Assignment Authority**: +- Only `BaseRuntimeHostProcess` may assign a new `correlation_id` (and only when missing) +- No other component (handler, event bus, NodeRuntime) may create or modify it +- All log entries MUST include `correlation_id` for traceability + +--- + +## Failure Examples (What NOT to Do) + +Concrete examples of invariant violations. These patterns MUST fail CI or cause startup errors. + +### FAIL: Core importing transport library + +```python +# FILE: omnibase_core/runtime/node_runtime.py +# THIS MUST FAIL CI + +import asyncpg # VIOLATION: Core cannot import transport libraries + +class NodeRuntime: + async def route_envelope(self, envelope): + # This code should never exist in core + conn = await asyncpg.connect() # VIOLATION: I/O in core +``` + +**Why it fails**: `omnibase_core` must remain transport-agnostic. CI grep check blocks this. + +--- + +### FAIL: Handler creating background tasks + +```python +# FILE: omnibase_infra/handlers/http_handler.py +# THIS MUST FAIL CODE REVIEW + +class HttpHandler(ProtocolHandler): + async def execute(self, envelope): + # VIOLATION: Handler spawning background task + asyncio.create_task(self._background_work()) # FORBIDDEN + return response_envelope +``` + +**Why it fails**: Handlers MUST NOT create tasks. BaseRuntimeHostProcess owns all concurrency. + +--- + +### FAIL: Handler returning raw exception + +```python +# FILE: omnibase_infra/handlers/db_handler.py +# THIS MUST FAIL RUNTIME + +class DbHandler(ProtocolHandler): + async def execute(self, envelope): + try: + result = await self._pool.execute(query) + except asyncpg.PostgresError as e: + raise e # VIOLATION: Raw exception escaping handler + + # CORRECT: + # except asyncpg.PostgresError as e: + # raise HandlerExecutionError( + # message=str(e), + # handler_type=self.handler_type, + # correlation_id=envelope.correlation_id, + # ) from e +``` + +**Why it fails**: Raw exceptions leak internal details. All errors must be `RuntimeHostError` subclasses. + +--- + +### FAIL: NodeRuntime mutating envelope + +```python +# FILE: omnibase_core/runtime/node_runtime.py +# THIS MUST FAIL CODE REVIEW + +class NodeRuntime: + def route_envelope(self, envelope): + envelope.metadata["routed_at"] = datetime.now() # VIOLATION + envelope.correlation_id = uuid4() # VIOLATION + return self._dispatch(envelope) +``` + +**Why it fails**: NodeRuntime MUST NOT mutate envelopes. Only handlers or event bus may enrich. + +--- + +### FAIL: LocalHandler in production contract + +```yaml +# FILE: contracts/production_runtime.yaml +# THIS MUST FAIL STARTUP + +runtime: + name: "production_runtime" + handlers: + - type: "local" # VIOLATION: LocalHandler forbidden in infra + - type: "http" + - type: "db" +``` + +**Why it fails**: BaseRuntimeHostProcess MUST fail fast if LocalHandler detected in production. + +--- + +### FAIL: Blocking I/O in handler + +```python +# FILE: omnibase_infra/handlers/http_handler.py +# THIS MUST FAIL CODE REVIEW + +import requests # VIOLATION: Synchronous HTTP library + +class HttpHandler(ProtocolHandler): + async def execute(self, envelope): + # VIOLATION: Blocking call in async handler + response = requests.get(url) # Blocks event loop! + + # CORRECT: Use httpx (async) + # async with httpx.AsyncClient() as client: + # response = await client.get(url) +``` + +**Why it fails**: Blocking I/O freezes the entire BaseRuntimeHostProcess. All I/O must be async. + +--- + +## Success Metrics by Milestone + +### MVP (v0.1.0) Success Metrics + +| Metric | Target | Measurement | +|--------|--------|-------------| +| E2E flow works | Yes | Single test passes | +| Core Kafka imports | 0 | grep verification | +| Handler unit test coverage | >80% | pytest-cov | +| MVP issue count | 24 | Linear tracking | + +### Beta (v0.2.0) Success Metrics + +| Metric | Target | Measurement | +|--------|--------|-------------| +| Test coverage | >90% | pytest-cov | +| Integration tests pass | Yes | CI | +| Graceful shutdown drain | 100% | No dropped envelopes | +| Circuit breaker trigger | <10s | After threshold failures | +| Architecture violations | 0 | CI checks | + +### Production (v0.3.0) Success Metrics + +| Metric | Target | Measurement | +|--------|--------|-------------| +| Memory per 10 nodes | <200MB | tracemalloc | +| Envelope throughput | >100/sec | Benchmark suite | +| Handler latency (local) | <1ms | p99 latency | +| Handler latency (http) | <100ms | p99 latency | +| Handler latency (db) | <50ms | p99 latency | +| Chaos test survival | 100% | No crashes | + +--- + +## Issue Creation Guidelines + +When creating these issues in Linear: + +1. **Team**: Omninode +2. **Project**: MVP - ONEX Runtime Host Infrastructure +3. **Labels**: Apply as indicated + milestone tag (`mvp`, `beta`, `production`) +4. **Priority**: + - 1 = Urgent + - 2 = High + - 3 = Normal + - 4 = Low +5. **Dependencies**: Link related issues where indicated +6. **Repository**: Tag with appropriate repo (omnibase_core, omnibase_spi, omnibase_infra) +7. **Milestone**: v0.1.0 MVP, v0.2.0 Beta, or v0.3.0 Production + +--- + +## How to Read This Document + +Different roles need different sections: + +| Role | Start Here | Focus On | +|------|------------|----------| +| **New Engineer** | This Overview -> Map of Abstractions | [MVP_v0.1.0_CORE.md](./milestones/MVP_v0.1.0_CORE.md) | +| **Infra Engineer** | Map of Abstractions | [MVP_v0.1.0_CORE.md](./milestones/MVP_v0.1.0_CORE.md) Phase 3 (Handlers) | +| **QA Engineer** | Testing sections in milestone docs | [BETA_v0.2.0_HARDENING.md](./milestones/BETA_v0.2.0_HARDENING.md) Phase 4 | +| **DevOps** | Deployment sections | [PRODUCTION_v0.3.0.md](./milestones/PRODUCTION_v0.3.0.md) Phase 5 | +| **Architect** | Architecture Invariants -> Failure Examples | All milestone docs | +| **Product Manager** | This Overview -> Success Metrics | Success Metrics sections | + +**Quick Navigation**: +- "What is this?" -> This document (Overview) +- "What goes where?" -> Map of Abstractions (above) +- "What MUST work in MVP?" -> [MVP_v0.1.0_CORE.md](./milestones/MVP_v0.1.0_CORE.md) +- "What's added in Beta?" -> [BETA_v0.2.0_HARDENING.md](./milestones/BETA_v0.2.0_HARDENING.md) +- "What's needed for Production?" -> [PRODUCTION_v0.3.0.md](./milestones/PRODUCTION_v0.3.0.md) +- "What MUST NOT happen?" -> Failure Examples (above) + +--- + +## Milestone Documents + +### [MVP Core (v0.1.0)](./milestones/MVP_v0.1.0_CORE.md) + +Contains: +- MVP philosophy and scope boundaries +- Simplified contract format details +- All 24 MVP issues with full details +- MVP execution order +- Minimum reference contract +- Example node contracts + +### [Beta Hardening (v0.2.0)](./milestones/BETA_v0.2.0_HARDENING.md) + +Contains: +- Beta philosophy and dependencies on MVP +- Extended contract format with Kafka, retry policies +- Topic naming schema and validation +- All 22 Beta issues with full details +- Beta execution order +- Considerations for future work + +### [Production (v0.3.0)](./milestones/PRODUCTION_v0.3.0.md) + +Contains: +- Production philosophy and dependencies +- All 8 Production issues with full details +- Chaos test scenarios (detailed) +- Kubernetes deployment specifications +- Migration checklist + +--- + +**Last Updated**: 2025-12-04 +**Document Owner**: OmniNode Architecture Team +**Linear Project URL**: TBD diff --git a/docs/RUNTIME_HOST_IMPLEMENTATION_PLAN.md b/docs/RUNTIME_HOST_IMPLEMENTATION_PLAN.md deleted file mode 100644 index 2842d46017..0000000000 --- a/docs/RUNTIME_HOST_IMPLEMENTATION_PLAN.md +++ /dev/null @@ -1,2204 +0,0 @@ -# Runtime Host Architecture Implementation Plan - -**Created**: December 3, 2025 -**Updated**: December 3, 2025 (Architectural Refinements) -**Status**: Ready for Execution -**Estimated Duration**: 6-8 weeks -**Dependencies**: omnibase_core ^0.3.5, omnibase_spi ^0.2.0 - ---- - -## Executive Summary - -This plan details the complete implementation of the ONEX Runtime Host architecture, transitioning from a 1-container-per-node model to a unified Runtime Host model. The implementation spans three repositories (omnibase_core, omnibase_spi, omnibase_infra) with strict dependency ordering. - -### Key Architectural Invariants - -These invariants MUST be maintained throughout the implementation: - -1. **Core is transport-agnostic**: `omnibase_core` has NO Kafka/HTTP/DB/Vault imports anywhere -2. **NodeRuntime is a pure in-memory orchestrator**: It does NOT own any event loop or bus consumer -3. **Event bus consumption is an infra concern**: `RuntimeHostProcess` + `ProtocolEventBus` drive the runtime -4. **Handlers use strong typing**: `handler_type` returns `EnumHandlerType`, not `str` -5. **LocalHandler is dev/test-only**: Never used in production contracts -6. **Single source of truth for handler registry**: No duplicate registration logic - -### Architecture Overview - -``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ RuntimeHostProcess (omnibase_infra) β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ KafkaEventBus (ProtocolEventBus impl) β”‚ β”‚ -β”‚ β”‚ Consumes envelopes β†’ calls runtime.route_envelope(...) β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β”‚ β”‚ β”‚ -β”‚ β–Ό β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ NodeRuntime (omnibase_core) β”‚ β”‚ -β”‚ β”‚ Pure in-memory orchestrator - NO event loop, NO I/O β”‚ β”‚ -β”‚ β”‚ β”‚ β”‚ -β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ -β”‚ β”‚ β”‚NodeInstance β”‚ β”‚NodeInstance β”‚ β”‚NodeInstance β”‚ ... β”‚ β”‚ -β”‚ β”‚ β”‚ vault_ β”‚ β”‚ consul_ β”‚ β”‚ postgres_ β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ adapter β”‚ β”‚ projector β”‚ β”‚ adapter β”‚ β”‚ β”‚ -β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ -β”‚ β”‚ β”‚ β”‚ -β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ -β”‚ β”‚ β”‚ Handlers: [local*] [http] [db] [vault] [consul] β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ *local is dev/test only β”‚ β”‚ β”‚ -β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β”‚ β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ FileRegistry | HealthEndpoint | MetricsCollector β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -### Handler vs Event Bus Separation - -| Abstraction | Purpose | Location | Example | -|-------------|---------|----------|---------| -| `ProtocolHandler` | Per-request operations (db/http/vault/etc.) | SPI protocol, infra impl | `HttpHandler`, `DbHandler` | -| `ProtocolEventBus` | Message transport feeding envelopes into runtime | SPI protocol, infra impl | `KafkaEventBus` | - -**Key distinction**: -- `ProtocolHandler` = "do this specific thing for me" (synchronous request/response) -- `ProtocolEventBus` = "deliver messages to me" (async event consumption) - ---- - -## Phase 0: Prerequisites & Validation - -**Duration**: 1-2 days -**Repository**: All - -### 0.1 Dependency Validation - -```bash -# Verify PyPI packages -pip index versions omnibase-core # Should show β‰₯0.3.5 -pip index versions omnibase-spi # Should show β‰₯0.2.0 - -# Test current imports -python -c "from omnibase_core.nodes import NodeEffect; print('Core OK')" -python -c "from omnibase_spi.protocols import ProtocolEventBus; print('SPI OK')" -``` - -### 0.2 Repository State Verification - -| Repository | Branch | State Required | -|------------|--------|----------------| -| omnibase_core | main | Clean, passing CI | -| omnibase_spi | main | Clean, passing CI | -| omnibase_infra | main | Fresh setup complete | - -### 0.3 Success Criteria -- [ ] All dependencies installable from PyPI -- [ ] Import tests pass -- [ ] All repositories have clean main branches -- [ ] Development environments configured - ---- - -## Phase 1: Core Types (omnibase_core) - -**Duration**: 1 week -**Repository**: omnibase_core -**Priority**: CRITICAL - Must complete before all other phases - -### 1.1 New Enum Values - -**File**: `src/omnibase_core/enums/enum_node_kind.py` - -```python -from enum import Enum - -class EnumNodeKind(str, Enum): - """Types of ONEX nodes.""" - EFFECT = "effect" - COMPUTE = "compute" - REDUCER = "reducer" - ORCHESTRATOR = "orchestrator" - RUNTIME_HOST = "runtime_host" # NEW -``` - -**File**: `src/omnibase_core/enums/enum_handler_type.py` (NEW) - -```python -from enum import Enum - -class EnumHandlerType(str, Enum): - """Types of protocol handlers for per-request operations. - - Note: Kafka is NOT in this enum. Kafka is an event bus (ProtocolEventBus), - not a per-request handler (ProtocolHandler). Use ProtocolEventBus for - message consumption and delivery. - """ - LOCAL = "local" # Echo/test - dev/test only, no external deps - HTTP = "http" # HTTP REST calls - DB = "db" # Database operations - LLM = "llm" # LLM API calls - VAULT = "vault" # Vault secret management - CONSUL = "consul" # Consul service discovery -``` - -### 1.2 OnexEnvelope Model - -**File**: `src/omnibase_core/models/runtime/model_onex_envelope.py` (NEW) - -```python -"""Unified message envelope for all node communication.""" -from __future__ import annotations - -from datetime import datetime, UTC -from typing import Any -from uuid import UUID, uuid4 - -from pydantic import BaseModel, Field - -from omnibase_core.enums.enum_handler_type import EnumHandlerType - - -class ModelOnexEnvelope(BaseModel): - """Unified message envelope for Runtime Host communication. - - All communication between nodes, handlers, and external systems - flows through this envelope format. - """ - - # Identity - envelope_id: UUID = Field(default_factory=uuid4, description="Unique envelope identifier") - envelope_version: str = Field(default="1.0.0", description="Envelope schema version") - - # Correlation - correlation_id: UUID = Field(default_factory=uuid4, description="Request correlation ID") - causation_id: UUID | None = Field(default=None, description="ID of envelope that caused this one") - - # Routing - source_node: str = Field(description="Source node slug") - target_node: str | None = Field(default=None, description="Target node slug (None for broadcast)") - handler_type: EnumHandlerType = Field(description="Handler type for processing") - operation: str = Field(description="Operation to perform") - - # Payload - payload: dict[str, Any] = Field(default_factory=dict, description="Operation payload") - metadata: dict[str, Any] = Field(default_factory=dict, description="Additional metadata") - - # Timing - timestamp: datetime = Field(default_factory=lambda: datetime.now(UTC), description="Creation timestamp") - ttl_seconds: int | None = Field(default=None, description="Time-to-live in seconds") - - # Response (for reply envelopes) - is_response: bool = Field(default=False, description="Whether this is a response envelope") - success: bool | None = Field(default=None, description="Operation success (for responses)") - error: str | None = Field(default=None, description="Error message (for failures)") - - class Config: - json_encoders = { - datetime: lambda v: v.isoformat(), - UUID: str, - } -``` - -### 1.3 RuntimeHostContract Model - -**File**: `src/omnibase_core/models/contracts/model_runtime_host_contract.py` (NEW) - -```python -"""Runtime Host contract definition.""" -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from pydantic import BaseModel, Field - -from omnibase_core.enums.enum_handler_type import EnumHandlerType - - -class ModelHandlerConfig(BaseModel): - """Configuration for a protocol handler. - - TODO (post-MVP): Introduce per-handler config models (e.g. ModelHttpHandlerConfig, - ModelDbHandlerConfig) and validate `config` against them, instead of using - free-form `dict[str, Any]`. - """ - handler_type: EnumHandlerType - enabled: bool = True - config: dict[str, Any] = Field(default_factory=dict) - - -class ModelEventBusConfig(BaseModel): - """Configuration for the event bus. - - The event bus is separate from handlers - it's the transport that - feeds envelopes into NodeRuntime via RuntimeHostProcess. - """ - enabled: bool = True - # Transport-agnostic config - infra provides concrete implementation - config: dict[str, Any] = Field(default_factory=dict) - - -class ModelNodeRef(BaseModel): - """Reference to a node to be loaded.""" - slug: str = Field(description="Node slug identifier") - contract_path: str = Field(description="Path to node contract YAML") - enabled: bool = True - config_overrides: dict[str, Any] = Field(default_factory=dict) - - -class ModelRuntimeHostContract(BaseModel): - """Contract defining a Runtime Host configuration. - - The Runtime Host contract specifies which nodes to load, - which handlers to enable, and how to configure them. - - Note: Event bus configuration is separate from handlers. - Handlers are for per-request operations; the event bus is - for message transport. - """ - - # Identity - name: str = Field(description="Runtime host name") - version: str = Field(default="1.0.0", description="Contract version") - description: str = Field(default="", description="Human-readable description") - - # Node Configuration - nodes: list[ModelNodeRef] = Field(default_factory=list, description="Nodes to load") - contracts_directory: str = Field(default="contracts", description="Base directory for contracts") - - # Handler Configuration (for per-request operations) - handlers: list[ModelHandlerConfig] = Field(default_factory=list, description="Handler configurations") - - # Event Bus Configuration (transport-agnostic) - event_bus: ModelEventBusConfig = Field( - default_factory=ModelEventBusConfig, - description="Event bus config (transport implemented in infra)" - ) - - # Health & Metrics - health_endpoint: dict[str, Any] = Field( - default_factory=lambda: {"enabled": True, "port": 8080, "path": "/health"} - ) - metrics_endpoint: dict[str, Any] = Field( - default_factory=lambda: {"enabled": True, "port": 9090, "path": "/metrics"} - ) - - # Runtime Settings - max_concurrent_operations: int = Field(default=100, description="Max concurrent operations") - shutdown_timeout_seconds: int = Field(default=30, description="Graceful shutdown timeout") - - @classmethod - def from_yaml(cls, path: Path) -> "ModelRuntimeHostContract": - """Load contract from YAML file.""" - from omnibase_core.utils.util_safe_yaml_loader import load_and_validate_yaml_model - return load_and_validate_yaml_model(path, cls) -``` - -### 1.4 Handler Protocol Interface (Strongly Typed) - -**File**: `src/omnibase_core/protocols/protocol_handler.py` (NEW) - -```python -"""Protocol interface for Runtime Host handlers.""" -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import TYPE_CHECKING - -from omnibase_core.enums.enum_handler_type import EnumHandlerType - -if TYPE_CHECKING: - from omnibase_core.models.runtime.model_onex_envelope import ModelOnexEnvelope - - -class ProtocolHandler(ABC): - """Abstract protocol for Runtime Host handlers. - - Handlers are responsible for executing per-request operations defined in - OnexEnvelopes. Each handler type (http, db, vault, etc.) implements - this protocol to provide its specific functionality. - - Note: This is for synchronous request/response operations. - For message transport (event consumption/production), use ProtocolEventBus. - """ - - @property - @abstractmethod - def handler_type(self) -> EnumHandlerType: - """Return the handler type identifier. - - Returns: - EnumHandlerType - strongly typed, not a string - """ - ... - - @abstractmethod - async def initialize(self, config: dict) -> None: - """Initialize handler with configuration. - - Args: - config: Handler-specific configuration dictionary - """ - ... - - @abstractmethod - async def shutdown(self) -> None: - """Gracefully shutdown the handler.""" - ... - - @abstractmethod - async def execute(self, envelope: "ModelOnexEnvelope") -> "ModelOnexEnvelope": - """Execute an operation from an envelope. - - Args: - envelope: Input envelope with operation details - - Returns: - Response envelope with operation results - """ - ... - - @abstractmethod - async def health_check(self) -> dict: - """Check handler health. - - Returns: - Dict with 'healthy' bool and optional details - """ - ... -``` - -### 1.5 NodeInstance Class - -**File**: `src/omnibase_core/runtime/node_instance.py` (NEW) - -```python -"""Lightweight node instance wrapper for Runtime Host.""" -from __future__ import annotations - -import logging -from typing import TYPE_CHECKING, Any, Callable - -from omnibase_core.models.runtime.model_onex_envelope import ModelOnexEnvelope - -if TYPE_CHECKING: - from omnibase_core.models.contracts.model_node_contract import ModelNodeContract - from omnibase_core.runtime.node_runtime import NodeRuntime - - -class NodeInstance: - """Lightweight wrapper around node business logic. - - NodeInstance represents a single node within the Runtime Host. - It contains no event loop or I/O code - all operations are - delegated to handlers through the parent runtime. - - Implementation Note: - Currently, NodeInstance delegates all operation execution to NodeRuntime, - which in turn routes to the appropriate ProtocolHandler based on - `envelope.handler_type`. This keeps NodeInstance free of I/O details - and maintains the architectural invariant that nodes are pure logic. - """ - - def __init__( - self, - contract: "ModelNodeContract", - runtime: "NodeRuntime", - ) -> None: - self._contract = contract - self._runtime = runtime - self._logger = logging.getLogger(f"node.{contract.name}") - self._initialized = False - self._operation_handlers: dict[str, Callable] = {} - - @property - def slug(self) -> str: - """Return node slug identifier.""" - return self._contract.name - - @property - def node_type(self) -> str: - """Return node type (effect, compute, reducer, orchestrator).""" - return self._contract.node_type - - @property - def contract(self) -> "ModelNodeContract": - """Return the node contract.""" - return self._contract - - async def initialize(self) -> None: - """Initialize the node instance.""" - self._logger.info(f"Initializing node instance: {self.slug}") - - # Register operation handlers from contract - for op in self._contract.io_operations or []: - self._operation_handlers[op.operation] = self._create_operation_handler(op) - - self._initialized = True - self._logger.info(f"Node instance initialized: {self.slug}") - - async def shutdown(self) -> None: - """Shutdown the node instance.""" - self._logger.info(f"Shutting down node instance: {self.slug}") - self._initialized = False - - async def handle(self, envelope: ModelOnexEnvelope) -> ModelOnexEnvelope: - """Handle an incoming envelope. - - Delegates execution to NodeRuntime which routes to the appropriate - ProtocolHandler based on envelope.handler_type. - - Args: - envelope: Input envelope to process - - Returns: - Response envelope with operation results - """ - if not self._initialized: - return self._error_response(envelope, "Node not initialized") - - operation = envelope.operation - - # Validate operation is supported - if operation not in self._operation_handlers: - return self._error_response( - envelope, - f"Unknown operation: {operation}. Available: {list(self._operation_handlers.keys())}" - ) - - # Delegate to runtime for handler execution - try: - return await self._runtime.execute_with_handler(envelope) - except Exception as e: - self._logger.exception(f"Error handling operation {operation}") - return self._error_response(envelope, str(e)) - - def _create_operation_handler(self, operation_config: Any) -> Callable: - """Create a handler function for an operation.""" - async def handler(envelope: ModelOnexEnvelope) -> ModelOnexEnvelope: - return await self._runtime.execute_with_handler(envelope) - return handler - - def _error_response(self, envelope: ModelOnexEnvelope, error: str) -> ModelOnexEnvelope: - """Create an error response envelope.""" - return ModelOnexEnvelope( - correlation_id=envelope.correlation_id, - causation_id=envelope.envelope_id, - source_node=self.slug, - target_node=envelope.source_node, - handler_type=envelope.handler_type, - operation=envelope.operation, - is_response=True, - success=False, - error=error, - ) -``` - -### 1.6 NodeRuntime Class (Transport-Agnostic) - -**File**: `src/omnibase_core/runtime/node_runtime.py` (NEW) - -```python -"""Core NodeRuntime implementation for Runtime Host.""" -from __future__ import annotations - -import logging -from pathlib import Path -from typing import TYPE_CHECKING - -from omnibase_core.enums.enum_handler_type import EnumHandlerType -from omnibase_core.models.runtime.model_onex_envelope import ModelOnexEnvelope -from omnibase_core.runtime.node_instance import NodeInstance - -if TYPE_CHECKING: - from omnibase_core.models.contracts.model_runtime_host_contract import ModelRuntimeHostContract - from omnibase_core.protocols.protocol_handler import ProtocolHandler - - -class NodeRuntime: - """Core runtime that hosts multiple node instances. - - NodeRuntime is a pure in-memory orchestrator. It manages node instances, - routes operations to handlers, and provides shared infrastructure for - all hosted nodes. - - IMPORTANT ARCHITECTURAL INVARIANT: - NodeRuntime does NOT own any event loop or message consumer. - Event bus consumption is implemented in RuntimeHostProcess (omnibase_infra) - using ProtocolEventBus. RuntimeHostProcess calls route_envelope() for - each received message. - - This separation ensures: - - Core remains transport-agnostic (no Kafka/NATS/etc. imports) - - Testing is simplified (no real message bus needed) - - Different transports can be swapped in infra without touching core - """ - - def __init__(self, contract: "ModelRuntimeHostContract") -> None: - self._contract = contract - self._logger = logging.getLogger(f"runtime.{contract.name}") - - # Node instances - self._nodes: dict[str, NodeInstance] = {} - - # Handlers (for per-request operations) - self._handlers: dict[EnumHandlerType, "ProtocolHandler"] = {} - - # State - self._running = False - - @property - def name(self) -> str: - """Return runtime name.""" - return self._contract.name - - @property - def nodes(self) -> dict[str, NodeInstance]: - """Return registered node instances.""" - return self._nodes - - @property - def handlers(self) -> dict[EnumHandlerType, "ProtocolHandler"]: - """Return registered handlers.""" - return self._handlers - - @property - def is_running(self) -> bool: - """Return whether runtime is running.""" - return self._running - - def register_handler(self, handler: "ProtocolHandler") -> None: - """Register a protocol handler. - - Args: - handler: Handler instance to register (must return EnumHandlerType) - """ - # handler.handler_type is now EnumHandlerType, no conversion needed - handler_type = handler.handler_type - self._handlers[handler_type] = handler - self._logger.info(f"Registered handler: {handler_type}") - - def register_node(self, node: NodeInstance) -> None: - """Register a node instance. - - Args: - node: Node instance to register - """ - self._nodes[node.slug] = node - self._logger.info(f"Registered node: {node.slug}") - - async def load_nodes_from_directory(self, contracts_dir: Path) -> None: - """Load all node contracts from a directory. - - Args: - contracts_dir: Directory containing node contract YAML files - """ - from omnibase_core.runtime.file_registry import FileRegistry - - registry = FileRegistry(contracts_dir) - contracts = registry.load_all() - - for contract in contracts: - node = NodeInstance(contract, self) - self.register_node(node) - - async def initialize(self) -> None: - """Initialize the runtime and all components.""" - self._logger.info(f"Initializing runtime: {self.name}") - - # Initialize handlers - for handler_type, handler in self._handlers.items(): - config = self._get_handler_config(handler_type) - await handler.initialize(config) - self._logger.info(f"Handler initialized: {handler_type}") - - # Initialize nodes - for slug, node in self._nodes.items(): - await node.initialize() - self._logger.info(f"Node initialized: {slug}") - - self._running = True - self._logger.info(f"Runtime initialized with {len(self._nodes)} nodes and {len(self._handlers)} handlers") - - async def shutdown(self) -> None: - """Shutdown the runtime gracefully.""" - self._logger.info(f"Shutting down runtime: {self.name}") - self._running = False - - # Shutdown nodes - for slug, node in self._nodes.items(): - await node.shutdown() - self._logger.info(f"Node shutdown: {slug}") - - # Shutdown handlers - for handler_type, handler in self._handlers.items(): - await handler.shutdown() - self._logger.info(f"Handler shutdown: {handler_type}") - - self._logger.info(f"Runtime stopped: {self.name}") - - async def execute_with_handler(self, envelope: ModelOnexEnvelope) -> ModelOnexEnvelope: - """Execute an envelope using the appropriate handler. - - Args: - envelope: Envelope to execute - - Returns: - Response envelope - """ - handler_type = envelope.handler_type - - if handler_type not in self._handlers: - return ModelOnexEnvelope( - correlation_id=envelope.correlation_id, - causation_id=envelope.envelope_id, - source_node="runtime", - handler_type=handler_type, - operation=envelope.operation, - is_response=True, - success=False, - error=f"No handler registered for type: {handler_type}", - ) - - handler = self._handlers[handler_type] - return await handler.execute(envelope) - - async def route_envelope(self, envelope: ModelOnexEnvelope) -> ModelOnexEnvelope: - """Route an envelope to the appropriate node. - - This is the main entry point for envelopes coming from the event bus. - RuntimeHostProcess calls this method for each message received. - - Args: - envelope: Envelope to route - - Returns: - Response envelope - """ - target = envelope.target_node - - if target is None: - # Broadcast to all nodes (not typical) - return await self._broadcast_envelope(envelope) - - if target not in self._nodes: - return ModelOnexEnvelope( - correlation_id=envelope.correlation_id, - causation_id=envelope.envelope_id, - source_node="runtime", - handler_type=envelope.handler_type, - operation=envelope.operation, - is_response=True, - success=False, - error=f"Unknown target node: {target}", - ) - - node = self._nodes[target] - return await node.handle(envelope) - - async def _broadcast_envelope(self, envelope: ModelOnexEnvelope) -> ModelOnexEnvelope: - """Broadcast envelope to all nodes.""" - results = [] - for node in self._nodes.values(): - result = await node.handle(envelope) - results.append(result) - - return ModelOnexEnvelope( - correlation_id=envelope.correlation_id, - causation_id=envelope.envelope_id, - source_node="runtime", - handler_type=envelope.handler_type, - operation=envelope.operation, - is_response=True, - success=all(r.success for r in results if r.success is not None), - payload={"results": [r.model_dump() for r in results]}, - ) - - def _get_handler_config(self, handler_type: EnumHandlerType) -> dict: - """Get configuration for a handler type.""" - for handler_config in self._contract.handlers: - if handler_config.handler_type == handler_type: - return handler_config.config - return {} - - async def health_check(self) -> dict: - """Check health of runtime and all components.""" - handler_health = {} - for handler_type, handler in self._handlers.items(): - handler_health[handler_type.value] = await handler.health_check() - - node_health = {} - for slug, node in self._nodes.items(): - node_health[slug] = {"initialized": node._initialized} - - return { - "runtime": self.name, - "running": self._running, - "handlers": handler_health, - "nodes": node_health, - "healthy": self._running and all( - h.get("healthy", False) for h in handler_health.values() - ), - } -``` - -### 1.7 FileRegistry Class - -**File**: `src/omnibase_core/runtime/file_registry.py` (NEW) - -```python -"""File-based contract registry for Runtime Host.""" -from __future__ import annotations - -import logging -from pathlib import Path -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from omnibase_core.models.contracts.model_node_contract import ModelNodeContract - - -class FileRegistry: - """Registry that loads node contracts from filesystem. - - FileRegistry scans a directory for YAML contract files and - loads them into ModelNodeContract instances. - """ - - def __init__(self, contracts_dir: Path) -> None: - self._contracts_dir = contracts_dir - self._contracts: dict[str, "ModelNodeContract"] = {} - self._logger = logging.getLogger("file_registry") - - def load_all(self) -> list["ModelNodeContract"]: - """Load all contracts from the directory. - - Returns: - List of loaded node contracts - """ - from omnibase_core.models.contracts.model_node_contract import ModelNodeContract - from omnibase_core.utils.util_safe_yaml_loader import load_and_validate_yaml_model - - contracts = [] - - if not self._contracts_dir.exists(): - self._logger.warning(f"Contracts directory does not exist: {self._contracts_dir}") - return contracts - - # Find all YAML files - for yaml_file in self._contracts_dir.glob("**/*.yaml"): - if yaml_file.name.startswith("_"): - continue # Skip private/schema files - - try: - contract = load_and_validate_yaml_model(yaml_file, ModelNodeContract) - self._contracts[contract.name] = contract - contracts.append(contract) - self._logger.info(f"Loaded contract: {contract.name} from {yaml_file}") - except Exception as e: - self._logger.error(f"Failed to load contract from {yaml_file}: {e}") - - return contracts - - def get(self, slug: str) -> "ModelNodeContract | None": - """Get a contract by slug. - - Args: - slug: Node slug identifier - - Returns: - Contract if found, None otherwise - """ - return self._contracts.get(slug) - - def list_all(self) -> list[str]: - """List all loaded contract slugs.""" - return list(self._contracts.keys()) -``` - -### 1.8 Local Handler (Dev/Test Only) - -**File**: `src/omnibase_core/runtime/handlers/local_handler.py` (NEW) - -```python -"""Local echo handler for testing - no external dependencies.""" -from __future__ import annotations - -import logging -from typing import Any - -from omnibase_core.enums.enum_handler_type import EnumHandlerType -from omnibase_core.models.runtime.model_onex_envelope import ModelOnexEnvelope -from omnibase_core.protocols.protocol_handler import ProtocolHandler - - -class LocalHandler(ProtocolHandler): - """Local echo handler for testing and development only. - - WARNING: - This handler is not intended for production use. It exists to - validate Runtime Host wiring without external dependencies. - Production handlers (http/db/vault/etc.) are all implemented - in `omnibase_infra`. - - This handler has no external dependencies and simply echoes - back the input payload. Used for testing the runtime infrastructure. - """ - - def __init__(self) -> None: - self._logger = logging.getLogger("handler.local") - self._initialized = False - self._config: dict[str, Any] = {} - - @property - def handler_type(self) -> EnumHandlerType: - """Return handler type as EnumHandlerType (not str).""" - return EnumHandlerType.LOCAL - - async def initialize(self, config: dict) -> None: - """Initialize the local handler.""" - self._config = config - self._initialized = True - self._logger.info("Local handler initialized (dev/test only)") - - async def shutdown(self) -> None: - """Shutdown the local handler.""" - self._initialized = False - self._logger.info("Local handler shutdown") - - async def execute(self, envelope: ModelOnexEnvelope) -> ModelOnexEnvelope: - """Echo back the envelope payload. - - Supports operations: - - echo: Return payload as-is - - transform: Apply simple transformations - - error: Simulate an error response - """ - operation = envelope.operation - - if operation == "echo": - return self._success_response(envelope, envelope.payload) - - elif operation == "transform": - # Simple transformation: uppercase all string values - transformed = self._transform_payload(envelope.payload) - return self._success_response(envelope, transformed) - - elif operation == "error": - error_msg = envelope.payload.get("error_message", "Simulated error") - return self._error_response(envelope, error_msg) - - else: - return self._success_response(envelope, { - "operation": operation, - "payload": envelope.payload, - "message": "Unknown operation, echoing back", - }) - - async def health_check(self) -> dict: - """Check handler health.""" - return { - "healthy": self._initialized, - "handler_type": self.handler_type.value, - "dev_test_only": True, - } - - def _success_response(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: - """Create a success response envelope.""" - return ModelOnexEnvelope( - correlation_id=envelope.correlation_id, - causation_id=envelope.envelope_id, - source_node="local_handler", - target_node=envelope.source_node, - handler_type=envelope.handler_type, - operation=envelope.operation, - payload=payload, - is_response=True, - success=True, - ) - - def _error_response(self, envelope: ModelOnexEnvelope, error: str) -> ModelOnexEnvelope: - """Create an error response envelope.""" - return ModelOnexEnvelope( - correlation_id=envelope.correlation_id, - causation_id=envelope.envelope_id, - source_node="local_handler", - target_node=envelope.source_node, - handler_type=envelope.handler_type, - operation=envelope.operation, - is_response=True, - success=False, - error=error, - ) - - def _transform_payload(self, payload: dict) -> dict: - """Transform payload values.""" - result = {} - for key, value in payload.items(): - if isinstance(value, str): - result[key] = value.upper() - elif isinstance(value, dict): - result[key] = self._transform_payload(value) - else: - result[key] = value - return result -``` - -### 1.9 CLI Entry Point (Dev/Test) - -**File**: `src/omnibase_core/cli/runtime_host_cli.py` (NEW) - -```python -"""CLI for running the Runtime Host (dev/test mode - local handler only). - -For production use, use omnibase_infra.runtime.runtime_host_process which -provides real handlers and event bus integration. -""" -from __future__ import annotations - -import argparse -import asyncio -import logging -import sys -from pathlib import Path - -from omnibase_core.models.contracts.model_runtime_host_contract import ModelRuntimeHostContract -from omnibase_core.runtime.node_runtime import NodeRuntime -from omnibase_core.runtime.handlers.local_handler import LocalHandler - - -def setup_logging(level: str = "INFO") -> None: - """Configure logging.""" - logging.basicConfig( - level=getattr(logging, level.upper()), - format="%(asctime)s | %(levelname)-8s | %(name)s | %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - ) - - -async def run_runtime(contract_path: Path) -> None: - """Run the runtime host with the given contract (dev/test mode). - - This CLI only registers LocalHandler for testing purposes. - For production, use RuntimeHostProcess from omnibase_infra. - """ - # Load contract - contract = ModelRuntimeHostContract.from_yaml(contract_path) - - # Create runtime - runtime = NodeRuntime(contract) - - # Register local handler only (dev/test) - runtime.register_handler(LocalHandler()) - - # Load nodes - contracts_dir = contract_path.parent / contract.contracts_directory - if contracts_dir.exists(): - await runtime.load_nodes_from_directory(contracts_dir) - - # Initialize - await runtime.initialize() - - logging.info("Runtime initialized (dev/test mode - LocalHandler only)") - logging.info("For production, use omnibase_infra.runtime.runtime_host_process") - - # Keep running until interrupted - try: - while runtime.is_running: - await asyncio.sleep(1) - except KeyboardInterrupt: - pass - finally: - await runtime.shutdown() - - -def main() -> None: - """Main entry point.""" - parser = argparse.ArgumentParser( - description="ONEX Runtime Host (dev/test mode)", - epilog="For production use, see omnibase_infra.runtime.runtime_host_process", - ) - parser.add_argument( - "contract", - type=Path, - help="Path to runtime host contract YAML", - ) - parser.add_argument( - "--log-level", - default="INFO", - choices=["DEBUG", "INFO", "WARNING", "ERROR"], - help="Logging level", - ) - - args = parser.parse_args() - - setup_logging(args.log_level) - - if not args.contract.exists(): - print(f"Contract file not found: {args.contract}", file=sys.stderr) - sys.exit(1) - - asyncio.run(run_runtime(args.contract)) - - -if __name__ == "__main__": - main() -``` - -### 1.10 Phase 1 Success Criteria - -- [ ] `EnumNodeKind.RUNTIME_HOST` added and exported -- [ ] `EnumHandlerType` enum created (WITHOUT Kafka - that's an event bus) -- [ ] `ModelOnexEnvelope` model complete with serialization -- [ ] `ModelRuntimeHostContract` model with separate event_bus config -- [ ] `ProtocolHandler` abstract class with `EnumHandlerType` return type -- [ ] `NodeInstance` class with envelope handling (pass-through documented) -- [ ] `NodeRuntime` class - NO event loop, NO bus consumer -- [ ] `FileRegistry` class for contract loading -- [ ] `LocalHandler` working with dev/test warnings -- [ ] CLI entry point clearly marked as dev/test only -- [ ] Unit tests for all new classes (>90% coverage) -- [ ] mypy --strict passes -- [ ] **VERIFICATION**: No Kafka/HTTP/DB/Vault imports in omnibase_core - ---- - -## Phase 2: SPI Protocol Updates (omnibase_spi) - -**Duration**: 3-4 days -**Repository**: omnibase_spi -**Depends On**: Phase 1 complete - -### 2.1 Handler Protocol Export - -**File**: `src/omnibase_spi/protocols/__init__.py` (UPDATE) - -```python -# Add to existing exports -from omnibase_core.protocols.protocol_handler import ProtocolHandler - -__all__ = [ - # ... existing exports - "ProtocolHandler", -] -``` - -### 2.2 Event Bus Protocol Updates - -**File**: `src/omnibase_spi/protocols/protocol_event_bus.py` (UPDATE) - -The event bus protocol handles message transport - separate from handlers: - -```python -"""Protocol for event bus implementations.""" -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Callable, Awaitable - -if TYPE_CHECKING: - from omnibase_core.models.runtime.model_onex_envelope import ModelOnexEnvelope - - -class ProtocolEventBus(ABC): - """Abstract protocol for event bus implementations. - - The event bus is responsible for message transport - delivering envelopes - to NodeRuntime via RuntimeHostProcess. This is separate from ProtocolHandler - which handles per-request operations. - - Implementation Note: - - KafkaEventBus implements this protocol in omnibase_infra - - RuntimeHostProcess owns the event bus and calls runtime.route_envelope() - - NodeRuntime is transport-agnostic and only knows about envelopes - """ - - @abstractmethod - async def initialize(self, config: dict) -> None: - """Initialize the event bus connection.""" - ... - - @abstractmethod - async def shutdown(self) -> None: - """Gracefully shutdown the event bus.""" - ... - - @abstractmethod - async def publish_envelope(self, envelope: "ModelOnexEnvelope", topic: str) -> None: - """Publish an OnexEnvelope to a topic. - - Args: - envelope: Envelope to publish - topic: Target topic name - """ - ... - - @abstractmethod - async def subscribe( - self, - topic: str, - handler: Callable[["ModelOnexEnvelope"], Awaitable["ModelOnexEnvelope"]], - ) -> None: - """Subscribe to envelopes on a topic. - - Args: - topic: Topic to subscribe to - handler: Async callback for each received envelope - """ - ... - - @abstractmethod - async def start_consuming(self) -> None: - """Start the consumer loop. - - This runs until shutdown() is called. - """ - ... - - @abstractmethod - async def health_check(self) -> dict: - """Check event bus health.""" - ... -``` - -### 2.3 Phase 2 Success Criteria - -- [ ] `ProtocolHandler` exported from SPI -- [ ] `ProtocolEventBus` updated with envelope methods -- [ ] Clear separation documented: Handler vs EventBus -- [ ] Backward compatibility maintained -- [ ] Tests passing - ---- - -## Phase 3: Infrastructure Handlers (omnibase_infra) - -**Duration**: 2 weeks -**Repository**: omnibase_infra -**Depends On**: Phase 1 and Phase 2 complete - -### 3.1 Directory Structure - -``` -src/omnibase_infra/ -β”œβ”€β”€ handlers/ # Protocol handlers (per-request ops) -β”‚ β”œβ”€β”€ __init__.py -β”‚ β”œβ”€β”€ http_handler.py # HTTP REST handler -β”‚ β”œβ”€β”€ db_handler.py # PostgreSQL handler -β”‚ β”œβ”€β”€ vault_handler.py # Vault secrets handler -β”‚ β”œβ”€β”€ consul_handler.py # Consul service discovery handler -β”‚ └── llm_handler.py # LLM API handler (Phase 4) -β”œβ”€β”€ event_bus/ # Event bus implementations (transport) -β”‚ β”œβ”€β”€ __init__.py -β”‚ └── kafka_event_bus.py # KafkaEventBus (ProtocolEventBus impl) -β”œβ”€β”€ runtime/ # Runtime host process -β”‚ β”œβ”€β”€ __init__.py -β”‚ β”œβ”€β”€ runtime_host_process.py # Main process wrapper -β”‚ └── wiring.py # Single source of truth for handlers -β”œβ”€β”€ contracts/ # Example contracts -β”‚ β”œβ”€β”€ runtime/ -β”‚ β”‚ └── infra_runtime_host.yaml # Example runtime contract -β”‚ └── nodes/ -β”‚ β”œβ”€β”€ vault_adapter.yaml -β”‚ β”œβ”€β”€ consul_adapter.yaml -β”‚ └── postgres_adapter.yaml -└── ... (existing structure) -``` - -### 3.2 HTTP Handler (Strongly Typed) - -**File**: `src/omnibase_infra/handlers/http_handler.py` - -```python -"""HTTP REST protocol handler using httpx.""" -from __future__ import annotations - -import logging -from typing import Any - -import httpx - -from omnibase_core.enums.enum_handler_type import EnumHandlerType -from omnibase_core.models.runtime.model_onex_envelope import ModelOnexEnvelope -from omnibase_core.protocols.protocol_handler import ProtocolHandler - - -class HttpHandler(ProtocolHandler): - """HTTP REST protocol handler. - - Executes HTTP requests defined in OnexEnvelopes. - """ - - def __init__(self) -> None: - self._logger = logging.getLogger("handler.http") - self._client: httpx.AsyncClient | None = None - self._config: dict[str, Any] = {} - - @property - def handler_type(self) -> EnumHandlerType: - """Return handler type as EnumHandlerType.""" - return EnumHandlerType.HTTP - - async def initialize(self, config: dict) -> None: - """Initialize HTTP client.""" - self._config = config - timeout = config.get("timeout_seconds", 30) - - self._client = httpx.AsyncClient( - timeout=httpx.Timeout(timeout), - follow_redirects=True, - ) - self._logger.info("HTTP handler initialized") - - async def shutdown(self) -> None: - """Close HTTP client.""" - if self._client: - await self._client.aclose() - self._logger.info("HTTP handler shutdown") - - async def execute(self, envelope: ModelOnexEnvelope) -> ModelOnexEnvelope: - """Execute HTTP request from envelope.""" - if not self._client: - return self._error_response(envelope, "HTTP client not initialized") - - payload = envelope.payload - - try: - method = payload.get("method", "GET").upper() - url = payload.get("url") - headers = payload.get("headers", {}) - body = payload.get("body") - params = payload.get("params", {}) - - if not url: - return self._error_response(envelope, "URL is required") - - response = await self._client.request( - method=method, - url=url, - headers=headers, - json=body if body else None, - params=params, - ) - - return self._success_response(envelope, { - "status_code": response.status_code, - "headers": dict(response.headers), - "body": response.json() if response.headers.get("content-type", "").startswith("application/json") else response.text, - }) - - except httpx.TimeoutException: - return self._error_response(envelope, "Request timed out") - except Exception as e: - return self._error_response(envelope, str(e)) - - async def health_check(self) -> dict: - """Check HTTP handler health.""" - return { - "healthy": self._client is not None, - "handler_type": self.handler_type.value, - } - - def _success_response(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: - return ModelOnexEnvelope( - correlation_id=envelope.correlation_id, - causation_id=envelope.envelope_id, - source_node="http_handler", - target_node=envelope.source_node, - handler_type=envelope.handler_type, - operation=envelope.operation, - payload=payload, - is_response=True, - success=True, - ) - - def _error_response(self, envelope: ModelOnexEnvelope, error: str) -> ModelOnexEnvelope: - return ModelOnexEnvelope( - correlation_id=envelope.correlation_id, - causation_id=envelope.envelope_id, - source_node="http_handler", - target_node=envelope.source_node, - handler_type=envelope.handler_type, - operation=envelope.operation, - is_response=True, - success=False, - error=error, - ) -``` - -### 3.3 Database Handler (Strongly Typed) - -**File**: `src/omnibase_infra/handlers/db_handler.py` - -```python -"""PostgreSQL database protocol handler.""" -from __future__ import annotations - -import logging -from typing import Any - -import asyncpg - -from omnibase_core.enums.enum_handler_type import EnumHandlerType -from omnibase_core.models.runtime.model_onex_envelope import ModelOnexEnvelope -from omnibase_core.protocols.protocol_handler import ProtocolHandler - - -class DbHandler(ProtocolHandler): - """PostgreSQL database protocol handler. - - Executes database operations defined in OnexEnvelopes. - """ - - def __init__(self) -> None: - self._logger = logging.getLogger("handler.db") - self._pool: asyncpg.Pool | None = None - self._config: dict[str, Any] = {} - - @property - def handler_type(self) -> EnumHandlerType: - """Return handler type as EnumHandlerType.""" - return EnumHandlerType.DB - - async def initialize(self, config: dict) -> None: - """Initialize database connection pool.""" - self._config = config - - self._pool = await asyncpg.create_pool( - host=config.get("host", "localhost"), - port=config.get("port", 5432), - database=config.get("database", "omnibase"), - user=config.get("user", "postgres"), - password=config.get("password", ""), - min_size=config.get("pool_min_size", 2), - max_size=config.get("pool_max_size", 10), - ) - self._logger.info("Database handler initialized") - - async def shutdown(self) -> None: - """Close database connection pool.""" - if self._pool: - await self._pool.close() - self._logger.info("Database handler shutdown") - - async def execute(self, envelope: ModelOnexEnvelope) -> ModelOnexEnvelope: - """Execute database operation from envelope.""" - if not self._pool: - return self._error_response(envelope, "Database pool not initialized") - - payload = envelope.payload - operation = envelope.operation - - try: - if operation == "query": - return await self._execute_query(envelope, payload) - elif operation == "execute": - return await self._execute_command(envelope, payload) - elif operation == "transaction": - return await self._execute_transaction(envelope, payload) - else: - return self._error_response(envelope, f"Unknown database operation: {operation}") - - except asyncpg.PostgresError as e: - return self._error_response(envelope, f"Database error: {e}") - except Exception as e: - return self._error_response(envelope, str(e)) - - async def _execute_query(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: - """Execute a query and return results.""" - sql = payload.get("sql") - params = payload.get("params", []) - - if not sql: - return self._error_response(envelope, "SQL query is required") - - async with self._pool.acquire() as conn: - rows = await conn.fetch(sql, *params) - return self._success_response(envelope, { - "rows": [dict(row) for row in rows], - "row_count": len(rows), - }) - - async def _execute_command(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: - """Execute a command (INSERT, UPDATE, DELETE).""" - sql = payload.get("sql") - params = payload.get("params", []) - - if not sql: - return self._error_response(envelope, "SQL command is required") - - async with self._pool.acquire() as conn: - result = await conn.execute(sql, *params) - return self._success_response(envelope, { - "result": result, - }) - - async def _execute_transaction(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: - """Execute multiple statements in a transaction.""" - statements = payload.get("statements", []) - - if not statements: - return self._error_response(envelope, "Transaction statements required") - - async with self._pool.acquire() as conn: - async with conn.transaction(): - results = [] - for stmt in statements: - sql = stmt.get("sql") - params = stmt.get("params", []) - result = await conn.execute(sql, *params) - results.append(result) - - return self._success_response(envelope, { - "results": results, - "statement_count": len(results), - }) - - async def health_check(self) -> dict: - """Check database handler health.""" - if not self._pool: - return {"healthy": False, "handler_type": self.handler_type.value, "error": "Pool not initialized"} - - try: - async with self._pool.acquire() as conn: - await conn.fetchval("SELECT 1") - return {"healthy": True, "handler_type": self.handler_type.value} - except Exception as e: - return {"healthy": False, "handler_type": self.handler_type.value, "error": str(e)} - - def _success_response(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: - return ModelOnexEnvelope( - correlation_id=envelope.correlation_id, - causation_id=envelope.envelope_id, - source_node="db_handler", - target_node=envelope.source_node, - handler_type=envelope.handler_type, - operation=envelope.operation, - payload=payload, - is_response=True, - success=True, - ) - - def _error_response(self, envelope: ModelOnexEnvelope, error: str) -> ModelOnexEnvelope: - return ModelOnexEnvelope( - correlation_id=envelope.correlation_id, - causation_id=envelope.envelope_id, - source_node="db_handler", - target_node=envelope.source_node, - handler_type=envelope.handler_type, - operation=envelope.operation, - is_response=True, - success=False, - error=error, - ) -``` - -### 3.4 Kafka Event Bus (ProtocolEventBus, NOT ProtocolHandler) - -**File**: `src/omnibase_infra/event_bus/kafka_event_bus.py` - -```python -"""Kafka event bus implementation.""" -from __future__ import annotations - -import json -import logging -from typing import Any, Callable, Awaitable - -from aiokafka import AIOKafkaConsumer, AIOKafkaProducer - -from omnibase_core.models.runtime.model_onex_envelope import ModelOnexEnvelope -from omnibase_spi.protocols.protocol_event_bus import ProtocolEventBus - - -class KafkaEventBus(ProtocolEventBus): - """Kafka implementation of ProtocolEventBus. - - This is the transport layer that feeds envelopes into NodeRuntime. - It implements ProtocolEventBus, NOT ProtocolHandler. - - RuntimeHostProcess owns this and calls runtime.route_envelope() - for each message received. - """ - - def __init__(self) -> None: - self._logger = logging.getLogger("event_bus.kafka") - self._producer: AIOKafkaProducer | None = None - self._consumer: AIOKafkaConsumer | None = None - self._config: dict[str, Any] = {} - self._running = False - self._handler: Callable[[ModelOnexEnvelope], Awaitable[ModelOnexEnvelope]] | None = None - self._subscribed_topics: list[str] = [] - - async def initialize(self, config: dict) -> None: - """Initialize Kafka connections.""" - self._config = config - bootstrap_servers = config.get("bootstrap_servers", "localhost:9092") - consumer_group = config.get("consumer_group", "runtime-host") - - # Producer for publishing responses - self._producer = AIOKafkaProducer( - bootstrap_servers=bootstrap_servers, - value_serializer=lambda v: json.dumps(v).encode("utf-8"), - ) - await self._producer.start() - - # Consumer will be started when subscribe is called - self._bootstrap_servers = bootstrap_servers - self._consumer_group = consumer_group - - self._logger.info("Kafka event bus initialized") - - async def shutdown(self) -> None: - """Close Kafka connections.""" - self._running = False - - if self._producer: - await self._producer.stop() - - if self._consumer: - await self._consumer.stop() - - self._logger.info("Kafka event bus shutdown") - - async def publish_envelope(self, envelope: ModelOnexEnvelope, topic: str) -> None: - """Publish an envelope to a topic.""" - if not self._producer: - raise RuntimeError("Kafka producer not initialized") - - key = str(envelope.correlation_id).encode("utf-8") - value = envelope.model_dump(mode="json") - - await self._producer.send_and_wait(topic, value=value, key=key) - self._logger.debug(f"Published envelope to {topic}: {envelope.envelope_id}") - - async def subscribe( - self, - topic: str, - handler: Callable[[ModelOnexEnvelope], Awaitable[ModelOnexEnvelope]], - ) -> None: - """Subscribe to a topic with a handler.""" - self._handler = handler - self._subscribed_topics.append(topic) - self._logger.info(f"Subscribed to topic: {topic}") - - async def start_consuming(self) -> None: - """Start the consumer loop.""" - if not self._subscribed_topics: - self._logger.warning("No topics subscribed, not starting consumer") - return - - self._consumer = AIOKafkaConsumer( - *self._subscribed_topics, - bootstrap_servers=self._bootstrap_servers, - group_id=self._consumer_group, - value_deserializer=lambda v: json.loads(v.decode("utf-8")), - ) - await self._consumer.start() - - self._running = True - self._logger.info(f"Started consuming from: {self._subscribed_topics}") - - try: - async for msg in self._consumer: - if not self._running: - break - - try: - # Parse envelope from message - envelope = ModelOnexEnvelope.model_validate(msg.value) - - # Call the handler (which calls runtime.route_envelope) - if self._handler: - response = await self._handler(envelope) - - # Optionally publish response to response topic - if response.is_response and response.target_node: - response_topic = self._config.get("response_topic") - if response_topic: - await self.publish_envelope(response, response_topic) - - except Exception as e: - self._logger.exception(f"Error processing message: {e}") - - except Exception as e: - self._logger.exception(f"Consumer loop error: {e}") - finally: - self._running = False - - async def health_check(self) -> dict: - """Check event bus health.""" - return { - "healthy": self._producer is not None, - "running": self._running, - "subscribed_topics": self._subscribed_topics, - } -``` - -### 3.5 Vault Handler (Strongly Typed) - -**File**: `src/omnibase_infra/handlers/vault_handler.py` - -```python -"""Vault secrets management protocol handler.""" -from __future__ import annotations - -import logging -from typing import Any - -import hvac - -from omnibase_core.enums.enum_handler_type import EnumHandlerType -from omnibase_core.models.runtime.model_onex_envelope import ModelOnexEnvelope -from omnibase_core.protocols.protocol_handler import ProtocolHandler - - -class VaultHandler(ProtocolHandler): - """Vault secrets management protocol handler. - - Handles secret read/write operations via HashiCorp Vault. - """ - - def __init__(self) -> None: - self._logger = logging.getLogger("handler.vault") - self._client: hvac.Client | None = None - self._config: dict[str, Any] = {} - - @property - def handler_type(self) -> EnumHandlerType: - """Return handler type as EnumHandlerType.""" - return EnumHandlerType.VAULT - - async def initialize(self, config: dict) -> None: - """Initialize Vault client.""" - self._config = config - - self._client = hvac.Client( - url=config.get("url", "http://localhost:8200"), - token=config.get("token"), - namespace=config.get("namespace"), - ) - self._logger.info("Vault handler initialized") - - async def shutdown(self) -> None: - """Close Vault client.""" - self._client = None - self._logger.info("Vault handler shutdown") - - async def execute(self, envelope: ModelOnexEnvelope) -> ModelOnexEnvelope: - """Execute Vault operation from envelope.""" - if not self._client: - return self._error_response(envelope, "Vault client not initialized") - - operation = envelope.operation - payload = envelope.payload - - try: - if operation == "get_secret": - return await self._get_secret(envelope, payload) - elif operation == "set_secret": - return await self._set_secret(envelope, payload) - elif operation == "delete_secret": - return await self._delete_secret(envelope, payload) - elif operation == "list_secrets": - return await self._list_secrets(envelope, payload) - else: - return self._error_response(envelope, f"Unknown Vault operation: {operation}") - - except hvac.exceptions.VaultError as e: - return self._error_response(envelope, f"Vault error: {e}") - except Exception as e: - return self._error_response(envelope, str(e)) - - async def _get_secret(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: - """Get a secret from Vault.""" - path = payload.get("path") - mount_point = payload.get("mount_point", "secret") - - if not path: - return self._error_response(envelope, "Secret path is required") - - response = self._client.secrets.kv.v2.read_secret_version( - path=path, - mount_point=mount_point, - ) - - return self._success_response(envelope, { - "data": response["data"]["data"], - "metadata": response["data"]["metadata"], - }) - - async def _set_secret(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: - """Set a secret in Vault.""" - path = payload.get("path") - data = payload.get("data") - mount_point = payload.get("mount_point", "secret") - - if not path or not data: - return self._error_response(envelope, "Secret path and data are required") - - response = self._client.secrets.kv.v2.create_or_update_secret( - path=path, - secret=data, - mount_point=mount_point, - ) - - return self._success_response(envelope, { - "version": response["data"]["version"], - "created_time": response["data"]["created_time"], - }) - - async def _delete_secret(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: - """Delete a secret from Vault.""" - path = payload.get("path") - mount_point = payload.get("mount_point", "secret") - - if not path: - return self._error_response(envelope, "Secret path is required") - - self._client.secrets.kv.v2.delete_metadata_and_all_versions( - path=path, - mount_point=mount_point, - ) - - return self._success_response(envelope, {"deleted": True}) - - async def _list_secrets(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: - """List secrets at a path.""" - path = payload.get("path", "") - mount_point = payload.get("mount_point", "secret") - - response = self._client.secrets.kv.v2.list_secrets( - path=path, - mount_point=mount_point, - ) - - return self._success_response(envelope, { - "keys": response["data"]["keys"], - }) - - async def health_check(self) -> dict: - """Check Vault handler health.""" - if not self._client: - return {"healthy": False, "handler_type": self.handler_type.value, "error": "Client not initialized"} - - try: - health = self._client.sys.read_health_status(method="GET") - return { - "healthy": health.get("initialized", False) and not health.get("sealed", True), - "handler_type": self.handler_type.value, - "vault_version": health.get("version"), - } - except Exception as e: - return {"healthy": False, "handler_type": self.handler_type.value, "error": str(e)} - - def _success_response(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: - return ModelOnexEnvelope( - correlation_id=envelope.correlation_id, - causation_id=envelope.envelope_id, - source_node="vault_handler", - target_node=envelope.source_node, - handler_type=envelope.handler_type, - operation=envelope.operation, - payload=payload, - is_response=True, - success=True, - ) - - def _error_response(self, envelope: ModelOnexEnvelope, error: str) -> ModelOnexEnvelope: - return ModelOnexEnvelope( - correlation_id=envelope.correlation_id, - causation_id=envelope.envelope_id, - source_node="vault_handler", - target_node=envelope.source_node, - handler_type=envelope.handler_type, - operation=envelope.operation, - is_response=True, - success=False, - error=error, - ) -``` - -### 3.6 Handler Wiring (Single Source of Truth) - -**File**: `src/omnibase_infra/runtime/wiring.py` - -```python -"""Handler registration and wiring utilities. - -This is the SINGLE SOURCE OF TRUTH for handler registration. -RuntimeHostProcess uses this - no duplicate logic. -""" -from __future__ import annotations - -from typing import TYPE_CHECKING - -from omnibase_core.enums.enum_handler_type import EnumHandlerType -from omnibase_core.runtime.handlers.local_handler import LocalHandler - -from omnibase_infra.handlers.http_handler import HttpHandler -from omnibase_infra.handlers.db_handler import DbHandler -from omnibase_infra.handlers.vault_handler import VaultHandler - -if TYPE_CHECKING: - from omnibase_core.protocols.protocol_handler import ProtocolHandler - from omnibase_core.runtime.node_runtime import NodeRuntime - - -# Single source of truth for handler classes -HANDLER_REGISTRY: dict[EnumHandlerType, type["ProtocolHandler"]] = { - EnumHandlerType.LOCAL: LocalHandler, # Dev/test only - EnumHandlerType.HTTP: HttpHandler, - EnumHandlerType.DB: DbHandler, - EnumHandlerType.VAULT: VaultHandler, - # EnumHandlerType.CONSUL: ConsulHandler, # TODO: implement - # EnumHandlerType.LLM: LlmHandler, # TODO: implement -} - - -def register_handlers_from_config( - runtime: "NodeRuntime", - handler_configs: list[dict], -) -> None: - """Register handlers with the runtime based on config. - - This is the ONLY place where handlers are registered. - - Args: - runtime: NodeRuntime instance - handler_configs: List of handler config dicts from contract - """ - for config in handler_configs: - handler_type = EnumHandlerType(config["handler_type"]) - enabled = config.get("enabled", True) - - if not enabled: - continue - - handler_class = HANDLER_REGISTRY.get(handler_type) - if handler_class: - handler = handler_class() - runtime.register_handler(handler) - else: - raise ValueError(f"Unknown handler type: {handler_type}") - - -def get_handler_class(handler_type: EnumHandlerType) -> type["ProtocolHandler"] | None: - """Get handler class by type.""" - return HANDLER_REGISTRY.get(handler_type) - - -def list_available_handlers() -> list[EnumHandlerType]: - """List all available handler types.""" - return list(HANDLER_REGISTRY.keys()) -``` - -### 3.7 RuntimeHostProcess (Uses Wiring, Owns Event Bus) - -**File**: `src/omnibase_infra/runtime/runtime_host_process.py` - -```python -"""Runtime Host Process - Infrastructure-level wrapper for NodeRuntime.""" -from __future__ import annotations - -import asyncio -import logging -import signal -from pathlib import Path -from typing import TYPE_CHECKING - -from omnibase_core.models.contracts.model_runtime_host_contract import ModelRuntimeHostContract -from omnibase_core.runtime.node_runtime import NodeRuntime - -from omnibase_infra.event_bus.kafka_event_bus import KafkaEventBus -from omnibase_infra.runtime.wiring import register_handlers_from_config - -if TYPE_CHECKING: - from omnibase_spi.protocols.protocol_event_bus import ProtocolEventBus - - -class RuntimeHostProcess: - """Infrastructure-level process wrapper for NodeRuntime. - - RuntimeHostProcess is responsible for: - - Loading and validating the runtime contract - - Registering infrastructure handlers (via wiring.py) - - Owning and managing the event bus (ProtocolEventBus) - - Driving NodeRuntime with envelopes from the event bus - - Managing process lifecycle and signals - - Providing health/metrics endpoints - - IMPORTANT: - This is where event bus consumption happens. - NodeRuntime is transport-agnostic - it only knows about envelopes. - This class bridges the transport (Kafka) to the runtime. - """ - - def __init__(self, contract_path: Path) -> None: - self._contract_path = contract_path - self._logger = logging.getLogger("runtime_host_process") - self._runtime: NodeRuntime | None = None - self._event_bus: "ProtocolEventBus | None" = None - self._shutdown_requested = False - - async def start(self) -> None: - """Start the runtime host process.""" - self._logger.info(f"Starting Runtime Host Process: {self._contract_path}") - - # Load contract - contract = ModelRuntimeHostContract.from_yaml(self._contract_path) - - # Create runtime - self._runtime = NodeRuntime(contract) - - # Register handlers (single source of truth: wiring.py) - handler_configs = [ - {"handler_type": h.handler_type.value, "enabled": h.enabled, **h.config} - for h in contract.handlers - ] - register_handlers_from_config(self._runtime, handler_configs) - - # Load nodes - contracts_dir = self._contract_path.parent / contract.contracts_directory - if contracts_dir.exists(): - await self._runtime.load_nodes_from_directory(contracts_dir) - - # Initialize runtime (handlers + nodes) - await self._runtime.initialize() - - # Initialize event bus if enabled - if contract.event_bus.enabled: - await self._setup_event_bus(contract) - - # Setup signal handlers - self._setup_signals() - - self._logger.info("Runtime Host Process started") - - # Start event bus consumer loop (this is where envelopes flow in) - if self._event_bus: - await self._event_bus.start_consuming() - else: - # No event bus - just wait for shutdown - while not self._shutdown_requested: - await asyncio.sleep(1) - - async def _setup_event_bus(self, contract: ModelRuntimeHostContract) -> None: - """Setup the event bus.""" - self._event_bus = KafkaEventBus() - await self._event_bus.initialize(contract.event_bus.config) - - # Subscribe to command topic - command_topic = contract.event_bus.config.get( - "command_topic", - f"onex.cmd.runtime.{contract.name}.v1" - ) - - # The handler bridges event bus to NodeRuntime - await self._event_bus.subscribe( - command_topic, - self._runtime.route_envelope, # This is where the bridge happens - ) - - self._logger.info(f"Event bus subscribed to: {command_topic}") - - async def stop(self) -> None: - """Stop the runtime host process.""" - self._shutdown_requested = True - - if self._event_bus: - await self._event_bus.shutdown() - - if self._runtime: - await self._runtime.shutdown() - - self._logger.info("Runtime Host Process stopped") - - def _setup_signals(self) -> None: - """Setup signal handlers for graceful shutdown.""" - loop = asyncio.get_running_loop() - - for sig in (signal.SIGTERM, signal.SIGINT): - loop.add_signal_handler(sig, self._signal_handler) - - def _signal_handler(self) -> None: - """Handle shutdown signals.""" - if not self._shutdown_requested: - self._shutdown_requested = True - self._logger.info("Shutdown signal received") - asyncio.create_task(self.stop()) -``` - -### 3.8 Example Runtime Contract (Clean Separation) - -**File**: `src/omnibase_infra/contracts/runtime/infra_runtime_host.yaml` - -```yaml -# Infrastructure Runtime Host Contract -name: "infra_runtime_host" -version: "1.0.0" -description: "ONEX Infrastructure Runtime Host - hosts all infrastructure nodes" - -# Node Configuration -contracts_directory: "../nodes" -nodes: - - slug: "vault_adapter" - contract_path: "vault_adapter.yaml" - enabled: true - - slug: "consul_adapter" - contract_path: "consul_adapter.yaml" - enabled: true - - slug: "postgres_adapter" - contract_path: "postgres_adapter.yaml" - enabled: true - -# Handler Configuration (per-request operations) -# Note: Kafka is NOT a handler - it's the event bus (see below) -handlers: - # LocalHandler is dev/test only - DO NOT enable in production - # - handler_type: "local" - # enabled: false - - - handler_type: "http" - enabled: true - config: - timeout_seconds: 30 - - - handler_type: "db" - enabled: true - config: - host: "${POSTGRES_HOST}" - port: "${POSTGRES_PORT}" - database: "${POSTGRES_DB}" - user: "${POSTGRES_USER}" - password: "${POSTGRES_PASSWORD}" - pool_min_size: 2 - pool_max_size: 10 - - - handler_type: "vault" - enabled: true - config: - url: "${VAULT_ADDR}" - token: "${VAULT_TOKEN}" - namespace: "${VAULT_NAMESPACE}" - -# Event Bus Configuration (transport layer - feeds envelopes to runtime) -# This is SEPARATE from handlers. Kafka is a transport, not a per-request handler. -event_bus: - enabled: true - config: - bootstrap_servers: "${KAFKA_BOOTSTRAP_SERVERS}" - consumer_group: "infra-runtime-host" - command_topic: "onex.cmd.runtime.infra.v1" - response_topic: "onex.evt.runtime.infra.v1" - -# Health & Metrics -health_endpoint: - enabled: true - port: 8080 - path: "/health" - -metrics_endpoint: - enabled: true - port: 9090 - path: "/metrics" - -# Runtime Settings -max_concurrent_operations: 100 -shutdown_timeout_seconds: 30 -``` - -### 3.9 Phase 3 Success Criteria - -- [ ] HttpHandler with `EnumHandlerType.HTTP` return type -- [ ] DbHandler with `EnumHandlerType.DB` return type -- [ ] VaultHandler with `EnumHandlerType.VAULT` return type -- [ ] KafkaEventBus implements `ProtocolEventBus` (NOT ProtocolHandler) -- [ ] `wiring.py` is single source of truth for handler registration -- [ ] RuntimeHostProcess uses `wiring.py` (no duplicate handler map) -- [ ] RuntimeHostProcess owns event bus and calls `runtime.route_envelope()` -- [ ] Example contract shows clean handler vs event_bus separation -- [ ] Integration tests for each handler -- [ ] End-to-end test: event bus β†’ runtime β†’ handler β†’ response - ---- - -## Phase 4-5: Integration, Testing & Deployment - -*(Phases 4 and 5 remain largely the same as before, with tests updated to use the new architecture)* - -### Key Test Scenarios - -1. **LocalHandler echo** (dev/test validation) -2. **HttpHandler external call** (real HTTP request) -3. **DbHandler query** (real database query) -4. **VaultHandler secret** (real Vault operation) -5. **Event bus β†’ NodeRuntime β†’ handler flow** (full integration) -6. **Multi-node routing** (envelope routing to correct node) - ---- - -## Architectural Sanity Checklist - -Use this checklist to verify the implementation maintains architectural invariants: - -### omnibase_core -- [ ] Has: `EnumHandlerType`, `ModelOnexEnvelope`, `ModelRuntimeHostContract`, `NodeInstance`, `NodeRuntime`, `ProtocolHandler`, `LocalHandler` (dev-only), CLI test runtime -- [ ] Has **NO** Kafka/HTTP/DB/Vault imports anywhere -- [ ] `NodeRuntime` has NO `_event_bus_loop` method -- [ ] `ProtocolHandler.handler_type` returns `EnumHandlerType` (not `str`) -- [ ] `LocalHandler` has dev/test warnings in docstring - -### omnibase_spi -- [ ] Exposes: `ProtocolHandler`, `ProtocolEventBus` -- [ ] `ProtocolEventBus` has envelope methods -- [ ] Clear documentation of handler vs event bus distinction - -### omnibase_infra -- [ ] Handlers return `EnumHandlerType` (not `str`) -- [ ] `KafkaEventBus` implements `ProtocolEventBus` (NOT `ProtocolHandler`) -- [ ] `wiring.py` is single source of truth -- [ ] `RuntimeHostProcess` uses `wiring.py` for handler registration -- [ ] `RuntimeHostProcess` owns event bus, calls `runtime.route_envelope()` -- [ ] No duplicate handler registration logic - -### Contracts -- [ ] `handlers` section contains only per-request handlers (http, db, vault) -- [ ] `event_bus` section is separate from handlers -- [ ] `local` handler is NOT enabled in production contracts - ---- - -## Summary Timeline - -| Week | Phase | Deliverables | -|------|-------|--------------| -| 1 | Phase 0 + Phase 1 Start | Prerequisites validated, core types begun | -| 2 | Phase 1 Complete | All core classes (transport-agnostic), CLI, local handler | -| 3 | Phase 2 + Phase 3 Start | SPI updates, HTTP handler, KafkaEventBus | -| 4 | Phase 3 Continue | DB, Vault handlers, wiring | -| 5 | Phase 3 Complete + Phase 4 Start | RuntimeHostProcess, integration tests | -| 6 | Phase 4 Complete | All tests passing, benchmarks met | -| 7-8 | Phase 5 | Docker, deployment, migration | - ---- - -## Success Metrics - -| Metric | Target | Measurement | -|--------|--------|-------------| -| Memory per 10 nodes | <200MB | tracemalloc | -| Envelope throughput | >100/sec | Benchmark suite | -| Handler latency (local) | <1ms | p99 latency | -| Handler latency (http) | <100ms | p99 latency | -| Handler latency (db) | <50ms | p99 latency | -| Test coverage | >90% | pytest-cov | -| Migration downtime | 0 | Shadow deployment | -| Core Kafka imports | 0 | grep verification | - ---- - -*This implementation plan was created on December 3, 2025 and updated with architectural refinements.* diff --git a/docs/milestones/BETA_v0.2.0_HARDENING.md b/docs/milestones/BETA_v0.2.0_HARDENING.md new file mode 100644 index 0000000000..fe979f9e69 --- /dev/null +++ b/docs/milestones/BETA_v0.2.0_HARDENING.md @@ -0,0 +1,860 @@ +# Beta Hardening (v0.2.0) - Milestone Details + +> **Navigation**: [Back to Overview](../MVP_PLAN.md) | [Previous: MVP Core](./MVP_v0.1.0_CORE.md) | [Next: Production](./PRODUCTION_v0.3.0.md) + +**Repository**: omnibase_infra +**Target Version**: v0.2.0 +**Timeline**: Sprint 3-4 +**Issue Count**: 22 +**Prerequisite**: MVP Core (v0.1.0) must be complete + +--- + +## Beta Philosophy + +**Beta (v0.2.0)**: Harden for production use +- KafkaEventBus with backpressure +- Vault + Consul handlers +- Full retry/rate-limit policies +- Integration tests with real services +- Observability layer + +This milestone builds upon the MVP foundation to add production-readiness features including: +- External service integration (Kafka, Vault, Consul) +- Resilience patterns (retries, circuit breakers, rate limiting) +- Full observability (structured logging, metrics) +- Comprehensive integration testing + +--- + +## Beta Scope Boundaries (Do Not Implement in Beta) + +The following are explicitly OUT OF SCOPE until Production (v0.3.0): + +- **No multi-region failover** - Single cluster only +- **No automatic topic creation** - Manual infra bootstrap +- **No dynamic handler discovery** - Static wiring only + +--- + +## Beta Contract Additions + +Building on MVP simplified contracts, Beta adds full configuration: + +```yaml +runtime: + name: "my_runtime_host" + version: "1.0.0" + + event_bus: + kind: "kafka" + config: + bootstrap_servers: "${KAFKA_BOOTSTRAP_SERVERS}" + consumer_group: "runtime-host-group" + topics: + input: "onex.tenant.domain.cmd.v1" + output: "onex.tenant.domain.evt.v1" + + handlers: + - type: "http" + config: + timeout_ms: 30000 + retry_policy: + max_retries: 3 + backoff_strategy: "exponential" + - type: "db" + config: + pool_size: 10 + - type: "vault" + config: + address: "${VAULT_ADDR}" + - type: "consul" + config: + address: "${CONSUL_ADDR}" + + nodes: + - slug: "node_a" + - slug: "node_b" +``` + +**Cross-Version Compatibility**: +- v0.2 infra MUST accept v0.1 contracts but MUST warn on deprecated fields +- v0.1 contracts MUST NOT use any Beta-only fields +- Contract version mismatch MUST produce clear error message with upgrade instructions + +--- + +## Topic Naming Schema + +Topics follow the pattern: `onex....v` + +**Required Directions**: +- `.cmd` - Command/input topic (requests INTO the runtime) +- `.evt` - Event/output topic (responses FROM the runtime) + +**MVP Validation Rules** (enforced by contract importer): +- No invalid characters (alphanumeric, dots, hyphens only) +- Version segment must be numeric (e.g., `v1`, `v2`) +- Tenant and domain segments required + +**Examples**: +- `onex.acme.orders.cmd.v1` - Input commands +- `onex.acme.orders.evt.v1` - Output events + +**Reserved Namespaces** (MUST NOT be used by user contracts): +- `onex.internal.*` - System-internal communication +- `onex.debug.*` - Debug and diagnostics +- `onex.metrics.*` - Metrics and telemetry +- `onex.health.*` - Health check signals +- `onex.admin.*` - Administrative commands + +**Allowed Characters in Segments**: `[a-z0-9_-]` (lowercase alphanumeric, underscore, hyphen) + +**Topic Naming Examples**: + +| Example | Valid? | Reason | +|---------|--------|--------| +| `onex.acme.orders.cmd.v1` | Valid | Correct format | +| `onex.acme.orders.evt.v1` | Valid | Correct format | +| `onex.acme.user-service.cmd.v1` | Valid | Hyphen allowed | +| `onex.acme.user_service.cmd.v1` | Valid | Underscore allowed | +| `onex.ACME.orders.cmd.v1` | Invalid | Uppercase not allowed | +| `onex.acme.orders.command.v1` | Invalid | Must be `cmd` or `evt` | +| `onex.acme.orders.cmd.1` | Invalid | Version must have `v` prefix | +| `onex.acme.orders.cmd` | Invalid | Missing version | +| `acme.orders.cmd.v1` | Invalid | Missing `onex.` prefix | +| `onex..orders.cmd.v1` | Invalid | Empty segment | +| `onex.acme.orders.query.v1` | Invalid | `query` not a valid direction | +| `onex.internal.debug.evt.v1` | Invalid | `internal` is reserved | + +**Validation Regex**: +```python +TOPIC_PATTERN = re.compile( + r"^onex\.[a-z0-9][a-z0-9_-]*\.[a-z0-9][a-z0-9_-]*\.(cmd|evt)\.v[0-9]+$" +) + +def validate_topic(topic: str) -> bool: + if topic.startswith("onex.internal.") or topic.startswith("onex.debug."): + raise ValueError(f"Reserved namespace: {topic}") + return bool(TOPIC_PATTERN.match(topic)) +``` + +--- + +## Phase 0: Cross-Repo Guardrails & Invariants (Beta Issues) + +--- + +### Issue 0.3: Protocol ownership verification [BETA] + +**Title**: Verify infra never declares new protocols +**Type**: Infrastructure +**Priority**: High +**Labels**: `architecture`, `ci`, `guardrails` +**Milestone**: v0.2.0 Beta + +**Description**: +Ensure `omnibase_infra` only IMPLEMENTS protocols defined in `omnibase_spi`, never declares new abstract protocols. + +**Acceptance Criteria**: +- [ ] AST or grep check for `class.*Protocol.*ABC` in infra +- [ ] Only allow concrete implementations of SPI protocols +- [ ] Clear error message on violation +- [ ] Runs in CI + +--- + +### Issue 0.4: Version compatibility matrix check [BETA] + +**Title**: Runtime version compatibility verification +**Type**: Infrastructure +**Priority**: High +**Labels**: `ci`, `versioning` +**Milestone**: v0.2.0 Beta + +**Description**: +On startup, verify that `omnibase_core` and `omnibase_spi` versions meet minimum requirements. + +**Compatibility Matrix**: +- Infra v0.1.0 -> Core >=0.4.0, SPI >=0.3.0 + +**Acceptance Criteria**: +- [ ] `BaseRuntimeHostProcess` logs resolved versions on startup +- [ ] Fails fast with clear message if incompatible +- [ ] Version matrix documented in README +- [ ] CI test for version check logic + +--- + +## Phase 1: Core Types (omnibase_core) - Beta Issues + +--- + +### Issue 1.12: Create ModelHandlerBindingConfig model [BETA] + +**Title**: Implement formalized handler config schema +**Type**: Feature +**Priority**: High +**Labels**: `architecture`, `model`, `core` +**Milestone**: v0.2.0 Beta + +**Description**: +Create a formal schema for handler binding configuration with validation and defaults. + +**File**: `src/omnibase_core/models/runtime/model_handler_binding_config.py` (NEW) + +**Fields**: +- `handler_type: EnumHandlerType` +- `name: str` (optional, defaults to handler_type) +- `enabled: bool = True` +- `priority: int = 0` +- `config_ref: str | None` (reference to external config) +- `retry_policy: ModelRetryPolicy | None` +- `timeout_ms: int = 30000` +- `rate_limit_per_second: float | None` + +**Sub-model** `ModelRetryPolicy`: +- `max_retries: int = 3` +- `backoff_strategy: Literal["fixed", "exponential"] = "exponential"` +- `base_delay_ms: int = 100` +- `max_delay_ms: int = 5000` + +**Acceptance Criteria**: +- [ ] Full validation with Pydantic +- [ ] Sensible defaults for all optional fields +- [ ] Used by `ModelRuntimeHostContract.handlers` +- [ ] Unit tests for validation edge cases +- [ ] mypy --strict passes + +--- + +### Issue 1.13: Extend error taxonomy (core) [BETA] + +**Title**: Complete core error hierarchy for runtime +**Type**: Feature +**Priority**: High +**Labels**: `architecture`, `errors`, `core` +**Milestone**: v0.2.0 Beta + +**Description**: +Complete the error hierarchy with additional error types. + +**File**: `src/omnibase_core/errors/runtime_errors.py` (UPDATE) + +**Additional Classes**: +- `HandlerNotFoundError(RuntimeHostError)` +- `NodeNotFoundError(RuntimeHostError)` +- `EnvelopeValidationError(RuntimeHostError)` +- `ProtocolConfigurationError(RuntimeHostError)` +- `SecretResolutionError(RuntimeHostError)` + +**Acceptance Criteria**: +- [ ] All inherit from `RuntimeHostError` +- [ ] Structured fields: `handler_type`, `operation`, `correlation_id` +- [ ] NodeRuntime only sees abstract errors, not library exceptions +- [ ] Unit tests for each error class +- [ ] mypy --strict passes + +--- + +## Phase 3: Infrastructure Handlers (omnibase_infra) - Beta Issues + +--- + +### Issue 3.9: Implement VaultHandler [BETA] + +**Title**: Create Vault secrets management protocol handler +**Type**: Feature +**Priority**: High +**Labels**: `handler`, `infrastructure`, `secrets` +**Milestone**: v0.2.0 Beta + +**Description**: +Implement `VaultHandler` for HashiCorp Vault operations using hvac. + +**File**: `src/omnibase_infra/handlers/vault_handler.py` + +**Operations**: `get_secret`, `set_secret`, `delete_secret`, `list_secrets` + +**Acceptance Criteria**: +- [ ] Returns `EnumHandlerType.VAULT` (not str) +- [ ] Uses hvac client +- [ ] KV v2 support +- [ ] Maps `hvac.VaultError` -> `HandlerExecutionError` +- [ ] Unit tests with mock client +- [ ] Integration test with dev Vault +- [ ] mypy --strict passes + +--- + +### Issue 3.10: Implement ConsulHandler [BETA] + +**Title**: Create Consul service discovery protocol handler +**Type**: Feature +**Priority**: Medium +**Labels**: `handler`, `infrastructure`, `discovery` +**Milestone**: v0.2.0 Beta + +**Description**: +Implement `ConsulHandler` for Consul service discovery operations. + +**File**: `src/omnibase_infra/handlers/consul_handler.py` + +**Operations**: `register_service`, `deregister_service`, `get_service`, `list_services`, `health_check_service` + +**Acceptance Criteria**: +- [ ] Returns `EnumHandlerType.CONSUL` (not str) +- [ ] Uses python-consul or httpx +- [ ] Service registration/deregistration +- [ ] Health check integration +- [ ] Unit tests with mock responses +- [ ] Integration test with dev Consul +- [ ] mypy --strict passes + +--- + +### Issue 3.11: Implement KafkaEventBus [BETA] + +**Title**: Create Kafka event bus implementation +**Type**: Feature +**Priority**: High +**Labels**: `event-bus`, `infrastructure`, `messaging` +**Milestone**: v0.2.0 Beta + +**Description**: +Implement `KafkaEventBus` that implements `ProtocolEventBus` (NOT ProtocolHandler). + +**File**: `src/omnibase_infra/event_bus/kafka_event_bus.py` + +**Methods**: `initialize`, `shutdown`, `publish_envelope`, `subscribe`, `start_consuming`, `health_check` + +**Backpressure Config**: +- `max_inflight_envelopes: int = 100` +- `pause_consumption_threshold: int = 80` +- `circuit_breaker_threshold: int = 5` (consecutive failures) + +**Ordering Guarantees**: Runtime MUST NOT assume global ordering. Kafka provides partition-level ordering only. + +**Response Pattern**: Event bus handlers must return envelopes; BaseRuntimeHostProcess is responsible for publishing responses. + +**Acceptance Criteria**: +- [ ] Implements `ProtocolEventBus` (NOT ProtocolHandler) +- [ ] NO `handler_type` property (event bus, not handler) +- [ ] Uses aiokafka +- [ ] Proper envelope serialization/deserialization +- [ ] Configurable topics and consumer group +- [ ] Backpressure: pauses consumption when queue full +- [ ] Circuit breaker: stops after N consecutive handler failures +- [ ] Unit tests with mock Kafka +- [ ] Integration test with test Kafka +- [ ] mypy --strict passes + +--- + +### Issue 3.12: Create SecretResolver [BETA] + +**Title**: Implement centralized secret resolution +**Type**: Feature +**Priority**: High +**Labels**: `infrastructure`, `secrets` +**Milestone**: v0.2.0 Beta + +**Description**: +Create a centralized secret resolver so handlers never call `os.getenv` directly. + +**File**: `src/omnibase_infra/runtime/secret_resolver.py` + +**Interface**: +```python +class SecretResolver: + async def get_secret(self, logical_name: str) -> str: ... + async def get_secrets(self, logical_names: list[str]) -> dict[str, str]: ... +``` + +**Sources** (priority order): +1. Vault (if configured) +2. Environment variables +3. File-based secrets (K8s secrets volume) + +**Acceptance Criteria**: +- [ ] Typed interface for secret requests +- [ ] Handlers never call `os.getenv` directly +- [ ] Vault integration optional +- [ ] Unit tests with mocked sources +- [ ] mypy --strict passes + +--- + +### Issue 3.13: Create HandlerConfigResolver [BETA] + +**Title**: Implement handler config resolution layer +**Type**: Feature +**Priority**: High +**Labels**: `infrastructure`, `configuration` +**Milestone**: v0.2.0 Beta + +**Description**: +Create a resolver that normalizes handler configs from multiple sources. + +**File**: `src/omnibase_infra/runtime/handler_config_resolver.py` + +**Sources**: +- Contract YAML +- Environment variables (override) +- Vault (secrets only via SecretResolver) +- Files (config refs) + +**Acceptance Criteria**: +- [ ] Resolves `config_ref` to actual config dict +- [ ] Merges environment overrides +- [ ] Returns fully validated `ModelHandlerBindingConfig` +- [ ] Clear error messages for missing required fields +- [ ] Unit tests for resolution logic +- [ ] mypy --strict passes + +--- + +### Issue 3.14: Extend infra error hierarchy [BETA] + +**Title**: Complete infrastructure error taxonomy +**Type**: Feature +**Priority**: High +**Labels**: `infrastructure`, `errors` +**Milestone**: v0.2.0 Beta + +**Description**: +Complete structured error hierarchy for all infra components. + +**File**: `src/omnibase_infra/errors/infra_errors.py` (UPDATE) + +**Additional Classes**: +- `ProtocolConfigurationError(RuntimeHostError)` +- `SecretResolutionError(RuntimeHostError)` + +**Acceptance Criteria**: +- [ ] All inherit from core `RuntimeHostError` +- [ ] Handlers map raw exceptions to these types +- [ ] Structured fields: `handler_type`, `operation`, `correlation_id` +- [ ] Unit tests for each error class +- [ ] mypy --strict passes + +--- + +### Issue 3.15: Implement observability layer [BETA] + +**Title**: Create structured logging and metrics infrastructure +**Type**: Feature +**Priority**: High +**Labels**: `infrastructure`, `observability` +**Milestone**: v0.2.0 Beta + +**Description**: +Create centralized observability configuration owned by BaseRuntimeHostProcess. + +**Files**: +- `src/omnibase_infra/observability/logging_config.py` +- `src/omnibase_infra/observability/metrics.py` + +**Logging Requirements**: +- JSON structured logs +- Standard fields: `runtime_id`, `node_id`, `handler_type`, `envelope_id`, `correlation_id` +- Handlers use shared logger (no ad-hoc loggers) + +**Metrics Hooks**: +- Envelope count in/out per handler +- Handler latency histogram +- Event bus lag metrics (consumer group offset) + +**Acceptance Criteria**: +- [ ] `BaseRuntimeHostProcess` sets up global logging config +- [ ] Handlers get loggers from centralized factory +- [ ] All logs include correlation_id when available +- [ ] Metrics exposed for collection +- [ ] Unit tests for log formatting +- [ ] mypy --strict passes + +--- + +### Issue 3.16: Implement health HTTP endpoint [BETA] + +**Title**: Create HTTP health endpoint server +**Type**: Feature +**Priority**: Medium +**Labels**: `infrastructure`, `health` +**Milestone**: v0.2.0 Beta + +**Description**: +Optional HTTP server exposing health endpoints for K8s probes. + +**File**: `src/omnibase_infra/runtime/health_server.py` + +**Endpoints**: +- `/health/live` - Is process alive +- `/health/ready` - Can process envelopes +- `/health/handlers` - Per-handler status snapshot + +**Acceptance Criteria**: +- [ ] Optional (can be disabled in config) +- [ ] Configurable port +- [ ] Minimal dependencies (use `aiohttp` or built-in) +- [ ] Integrates with Docker healthchecks +- [ ] Integrates with K8s probes +- [ ] Unit tests +- [ ] mypy --strict passes + +--- + +### Issue 3.17: Implement handler retry wrapper [BETA] + +**Title**: Create generic retry/rate-limit wrapper for handlers +**Type**: Feature +**Priority**: Medium +**Labels**: `infrastructure`, `handlers` +**Milestone**: v0.2.0 Beta + +**Description**: +Create a decorator/wrapper that adds retry and rate limiting to any handler. + +**File**: `src/omnibase_infra/handlers/handler_wrapper.py` + +**Features**: +- Retry with configurable backoff (from `ModelRetryPolicy`) +- Token bucket rate limiting +- Circuit breaker pattern + +**Acceptance Criteria**: +- [ ] Reads config from `ModelHandlerBindingConfig` +- [ ] Wraps `handler.execute()` transparently +- [ ] Logs retry attempts +- [ ] Rate limit enforced per handler instance +- [ ] Circuit breaker trips after threshold +- [ ] Unit tests for retry logic +- [ ] mypy --strict passes + +--- + +### Issue 3.18: Add contract schema and linting [BETA] + +**Title**: Create JSON Schema for runtime host contracts +**Type**: Feature +**Priority**: Medium +**Labels**: `infrastructure`, `validation` +**Milestone**: v0.2.0 Beta + +**Description**: +Create validation tooling for runtime host contracts. + +**Files**: +- `src/omnibase_infra/contracts/schema/runtime_host_contract.schema.json` +- `src/omnibase_infra/cli/validate_contract.py` + +**Lint Rules**: +- No embedded secrets (require `secret_ref`) +- No `LOCAL` handler in production contracts +- Topic names follow naming schema + +**CLI**: `omnibase-runtime-validate-contract PATH` + +**Acceptance Criteria**: +- [ ] JSON Schema created +- [ ] CLI validates contracts +- [ ] Lint rules enforced +- [ ] CI integration +- [ ] Unit tests for validation +- [ ] mypy --strict passes + +--- + +## Phase 4: Integration & Testing - Beta Issues + +--- + +### Issue 4.5: Unit tests for KafkaEventBus [BETA] + +**Title**: Unit tests for KafkaEventBus +**Type**: Testing +**Priority**: High +**Labels**: `testing`, `event-bus` +**Milestone**: v0.2.0 Beta + +**Description**: +Unit tests for KafkaEventBus with mocked Kafka. + +**File**: `tests/unit/event_bus/test_kafka_event_bus.py` + +**Mock Requirements**: MockKafka MUST match aiokafka interface exactly. This prevents Beta teams from mocking the wrong subset of APIs. + +**Acceptance Criteria**: +- [ ] Test envelope publishing +- [ ] Test subscription and consumption +- [ ] Test backpressure behavior +- [ ] Test circuit breaker +- [ ] Test error handling +- [ ] Test health check +- [ ] >90% coverage +- [ ] MockKafka implements complete aiokafka interface +- [ ] No aiokafka methods missing from mock + +--- + +### Issue 4.6: Integration tests with Docker [BETA] + +**Title**: Integration tests with real services +**Type**: Testing +**Priority**: High +**Labels**: `testing`, `integration`, `docker` +**Milestone**: v0.2.0 Beta + +**Description**: +Integration tests using docker-compose with real PostgreSQL, Kafka, Vault. + +**File**: `tests/integration/test_runtime_host_integration.py` + +**Tests**: +- DbHandler with real PostgreSQL +- VaultHandler with real Vault +- KafkaEventBus with real Kafka +- Full envelope flow + +**Test Performance Requirements**: +- Integration tests must complete within 3 seconds per test case. +- Tests MUST NOT contain sleeps >50ms (use proper async waiting). +- Flaky tests are not acceptable - deterministic behavior required. + +**Acceptance Criteria**: +- [ ] docker-compose.test.yaml with all services +- [ ] pytest fixtures for service setup +- [ ] Tests pass in CI +- [ ] Cleanup after tests +- [ ] Each test completes in <3 seconds +- [ ] No sleeps >50ms in test code +- [ ] Tests pass consistently (0 flaky tests) + +--- + +### Issue 4.7: Graceful shutdown tests [BETA] + +**Title**: Test graceful shutdown behavior +**Type**: Testing +**Priority**: High +**Labels**: `testing`, `shutdown` +**Milestone**: v0.2.0 Beta + +**Description**: +Verify shutdown semantics under load. + +**File**: `tests/integration/test_graceful_shutdown.py` + +**Scenarios**: +- SIGTERM while 50 envelopes in progress +- Verify no envelopes dropped +- Verify handlers' `shutdown()` called exactly once +- Verify exit within grace period + +**Acceptance Criteria**: +- [ ] Simulate SIGTERM under load +- [ ] Assert no silent envelope drops +- [ ] Assert all handlers shutdown +- [ ] Assert exit within configured timeout + +--- + +### Issue 4.8: Backpressure and overload tests [BETA] + +**Title**: Test backpressure under overload +**Type**: Testing +**Priority**: Medium +**Labels**: `testing`, `performance` +**Milestone**: v0.2.0 Beta + +**Description**: +Verify backpressure behavior when Kafka produces faster than handlers consume. + +**File**: `tests/integration/test_backpressure.py` + +**Acceptance Criteria**: +- [ ] Simulate high-volume Kafka production +- [ ] Verify event bus pauses consumption +- [ ] Verify no unbounded memory growth +- [ ] Verify recovery when backlog clears + +--- + +### Issue 4.9: Topic naming validation tests [BETA] + +**Title**: Validate Kafka topic naming schema +**Type**: Testing +**Priority**: Medium +**Labels**: `testing`, `kafka` +**Milestone**: v0.2.0 Beta + +**Description**: +Verify all configured topics follow naming schema. + +**Schema**: `onex....v` +**Signals**: `cmd`, `evt`, `state`, `error`, `log` + +**Acceptance Criteria**: +- [ ] Parse topics from all runtime contracts +- [ ] Validate tenant segment exists +- [ ] Validate signal is in allowed set +- [ ] CI integration + +--- + +## Phase 5: Deployment & Migration - Beta Issues + +--- + +### Issue 5.4: Expand docker-compose with full services [BETA] + +**Title**: Add Kafka, Vault, Consul to docker-compose +**Type**: DevOps +**Priority**: Medium +**Labels**: `deployment`, `docker`, `development` +**Milestone**: v0.2.0 Beta + +**Description**: +Expand docker-compose.yaml with all dependent services. + +**Additional Services**: +- kafka (redpanda) +- vault (dev mode) +- consul (dev mode) + +**Acceptance Criteria**: +- [ ] All services defined +- [ ] Health checks configured +- [ ] Volumes for persistence +- [ ] Service dependencies correct + +--- + +### Issue 5.5: Document CLIs and environments [BETA] + +**Title**: Create CLIs and Environments documentation +**Type**: Documentation +**Priority**: Medium +**Labels**: `documentation` +**Milestone**: v0.2.0 Beta + +**Description**: +Document the separation between dev and prod CLIs. + +**File**: `docs/CLI_ENVIRONMENTS.md` + +**Content**: +- `omninode-runtime-host-dev` (core, dev only, LocalHandler) +- `omnibase-runtime-host` (infra, production, no LocalHandler) +- Environment variables +- When to use each + +**Acceptance Criteria**: +- [ ] Clear distinction documented +- [ ] Examples for each scenario +- [ ] Warning about prod CLI restrictions + +--- + +### Issue 5.6: Create version compatibility documentation [BETA] + +**Title**: Document version compatibility matrix +**Type**: Documentation +**Priority**: Low +**Labels**: `documentation`, `versioning` +**Milestone**: v0.2.0 Beta + +**Description**: +Document which versions of core/spi work with which infra versions. + +**File**: `docs/VERSION_COMPATIBILITY.md` + +**Acceptance Criteria**: +- [ ] Compatibility matrix table +- [ ] Minimum version requirements +- [ ] How to check versions at runtime +- [ ] Upgrade path documentation + +--- + +## Beta Execution Order + +``` +Phase 0 (CI Guardrails - Beta) + | + +-- 0.3 Protocol ownership verification [BETA] + +-- 0.4 Version compatibility check [BETA] + | + v +Phase 1 (Core Types - Beta) + | + +-- 1.12 ModelHandlerBindingConfig [BETA] + +-- 1.13 Extended error taxonomy [BETA] + | + v +Phase 3 (Infra - Beta) + | + +-- 3.9 VaultHandler [BETA] + +-- 3.10 ConsulHandler [BETA] + +-- 3.11 KafkaEventBus [BETA] + +-- 3.12 SecretResolver [BETA] + +-- 3.13 HandlerConfigResolver [BETA] + +-- 3.14 Extended error hierarchy [BETA] + +-- 3.15 Observability layer [BETA] + +-- 3.16 Health HTTP endpoint [BETA] + +-- 3.17 Handler retry wrapper [BETA] + +-- 3.18 Contract schema/linting [BETA] + | + v +Phase 4 (Testing - Beta) + | + +-- 4.5 KafkaEventBus unit tests [BETA] + +-- 4.6 Integration tests with Docker [BETA] + +-- 4.7 Graceful shutdown tests [BETA] + +-- 4.8 Backpressure tests [BETA] + +-- 4.9 Topic naming validation [BETA] + | + v +Phase 5 (Deployment - Beta) + | + +-- 5.4 Expand docker-compose [BETA] + +-- 5.5 CLI environments doc [BETA] + +-- 5.6 Version compatibility doc [BETA] + | + v +v0.2.0 Release +``` + +--- + +## Beta Success Metrics + +| Metric | Target | Measurement | +|--------|--------|-------------| +| Test coverage | >90% | pytest-cov | +| Integration tests pass | Yes | CI | +| Graceful shutdown drain | 100% | No dropped envelopes | +| Circuit breaker trigger | <10s | After threshold failures | +| Architecture violations | 0 | CI checks | + +--- + +## Considerations for Future Work + +The following items should be considered during Beta development but may not be critical blockers: + +### Consider: Adding integration tests for error classes +- Validate error classes work correctly in real handler scenarios +- Test error propagation through KafkaEventBus +- Verify error serialization/deserialization in envelope flows + +### Consider: Creating error recovery examples in documentation +- Document best practices for handling `HandlerExecutionError` +- Provide examples of retry configuration for different error types +- Show how to implement custom error handling in handlers + +--- + +> **Navigation**: [Back to Overview](../MVP_PLAN.md) | [Previous: MVP Core](./MVP_v0.1.0_CORE.md) | [Next: Production](./PRODUCTION_v0.3.0.md) + +**Last Updated**: 2025-12-03 diff --git a/docs/MVP_PROPOSED_WORK_ISSUES.md b/docs/milestones/MVP_v0.1.0_CORE.md similarity index 50% rename from docs/MVP_PROPOSED_WORK_ISSUES.md rename to docs/milestones/MVP_v0.1.0_CORE.md index 65e37e1fcf..7938062f2d 100644 --- a/docs/MVP_PROPOSED_WORK_ISSUES.md +++ b/docs/milestones/MVP_v0.1.0_CORE.md @@ -1,27 +1,15 @@ -# MVP Proposed Work Issues - omnibase_infra +# MVP Core (v0.1.0) - Milestone Details -> **Why This Document Exists**: This is the canonical infrastructure plan for shipping ONEX Runtime Host. If a feature, requirement, or constraint is not in this document, it is not happening in MVP. This document is the single source of truth for scope, architecture, and acceptance criteria. When in doubt, check this document. +> **Navigation**: [Back to Overview](../MVP_PLAN.md) | [Next: Beta Hardening](./BETA_v0.2.0_HARDENING.md) **Repository**: omnibase_infra -**Generated**: 2025-12-03 -**Updated**: 2025-12-03 (Milestone Split - Scope Reduction) -**Linear Project**: MVP - ONEX Runtime Host Infrastructure +**Target Version**: v0.1.0 +**Timeline**: Sprint 1-2 +**Issue Count**: 24 --- -## Milestone Overview - -This document organizes issues into three milestones with progressive scope: - -> **Core Principle**: Core is **pure orchestrator**; infra is **pure transport**. This boundary is inviolable. - -| Milestone | Version | Focus | Issue Count | Timeline | -|-----------|---------|-------|-------------|----------| -| **MVP Core** | v0.1.0 | Minimal working runtime with InMemoryEventBus | 24 | Sprint 1-2 | -| **Beta Hardening** | v0.2.0 | Production handlers, Kafka, observability | 22 | Sprint 3-4 | -| **Production** | v0.3.0 | Full deployment, chaos testing, advanced features | 8 | Sprint 5 | - -### MVP vs Beta Philosophy +## MVP Philosophy **MVP (v0.1.0)**: Prove the architecture works end-to-end with minimal scope - InMemoryEventBus only (no Kafka complexity) @@ -30,19 +18,6 @@ This document organizes issues into three milestones with progressive scope: - Basic error handling - Unit tests with mocks -**Beta (v0.2.0)**: Harden for production use -- KafkaEventBus with backpressure -- Vault + Consul handlers -- Full retry/rate-limit policies -- Integration tests with real services -- Observability layer - -**Production (v0.3.0)**: Deploy and validate at scale -- Kubernetes manifests -- Chaos testing -- Performance benchmarks -- Complete documentation - **Terminology Clarification**: - **"Minimal"** = Temporary scaffolding with known limitations. Will be expanded in Beta. - **"Simple"** = Intentionally small feature set that may remain unchanged. @@ -50,87 +25,29 @@ This document organizes issues into three milestones with progressive scope: --- -## Summary Statistics - -| Phase | MVP (v0.1.0) | Beta (v0.2.0) | Production (v0.3.0) | -|-------|--------------|---------------|---------------------| -| Phase 0: CI Guardrails | 2 | 2 | 0 | -| Phase 1: Core Types | 9 | 2 | 0 | -| Phase 2: SPI Updates | 3 | 0 | 0 | -| Phase 3: Infrastructure | 8 | 10 | 0 | -| Phase 4: Testing | 2 | 5 | 3 | -| Phase 5: Deployment | 2 | 3 | 2 | -| **Total** | **24** | **22** | **8** | - ---- - -## Map of Abstractions - -Understanding which component lives where is critical. This diagram shows the dependency flow: +## MVP Scope Boundaries (Do Not Implement) -``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ ONEX ARCHITECTURE β”‚ -β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ omnibase_infra (YOU ARE HERE) β”‚ β”‚ -β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ -β”‚ β”‚ β”‚ RuntimeHost β”‚ β”‚ Handlers β”‚ β”‚ EventBus β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ Process β”‚ β”‚ β€’ HttpHandler β”‚ β”‚ β€’ InMemoryBus β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ β”‚ β”‚ β€’ DbHandler β”‚ β”‚ β€’ KafkaBus (Beta)β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ Owns event β”‚ β”‚ β€’ VaultHandler β”‚ β”‚ β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ loop + wiring β”‚ β”‚ (Beta) β”‚ β”‚ Transport layer β”‚ β”‚ β”‚ -β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β”‚ β”‚ β”‚ -β”‚ β–Ό implements β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ omnibase_spi β”‚ β”‚ -β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ -β”‚ β”‚ β”‚ ProtocolHandler β”‚ β”‚ ProtocolEventBus β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ (abstract) β”‚ β”‚ (abstract) β”‚ β”‚ β”‚ -β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ -β”‚ β”‚ Defines interfaces only. Zero implementations. β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β”‚ β”‚ β”‚ -β”‚ β–Ό depends on β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ omnibase_core β”‚ β”‚ -β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ -β”‚ β”‚ β”‚ NodeRuntime β”‚ β”‚ NodeInstance β”‚ β”‚ ModelOnex β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ Envelope β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ Routes β”‚ β”‚ Wraps node β”‚ β”‚ β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ envelopes to β”‚ β”‚ contracts β”‚ β”‚ Unified message β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ handlers β”‚ β”‚ β”‚ β”‚ format β”‚ β”‚ β”‚ -β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ -β”‚ β”‚ Pure orchestration. ZERO I/O. ZERO transport imports. β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β”‚ β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - -DEPENDENCY RULE: infra β†’ spi β†’ core (never reverse) -``` +The following are explicitly OUT OF SCOPE for MVP (v0.1.0): -**Repository Ownership Summary**: +- **No Vault, Consul, Kafka** - InMemoryEventBus only +- **No retries** - Single attempt per handler invocation +- **No rate limits** - No throttling in MVP +- **No multi-tenant isolation** - tenant_id deferred +- **No secrets resolution** - Direct config only +- **No graceful shutdown drain** - Basic shutdown only -| Component | Repository | Responsibility | -|-----------|------------|----------------| -| `NodeRuntime` | `omnibase_core` | Routes envelopes to handlers. NO I/O. | -| `NodeInstance` | `omnibase_core` | Wraps node contracts. Delegates to runtime. | -| `ModelOnexEnvelope` | `omnibase_core` | Unified message format. JSON serializable. | -| `ProtocolHandler` | `omnibase_spi` | Abstract interface for handlers. | -| `ProtocolEventBus` | `omnibase_spi` | Abstract interface for event buses. | -| `BaseBaseRuntimeHostProcess` | `omnibase_infra` | Owns event loop. Wires handlers. Drives runtime. Base class for app-specific hosts. | -| `HttpHandler`, `DbHandler` | `omnibase_infra` | Concrete handler implementations. | -| `InMemoryEventBus` | `omnibase_infra` | Concrete event bus for MVP. | +**Explicitly Forbidden in MVP**: +- Multiple handlers of the same type (one HttpHandler, one DbHandler only) +- Multi-topic subscription (single input topic, single output topic) +- Async background tasks in ANY MVP code +- Dynamic handler registration at runtime +- Handler hot-reload or reconfiguration +- Transactions in DbHandler (individual queries only) -**The Golden Rule**: -``` -CORE defines models and orchestration. -SPI defines abstract protocols. -INFRA implements handlers + event bus + BaseBaseRuntimeHostProcess bootstrap. -``` +**Operations MUST Error Loudly**: +- Attempting to register duplicate handler types -> `ProtocolConfigurationError` +- Attempting forbidden operations (e.g., `db.transaction`) -> `InvalidOperationError` +- Missing required envelope fields -> `EnvelopeValidationError` --- @@ -169,237 +86,6 @@ runtime: - `handlers.type: "vault"` - VaultHandler deferred to Beta - `handlers.type: "consul"` - ConsulHandler deferred to Beta -**Cross-Version Compatibility**: -- v0.2 infra MUST accept v0.1 contracts but MUST warn on deprecated fields -- v0.1 contracts MUST NOT use any Beta-only fields -- Contract version mismatch MUST produce clear error message with upgrade instructions - -**Beta Contract Additions:** -```yaml -runtime: - name: "my_runtime_host" - version: "1.0.0" - - event_bus: - kind: "kafka" - config: - bootstrap_servers: "${KAFKA_BOOTSTRAP_SERVERS}" - consumer_group: "runtime-host-group" - topics: - input: "onex.tenant.domain.cmd.v1" - output: "onex.tenant.domain.evt.v1" - - handlers: - - type: "http" - config: - timeout_ms: 30000 - retry_policy: - max_retries: 3 - backoff_strategy: "exponential" - - type: "db" - config: - pool_size: 10 - - type: "vault" - config: - address: "${VAULT_ADDR}" - - type: "consul" - config: - address: "${CONSUL_ADDR}" - - nodes: - - slug: "node_a" - - slug: "node_b" -``` - -**Topic Naming Schema**: - -Topics follow the pattern: `onex....v` - -**Required Directions**: -- `.cmd` - Command/input topic (requests INTO the runtime) -- `.evt` - Event/output topic (responses FROM the runtime) - -**MVP Validation Rules** (enforced by contract importer): -- No invalid characters (alphanumeric, dots, hyphens only) -- Version segment must be numeric (e.g., `v1`, `v2`) -- Tenant and domain segments required - -**Examples**: -- `onex.acme.orders.cmd.v1` - Input commands -- `onex.acme.orders.evt.v1` - Output events - -**Reserved Namespaces** (MUST NOT be used by user contracts): -- `onex.internal.*` - System-internal communication -- `onex.debug.*` - Debug and diagnostics -- `onex.metrics.*` - Metrics and telemetry -- `onex.health.*` - Health check signals -- `onex.admin.*` - Administrative commands - -**Allowed Characters in Segments**: `[a-z0-9_-]` (lowercase alphanumeric, underscore, hyphen) - -**Topic Naming Examples**: - -| Example | Valid? | Reason | -|---------|--------|--------| -| `onex.acme.orders.cmd.v1` | Valid | Correct format | -| `onex.acme.orders.evt.v1` | Valid | Correct format | -| `onex.acme.user-service.cmd.v1` | Valid | Hyphen allowed | -| `onex.acme.user_service.cmd.v1` | Valid | Underscore allowed | -| `onex.ACME.orders.cmd.v1` | Invalid | Uppercase not allowed | -| `onex.acme.orders.command.v1` | Invalid | Must be `cmd` or `evt` | -| `onex.acme.orders.cmd.1` | Invalid | Version must have `v` prefix | -| `onex.acme.orders.cmd` | Invalid | Missing version | -| `acme.orders.cmd.v1` | Invalid | Missing `onex.` prefix | -| `onex..orders.cmd.v1` | Invalid | Empty segment | -| `onex.acme.orders.query.v1` | Invalid | `query` not a valid direction | -| `onex.internal.debug.evt.v1` | Invalid | `internal` is reserved | - -**Validation Regex**: -```python -TOPIC_PATTERN = re.compile( - r"^onex\.[a-z0-9][a-z0-9_-]*\.[a-z0-9][a-z0-9_-]*\.(cmd|evt)\.v[0-9]+$" -) - -def validate_topic(topic: str) -> bool: - if topic.startswith("onex.internal.") or topic.startswith("onex.debug."): - raise ValueError(f"Reserved namespace: {topic}") - return bool(TOPIC_PATTERN.match(topic)) -``` - ---- - -## Non-Goals (All Milestones) - -To prevent scope creep, the following are explicitly **NOT** in scope: - -- **No multi-region failover** - Single cluster only -- **No automatic topic creation** - Assumes infra bootstrap handled separately -- **No dynamic handler discovery** - Static wiring from contract only -- **No auto-migration of legacy nodes** - Handled by higher-level repos -- **No LLM handler** - Deferred to future milestone -- **No Consul-based service mesh** - Basic discovery only (Beta) - ---- - -## MVP Scope Boundaries (Do Not Implement) - -The following are explicitly OUT OF SCOPE for MVP (v0.1.0): - -- **No Vault, Consul, Kafka** - InMemoryEventBus only -- **No retries** - Single attempt per handler invocation -- **No rate limits** - No throttling in MVP -- **No multi-tenant isolation** - tenant_id deferred -- **No secrets resolution** - Direct config only -- **No graceful shutdown drain** - Basic shutdown only - -**Explicitly Forbidden in MVP**: -- Multiple handlers of the same type (one HttpHandler, one DbHandler only) -- Multi-topic subscription (single input topic, single output topic) -- Async background tasks in ANY MVP code -- Dynamic handler registration at runtime -- Handler hot-reload or reconfiguration -- Transactions in DbHandler (individual queries only) - -**Operations MUST Error Loudly**: -- Attempting to register duplicate handler types β†’ `HandlerConfigurationError` -- Attempting forbidden operations (e.g., `db.transaction`) β†’ `InvalidOperationError` -- Missing required envelope fields β†’ `EnvelopeValidationError` - ---- - -## Beta Scope Boundaries (Do Not Implement in Beta) - -The following are explicitly OUT OF SCOPE until Production (v0.3.0): - -- **No multi-region failover** - Single cluster only -- **No automatic topic creation** - Manual infra bootstrap -- **No dynamic handler discovery** - Static wiring only - ---- - -## Architectural Invariants Checklist - -Before starting any issue, review these invariants: - -- [ ] **Core is transport-agnostic**: `omnibase_core` has NO Kafka/HTTP/DB/Vault imports -- [ ] **NodeRuntime is pure in-memory**: No event loop, no bus consumer -- [ ] **Event bus is infra concern**: `BaseRuntimeHostProcess` + `ProtocolEventBus` drives runtime -- [ ] **Handlers use strong typing**: `handler_type` returns `EnumHandlerType`, not `str` -- [ ] **LocalHandler is dev/test only**: Never in production contracts -- [ ] **Single source of truth**: `wiring.py` for handler registration -- [ ] **No secrets in contracts**: All secrets via `secret_ref` or resolver -- [ ] **Single BaseRuntimeHostProcess per OS process**: No nested runtimes -- [ ] **Error boundary**: Infra MUST map all raw exceptions into core RuntimeHostError types before returning to NodeRuntime -- [ ] **No threads in handlers**: No handler may create threads or executors. All concurrency must be async + event bus -- [ ] **Payload opacity**: Infra never inspects payload interpretation; all semantics live in core -- [ ] **Correlation tracking**: If an incoming envelope has no correlation_id, the runtime MUST assign one -- [ ] **Correlation ID immutability**: Once assigned, correlation_id MUST NOT be modified by any component -- [ ] **No dynamic imports**: Handlers MUST NOT use `importlib` or dynamic module loading -- [ ] **No blocking I/O**: All I/O operations MUST be async. No `time.sleep()`, no synchronous HTTP/DB calls -- [ ] **No top-level awaits**: Module-level code MUST NOT await; all async work happens in lifecycle methods -- [ ] **No task creation in handlers**: Handlers MUST NOT call `asyncio.create_task()`. All concurrency is managed by BaseRuntimeHostProcess -- [ ] **Single-threaded MVP**: BaseRuntimeHostProcess MUST be single-threaded in MVP. No thread pools, no multiprocessing -- [ ] **Envelope immutability in core**: NodeRuntime MUST NOT mutate envelopes. Only handlers or event bus may enrich envelopes -- [ ] **Fail-fast contracts**: BaseRuntimeHostProcess MUST crash fast on malformed contracts. Fail early, fail loudly -- [ ] **No raw exceptions on bus**: BaseRuntimeHostProcess MUST NOT emit raw exceptions to the bus. All errors wrapped in response envelopes - ---- - -## Blocking I/O Rules - -**Blocking I/O Rules** (Binary Yes/No): - -| Operation | Allowed in Handlers? | Allowed in Core? | Notes | -|-----------|---------------------|------------------|-------| -| `await asyncio.sleep()` | YES | NO | Use for backoff, rate limiting | -| `time.sleep()` | NO | NO | Blocks event loop - FORBIDDEN | -| `asyncpg` queries | YES | NO | Infra handler only | -| `httpx` async requests | YES | NO | Infra handler only | -| `requests.get()` | NO | NO | Synchronous HTTP - FORBIDDEN | -| `open()` file read | LIMITED | NO | Only during initialization | -| `os.environ.get()` | LIMITED | YES | Only during initialization | -| `logging.info()` | YES | YES | Sync logging is acceptable | -| `print()` | NO | NO | Use structured logging | -| `subprocess.run()` | NO | NO | Blocks - FORBIDDEN | -| `json.dumps()` | YES | YES | CPU-bound, not I/O | -| DNS resolution | ASYNC ONLY | NO | Use `aiodns` or let `httpx` handle | - -**Initialization Exception**: -During `initialize()` lifecycle method, synchronous operations ARE allowed: -- Reading config files -- Environment variable access -- Initial connection pool creation (async preferred) -- One-time setup operations - -After `initialize()` completes, ALL I/O must be async. - -**Detection in CI**: -```bash -# CI check for forbidden synchronous patterns -grep -r "time.sleep\|requests\.\|subprocess\." src/omnibase_infra/handlers/ -# Should return empty (excluding test files) -``` - ---- - -## Correlation ID Ownership Rules - -**Correlation ID Ownership Rules**: - -| Scenario | Behavior | -|----------|----------| -| Request envelope has `correlation_id` | MUST preserve exactly as-is | -| Request envelope missing `correlation_id` | BaseRuntimeHostProcess MUST assign new UUID | -| Handler receives envelope | MUST NOT modify `correlation_id` | -| Event bus receives envelope | MUST NOT modify `correlation_id` | -| Response envelope creation | MUST echo `correlation_id` from request | -| Error envelope creation | MUST echo `correlation_id` from request | - -**Assignment Authority**: -- Only `BaseRuntimeHostProcess` may assign a new `correlation_id` (and only when missing) -- No other component (handler, event bus, NodeRuntime) may create or modify it -- All log entries MUST include `correlation_id` for traceability - --- ## MVP Constraints Checklist @@ -419,7 +105,7 @@ grep -r "time.sleep\|requests\.\|subprocess\." src/omnibase_infra/handlers/ | 9 | `InMemoryEventBus` | Publishes and consumes envelopes per topic | [ ] | | 10 | `BaseRuntimeHostProcess` | Loads contract, wires handlers, starts consumption | [ ] | | 11 | `wiring.py` | Registers handlers from config without duplicates | [ ] | -| 12 | E2E Flow | Envelope in β†’ handler executes β†’ response out | [ ] | +| 12 | E2E Flow | Envelope in -> handler executes -> response out | [ ] | **MVP Smoke Test** (single command to verify): ```bash @@ -435,129 +121,7 @@ pytest tests/integration/test_e2e_envelope_flow.py -v --- -## Failure Examples (What NOT to Do) - -Concrete examples of invariant violations. These patterns MUST fail CI or cause startup errors. - -### FAIL: Core importing transport library - -```python -# FILE: omnibase_core/runtime/node_runtime.py -# THIS MUST FAIL CI - -import asyncpg # VIOLATION: Core cannot import transport libraries - -class NodeRuntime: - async def route_envelope(self, envelope): - # This code should never exist in core - conn = await asyncpg.connect() # VIOLATION: I/O in core -``` - -**Why it fails**: `omnibase_core` must remain transport-agnostic. CI grep check blocks this. - ---- - -### FAIL: Handler creating background tasks - -```python -# FILE: omnibase_infra/handlers/http_handler.py -# THIS MUST FAIL CODE REVIEW - -class HttpHandler(ProtocolHandler): - async def execute(self, envelope): - # VIOLATION: Handler spawning background task - asyncio.create_task(self._background_work()) # FORBIDDEN - return response_envelope -``` - -**Why it fails**: Handlers MUST NOT create tasks. BaseRuntimeHostProcess owns all concurrency. - ---- - -### FAIL: Handler returning raw exception - -```python -# FILE: omnibase_infra/handlers/db_handler.py -# THIS MUST FAIL RUNTIME - -class DbHandler(ProtocolHandler): - async def execute(self, envelope): - try: - result = await self._pool.execute(query) - except asyncpg.PostgresError as e: - raise e # VIOLATION: Raw exception escaping handler - - # CORRECT: - # except asyncpg.PostgresError as e: - # raise HandlerExecutionError( - # message=str(e), - # handler_type=self.handler_type, - # correlation_id=envelope.correlation_id, - # ) from e -``` - -**Why it fails**: Raw exceptions leak internal details. All errors must be `RuntimeHostError` subclasses. - ---- - -### FAIL: NodeRuntime mutating envelope - -```python -# FILE: omnibase_core/runtime/node_runtime.py -# THIS MUST FAIL CODE REVIEW - -class NodeRuntime: - def route_envelope(self, envelope): - envelope.metadata["routed_at"] = datetime.now() # VIOLATION - envelope.correlation_id = uuid4() # VIOLATION - return self._dispatch(envelope) -``` - -**Why it fails**: NodeRuntime MUST NOT mutate envelopes. Only handlers or event bus may enrich. - ---- - -### FAIL: LocalHandler in production contract - -```yaml -# FILE: contracts/production_runtime.yaml -# THIS MUST FAIL STARTUP - -runtime: - name: "production_runtime" - handlers: - - type: "local" # VIOLATION: LocalHandler forbidden in infra - - type: "http" - - type: "db" -``` - -**Why it fails**: BaseRuntimeHostProcess MUST fail fast if LocalHandler detected in production. - ---- - -### FAIL: Blocking I/O in handler - -```python -# FILE: omnibase_infra/handlers/http_handler.py -# THIS MUST FAIL CODE REVIEW - -import requests # VIOLATION: Synchronous HTTP library - -class HttpHandler(ProtocolHandler): - async def execute(self, envelope): - # VIOLATION: Blocking call in async handler - response = requests.get(url) # Blocks event loop! - - # CORRECT: Use httpx (async) - # async with httpx.AsyncClient() as client: - # response = await client.get(url) -``` - -**Why it fails**: Blocking I/O freezes the entire BaseRuntimeHostProcess. All I/O must be async. - ---- - -## Phase 0: Cross-Repo Guardrails & Invariants +## Phase 0: Cross-Repo Guardrails & Invariants (MVP Issues) **Repository**: All (CI Infrastructure) @@ -575,9 +139,9 @@ class HttpHandler(ProtocolHandler): 2. Contract adapters and validators are finalized 3. Protocol surfaces in SPI are frozen -### MVP Issues (v0.1.0) +--- -#### Issue 0.1: Create core transport import check [MVP] +### Issue 0.1: Create core transport import check [MVP] **Title**: Automated check for zero transport imports in omnibase_core **Type**: Infrastructure @@ -604,7 +168,7 @@ Create an automated CI check that verifies `omnibase_core` has NO imports of tra --- -#### Issue 0.2: Enforce LocalHandler dev-only usage [MVP] +### Issue 0.2: Enforce LocalHandler dev-only usage [MVP] **Title**: Automated check that LocalHandler not used in infra **Type**: Infrastructure @@ -625,58 +189,15 @@ Ensure `LocalHandler` is ONLY used in `omnibase_core` tests, never imported or u --- -### Beta Issues (v0.2.0) - -#### Issue 0.3: Protocol ownership verification [BETA] - -**Title**: Verify infra never declares new protocols -**Type**: Infrastructure -**Priority**: High -**Labels**: `architecture`, `ci`, `guardrails` -**Milestone**: v0.2.0 Beta - -**Description**: -Ensure `omnibase_infra` only IMPLEMENTS protocols defined in `omnibase_spi`, never declares new abstract protocols. - -**Acceptance Criteria**: -- [ ] AST or grep check for `class.*Protocol.*ABC` in infra -- [ ] Only allow concrete implementations of SPI protocols -- [ ] Clear error message on violation -- [ ] Runs in CI - ---- - -#### Issue 0.4: Version compatibility matrix check [BETA] - -**Title**: Runtime version compatibility verification -**Type**: Infrastructure -**Priority**: High -**Labels**: `ci`, `versioning` -**Milestone**: v0.2.0 Beta - -**Description**: -On startup, verify that `omnibase_core` and `omnibase_spi` versions meet minimum requirements. - -**Compatibility Matrix**: -- Infra v0.1.0 -> Core >=0.4.0, SPI >=0.3.0 - -**Acceptance Criteria**: -- [ ] `BaseRuntimeHostProcess` logs resolved versions on startup -- [ ] Fails fast with clear message if incompatible -- [ ] Version matrix documented in README -- [ ] CI test for version check logic - ---- - -## Phase 1: Core Types (omnibase_core) +## Phase 1: Core Types (omnibase_core) - MVP Issues **Priority**: HIGH **Dependencies**: Phase 0 CI in place **Repository**: omnibase_core -### MVP Issues (v0.1.0) +--- -#### Issue 1.1: Add RUNTIME_HOST to EnumNodeKind [MVP] +### Issue 1.1: Add RUNTIME_HOST to EnumNodeKind [MVP] **Title**: Add RUNTIME_HOST value to EnumNodeKind enum **Type**: Feature @@ -697,7 +218,7 @@ Add `RUNTIME_HOST` value to `EnumNodeKind` enum to support runtime host contract --- -#### Issue 1.2: Create EnumHandlerType enum [MVP] +### Issue 1.2: Create EnumHandlerType enum [MVP] **Title**: Create EnumHandlerType enum for protocol handlers **Type**: Feature @@ -728,7 +249,7 @@ Create new `EnumHandlerType` enum for per-request handler types. --- -#### Issue 1.3: Create ModelOnexEnvelope model (SIMPLIFIED) [MVP] +### Issue 1.3: Create ModelOnexEnvelope model (SIMPLIFIED) [MVP] **Title**: Implement ModelOnexEnvelope unified message format **Type**: Feature @@ -762,7 +283,7 @@ Create the unified message envelope for all Runtime Host communication. **Simpli --- -#### Issue 1.4: Create ModelRuntimeHostContract model (SIMPLIFIED) [MVP] +### Issue 1.4: Create ModelRuntimeHostContract model (SIMPLIFIED) [MVP] **Title**: Implement ModelRuntimeHostContract Pydantic model **Type**: Feature @@ -796,7 +317,7 @@ Create the runtime host contract model. **Simplified for MVP - no retry policies --- -#### Issue 1.5: Verify ProtocolHandler export from SPI [MVP] +### Issue 1.5: Verify ProtocolHandler export from SPI [MVP] **Title**: Verify ProtocolHandler is available from omnibase_spi **Type**: Task @@ -846,7 +367,7 @@ def describe(self) -> dict: --- -#### Issue 1.6: Implement NodeInstance class [MVP] +### Issue 1.6: Implement NodeInstance class [MVP] **Title**: Create NodeInstance execution wrapper **Type**: Feature @@ -872,7 +393,7 @@ Create the lightweight node instance wrapper that delegates to NodeRuntime for h --- -#### Issue 1.7: Implement NodeRuntime class (MINIMAL) [MVP] +### Issue 1.7: Implement NodeRuntime class (MINIMAL) [MVP] **Title**: Create NodeRuntime transport-agnostic orchestrator **Type**: Feature @@ -906,7 +427,7 @@ Create the core runtime that hosts multiple node instances. **MINIMAL for MVP - --- -#### Issue 1.8: Implement FileRegistry class (SIMPLE) [MVP] +### Issue 1.8: Implement FileRegistry class (SIMPLE) [MVP] **Title**: Create FileRegistry for contract loading **Type**: Feature @@ -969,7 +490,7 @@ ContractValidationError: Failed to load contract --- -#### Issue 1.9: Implement LocalHandler (dev/test only) [MVP] +### Issue 1.9: Implement LocalHandler (dev/test only) [MVP] **Title**: Create LocalHandler echo handler for testing **Type**: Feature @@ -994,7 +515,7 @@ Create the local echo handler for testing. Must have clear warnings that it's NO --- -#### Issue 1.10: Create dev/test CLI entry point (SIMPLE) [MVP] +### Issue 1.10: Create dev/test CLI entry point (SIMPLE) [MVP] **Title**: Add runtime-host CLI command for dev/test **Type**: Feature @@ -1019,7 +540,7 @@ Add CLI entry point for testing Runtime Host with LocalHandler only. **Simple fo --- -#### Issue 1.11: Create minimal error taxonomy (core) [MVP] +### Issue 1.11: Create minimal error taxonomy (core) [MVP] **Title**: Define minimal core error hierarchy for runtime **Type**: Feature @@ -1092,7 +613,7 @@ When a handler or runtime produces an error, the response envelope MUST: - `HandlerNotFoundError` - `NodeNotFoundError` - `EnvelopeValidationError` -- `HandlerConfigurationError` +- `ProtocolConfigurationError` - `SecretResolutionError` **Acceptance Criteria**: @@ -1104,84 +625,15 @@ When a handler or runtime produces an error, the response envelope MUST: --- -### Beta Issues (v0.2.0) - -#### Issue 1.12: Create ModelHandlerBindingConfig model [BETA] - -**Title**: Implement formalized handler config schema -**Type**: Feature -**Priority**: High -**Labels**: `architecture`, `model`, `core` -**Milestone**: v0.2.0 Beta - -**Description**: -Create a formal schema for handler binding configuration with validation and defaults. - -**File**: `src/omnibase_core/models/runtime/model_handler_binding_config.py` (NEW) - -**Fields**: -- `handler_type: EnumHandlerType` -- `name: str` (optional, defaults to handler_type) -- `enabled: bool = True` -- `priority: int = 0` -- `config_ref: str | None` (reference to external config) -- `retry_policy: ModelRetryPolicy | None` -- `timeout_ms: int = 30000` -- `rate_limit_per_second: float | None` - -**Sub-model** `ModelRetryPolicy`: -- `max_retries: int = 3` -- `backoff_strategy: Literal["fixed", "exponential"] = "exponential"` -- `base_delay_ms: int = 100` -- `max_delay_ms: int = 5000` - -**Acceptance Criteria**: -- [ ] Full validation with Pydantic -- [ ] Sensible defaults for all optional fields -- [ ] Used by `ModelRuntimeHostContract.handlers` -- [ ] Unit tests for validation edge cases -- [ ] mypy --strict passes - ---- - -#### Issue 1.13: Extend error taxonomy (core) [BETA] - -**Title**: Complete core error hierarchy for runtime -**Type**: Feature -**Priority**: High -**Labels**: `architecture`, `errors`, `core` -**Milestone**: v0.2.0 Beta - -**Description**: -Complete the error hierarchy with additional error types. - -**File**: `src/omnibase_core/errors/runtime_errors.py` (UPDATE) - -**Additional Classes**: -- `HandlerNotFoundError(RuntimeHostError)` -- `NodeNotFoundError(RuntimeHostError)` -- `EnvelopeValidationError(RuntimeHostError)` -- `HandlerConfigurationError(RuntimeHostError)` -- `SecretResolutionError(RuntimeHostError)` - -**Acceptance Criteria**: -- [ ] All inherit from `RuntimeHostError` -- [ ] Structured fields: `handler_type`, `operation`, `correlation_id` -- [ ] NodeRuntime only sees abstract errors, not library exceptions -- [ ] Unit tests for each error class -- [ ] mypy --strict passes - ---- - -## Phase 2: SPI Protocol Updates (omnibase_spi) +## Phase 2: SPI Protocol Updates (omnibase_spi) - MVP Issues **Priority**: HIGH **Dependencies**: Phase 1 complete **Repository**: omnibase_spi -### MVP Issues (v0.1.0) +--- -#### Issue 2.1: Export ProtocolHandler from SPI [MVP] +### Issue 2.1: Export ProtocolHandler from SPI [MVP] **Title**: Export ProtocolHandler from omnibase_spi.protocols **Type**: Task @@ -1202,7 +654,7 @@ Export `ProtocolHandler` from SPI protocols module for infrastructure implementa --- -#### Issue 2.2: Update ProtocolEventBus with envelope methods [MVP] +### Issue 2.2: Update ProtocolEventBus with envelope methods [MVP] **Title**: Add envelope methods to ProtocolEventBus **Type**: Feature @@ -1229,7 +681,7 @@ Update `ProtocolEventBus` abstract class with `ModelOnexEnvelope` support. --- -#### Issue 2.3: Document handler vs event bus distinction [MVP] +### Issue 2.3: Document handler vs event bus distinction [MVP] **Title**: Add documentation for handler vs event bus separation **Type**: Documentation @@ -1250,7 +702,7 @@ Add clear documentation explaining when to use ProtocolHandler vs ProtocolEventB --- -## Phase 3: Infrastructure Handlers (omnibase_infra) +## Phase 3: Infrastructure Handlers (omnibase_infra) - MVP Issues **Priority**: HIGH **Dependencies**: Phase 1 and Phase 2 complete @@ -1297,31 +749,31 @@ Unknown operations -> `InvalidOperationError` at runtime. Every handler transitions through these states: ``` - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ HANDLER LIFECYCLE β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” initialize() β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ CREATED β”‚ ─────────────────► β”‚ INITIALIZED β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ β”‚ - β”‚ (constructor called) β”‚ health_check() returns healthy - β”‚ β–Ό - β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ β”‚ HEALTHY β”‚ ◄────┐ - β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ - β”‚ β”‚ β”‚ - β”‚ β”‚ execute() β”‚ (success) - β”‚ β–Ό β”‚ - β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ - β”‚ β”‚ EXECUTING β”‚ β”€β”€β”€β”€β”€β”˜ - β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ β”‚ - β”‚ β”‚ (error or shutdown signal) - β”‚ β–Ό - β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - └─────────────────────────►│ SHUTDOWN β”‚ - (any error during β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +---------------------------------------------+ + | HANDLER LIFECYCLE | + +---------------------------------------------+ + + +----------+ initialize() +-------------+ + | CREATED | -----------------> | INITIALIZED | + +----------+ +-------------+ + | | + | (constructor called) | health_check() returns healthy + | v + | +-------------+ + | | HEALTHY | <----+ + | +-------------+ | + | | | + | | execute() | + | v | + | +-------------+ | + | | EXECUTING | -----+ + | +-------------+ (success) + | | + | | (error or shutdown signal) + | v + | +-------------+ + +------------------------->| SHUTDOWN | + (any error during +-------------+ init or fatal) ``` @@ -1348,9 +800,7 @@ Beta will add explicit state tracking for graceful shutdown and health degradati --- -### MVP Issues (v0.1.0) - -#### Issue 3.1: Create handlers directory structure (SIMPLIFIED) [MVP] +### Issue 3.1: Create handlers directory structure (SIMPLIFIED) [MVP] **Title**: Create handlers/ directory with __init__.py **Type**: Task @@ -1395,7 +845,7 @@ src/omnibase_infra/ --- -#### Issue 3.2: Implement HttpHandler (MINIMAL) [MVP] +### Issue 3.2: Implement HttpHandler (MINIMAL) [MVP] **Title**: Create HTTP REST protocol handler **Type**: Feature @@ -1435,7 +885,7 @@ Implement `HttpHandler` for HTTP REST operations using httpx. **Minimal for MVP --- -#### Issue 3.3: Implement DbHandler (MINIMAL) [MVP] +### Issue 3.3: Implement DbHandler (MINIMAL) [MVP] **Title**: Create PostgreSQL database protocol handler **Type**: Feature @@ -1480,7 +930,7 @@ Implement `DbHandler` for PostgreSQL operations using asyncpg. **Minimal for MVP --- -#### Issue 3.4: Implement InMemoryEventBus [MVP] +### Issue 3.4: Implement InMemoryEventBus [MVP] **Title**: Create in-memory event bus for local development **Type**: Feature @@ -1507,7 +957,7 @@ Implement `InMemoryEventBus` for local testing and CI without Kafka. **This is t --- -#### Issue 3.5: Create wiring.py (SIMPLE) [MVP] +### Issue 3.5: Create wiring.py (SIMPLE) [MVP] **Title**: Create wiring.py handler registration module **Type**: Feature @@ -1543,7 +993,7 @@ Create the single source of truth for handler registration. **Simple for MVP - n --- -#### Issue 3.6: Implement BaseRuntimeHostProcess (MINIMAL) [MVP] +### Issue 3.6: Implement BaseRuntimeHostProcess (MINIMAL) [MVP] **Title**: Create BaseRuntimeHostProcess infrastructure wrapper **Type**: Feature @@ -1576,17 +1026,6 @@ Create the infrastructure-level process wrapper that owns the event bus and driv - `SHUTDOWN_GRACE_SECONDS` configurable - Single instance enforcement per process -**Acceptance Criteria**: -- [ ] Uses `wiring.py` for handler registration -- [ ] Owns event bus instance (InMemory for MVP) -- [ ] Subscribes event bus to call `runtime.route_envelope()` -- [ ] Basic shutdown -- [ ] Unit tests with mock event bus -- [ ] mypy --strict passes -- [ ] NodeRuntime does not start any background tasks or event loops -- [ ] Errors produce `success=False` response envelopes -- [ ] Sequential envelope processing (no parallelism in MVP) - **Missing Handler Behavior**: When an envelope requests a handler type that is not registered: @@ -1615,9 +1054,20 @@ When an envelope requests a handler type that is not registered: - LocalHandler fallback is FORBIDDEN in infra (dev-only in core tests). - Future: Beta may add circuit breaker for degraded handlers. +**Acceptance Criteria**: +- [ ] Uses `wiring.py` for handler registration +- [ ] Owns event bus instance (InMemory for MVP) +- [ ] Subscribes event bus to call `runtime.route_envelope()` +- [ ] Basic shutdown +- [ ] Unit tests with mock event bus +- [ ] mypy --strict passes +- [ ] NodeRuntime does not start any background tasks or event loops +- [ ] Errors produce `success=False` response envelopes +- [ ] Sequential envelope processing (no parallelism in MVP) + --- -#### Issue 3.7: Create example runtime host contract (SIMPLE) [MVP] +### Issue 3.7: Create example runtime host contract (SIMPLE) [MVP] **Title**: Create infra_runtime_host.yaml example contract **Type**: Documentation @@ -1639,7 +1089,7 @@ Create an example runtime host contract showing minimal MVP configuration. --- -#### Issue 3.8: Add production CLI entry point (SIMPLE) [MVP] +### Issue 3.8: Add production CLI entry point (SIMPLE) [MVP] **Title**: Add omnibase-runtime-host CLI command **Type**: Feature @@ -1666,339 +1116,33 @@ Add production CLI entry point for BaseRuntimeHostProcess. **Simple for MVP - ju --- -### Beta Issues (v0.2.0) +## Phase 4: Integration & Testing - MVP Issues -#### Issue 3.9: Implement VaultHandler [BETA] +**Priority**: HIGH/MEDIUM +**Dependencies**: Phase 3 complete +**Repository**: omnibase_infra -**Title**: Create Vault secrets management protocol handler -**Type**: Feature -**Priority**: High -**Labels**: `handler`, `infrastructure`, `secrets` -**Milestone**: v0.2.0 Beta +**Testing Infrastructure Requirements**: -**Description**: -Implement `VaultHandler` for HashiCorp Vault operations using hvac. +**Deterministic Test Helpers** (MVP): +```python +# tests/helpers/deterministic.py +class DeterministicIdGenerator: + """Generates predictable UUIDs for testing.""" + def __init__(self, seed: int = 42): + self._counter = seed -**File**: `src/omnibase_infra/handlers/vault_handler.py` + def next_uuid(self) -> UUID: + self._counter += 1 + return UUID(int=self._counter) -**Operations**: `get_secret`, `set_secret`, `delete_secret`, `list_secrets` +class DeterministicClock: + """Provides controllable timestamps for testing.""" + def __init__(self, start: datetime = datetime(2024, 1, 1)): + self._now = start -**Acceptance Criteria**: -- [ ] Returns `EnumHandlerType.VAULT` (not str) -- [ ] Uses hvac client -- [ ] KV v2 support -- [ ] Maps `hvac.VaultError` -> `HandlerExecutionError` -- [ ] Unit tests with mock client -- [ ] Integration test with dev Vault -- [ ] mypy --strict passes - ---- - -#### Issue 3.10: Implement ConsulHandler [BETA] - -**Title**: Create Consul service discovery protocol handler -**Type**: Feature -**Priority**: Medium -**Labels**: `handler`, `infrastructure`, `discovery` -**Milestone**: v0.2.0 Beta - -**Description**: -Implement `ConsulHandler` for Consul service discovery operations. - -**File**: `src/omnibase_infra/handlers/consul_handler.py` - -**Operations**: `register_service`, `deregister_service`, `get_service`, `list_services`, `health_check_service` - -**Acceptance Criteria**: -- [ ] Returns `EnumHandlerType.CONSUL` (not str) -- [ ] Uses python-consul or httpx -- [ ] Service registration/deregistration -- [ ] Health check integration -- [ ] Unit tests with mock responses -- [ ] Integration test with dev Consul -- [ ] mypy --strict passes - ---- - -#### Issue 3.11: Implement KafkaEventBus [BETA] - -**Title**: Create Kafka event bus implementation -**Type**: Feature -**Priority**: High -**Labels**: `event-bus`, `infrastructure`, `messaging` -**Milestone**: v0.2.0 Beta - -**Description**: -Implement `KafkaEventBus` that implements `ProtocolEventBus` (NOT ProtocolHandler). - -**File**: `src/omnibase_infra/event_bus/kafka_event_bus.py` - -**Methods**: `initialize`, `shutdown`, `publish_envelope`, `subscribe`, `start_consuming`, `health_check` - -**Backpressure Config**: -- `max_inflight_envelopes: int = 100` -- `pause_consumption_threshold: int = 80` -- `circuit_breaker_threshold: int = 5` (consecutive failures) - -**Ordering Guarantees**: Runtime MUST NOT assume global ordering. Kafka provides partition-level ordering only. - -**Response Pattern**: Event bus handlers must return envelopes; BaseRuntimeHostProcess is responsible for publishing responses. - -**Acceptance Criteria**: -- [ ] Implements `ProtocolEventBus` (NOT ProtocolHandler) -- [ ] NO `handler_type` property (event bus, not handler) -- [ ] Uses aiokafka -- [ ] Proper envelope serialization/deserialization -- [ ] Configurable topics and consumer group -- [ ] Backpressure: pauses consumption when queue full -- [ ] Circuit breaker: stops after N consecutive handler failures -- [ ] Unit tests with mock Kafka -- [ ] Integration test with test Kafka -- [ ] mypy --strict passes - ---- - -#### Issue 3.12: Create SecretResolver [BETA] - -**Title**: Implement centralized secret resolution -**Type**: Feature -**Priority**: High -**Labels**: `infrastructure`, `secrets` -**Milestone**: v0.2.0 Beta - -**Description**: -Create a centralized secret resolver so handlers never call `os.getenv` directly. - -**File**: `src/omnibase_infra/runtime/secret_resolver.py` - -**Interface**: -```python -class SecretResolver: - async def get_secret(self, logical_name: str) -> str: ... - async def get_secrets(self, logical_names: list[str]) -> dict[str, str]: ... -``` - -**Sources** (priority order): -1. Vault (if configured) -2. Environment variables -3. File-based secrets (K8s secrets volume) - -**Acceptance Criteria**: -- [ ] Typed interface for secret requests -- [ ] Handlers never call `os.getenv` directly -- [ ] Vault integration optional -- [ ] Unit tests with mocked sources -- [ ] mypy --strict passes - ---- - -#### Issue 3.13: Create HandlerConfigResolver [BETA] - -**Title**: Implement handler config resolution layer -**Type**: Feature -**Priority**: High -**Labels**: `infrastructure`, `configuration` -**Milestone**: v0.2.0 Beta - -**Description**: -Create a resolver that normalizes handler configs from multiple sources. - -**File**: `src/omnibase_infra/runtime/handler_config_resolver.py` - -**Sources**: -- Contract YAML -- Environment variables (override) -- Vault (secrets only via SecretResolver) -- Files (config refs) - -**Acceptance Criteria**: -- [ ] Resolves `config_ref` to actual config dict -- [ ] Merges environment overrides -- [ ] Returns fully validated `ModelHandlerBindingConfig` -- [ ] Clear error messages for missing required fields -- [ ] Unit tests for resolution logic -- [ ] mypy --strict passes - ---- - -#### Issue 3.14: Extend infra error hierarchy [BETA] - -**Title**: Complete infrastructure error taxonomy -**Type**: Feature -**Priority**: High -**Labels**: `infrastructure`, `errors` -**Milestone**: v0.2.0 Beta - -**Description**: -Complete structured error hierarchy for all infra components. - -**File**: `src/omnibase_infra/errors/infra_errors.py` (UPDATE) - -**Additional Classes**: -- `HandlerConfigurationError(RuntimeHostError)` -- `SecretResolutionError(RuntimeHostError)` - -**Acceptance Criteria**: -- [ ] All inherit from core `RuntimeHostError` -- [ ] Handlers map raw exceptions to these types -- [ ] Structured fields: `handler_type`, `operation`, `correlation_id` -- [ ] Unit tests for each error class -- [ ] mypy --strict passes - ---- - -#### Issue 3.15: Implement observability layer [BETA] - -**Title**: Create structured logging and metrics infrastructure -**Type**: Feature -**Priority**: High -**Labels**: `infrastructure`, `observability` -**Milestone**: v0.2.0 Beta - -**Description**: -Create centralized observability configuration owned by BaseRuntimeHostProcess. - -**Files**: -- `src/omnibase_infra/observability/logging_config.py` -- `src/omnibase_infra/observability/metrics.py` - -**Logging Requirements**: -- JSON structured logs -- Standard fields: `runtime_id`, `node_id`, `handler_type`, `envelope_id`, `correlation_id` -- Handlers use shared logger (no ad-hoc loggers) - -**Metrics Hooks**: -- Envelope count in/out per handler -- Handler latency histogram -- Event bus lag metrics (consumer group offset) - -**Acceptance Criteria**: -- [ ] `BaseRuntimeHostProcess` sets up global logging config -- [ ] Handlers get loggers from centralized factory -- [ ] All logs include correlation_id when available -- [ ] Metrics exposed for collection -- [ ] Unit tests for log formatting -- [ ] mypy --strict passes - ---- - -#### Issue 3.16: Implement health HTTP endpoint [BETA] - -**Title**: Create HTTP health endpoint server -**Type**: Feature -**Priority**: Medium -**Labels**: `infrastructure`, `health` -**Milestone**: v0.2.0 Beta - -**Description**: -Optional HTTP server exposing health endpoints for K8s probes. - -**File**: `src/omnibase_infra/runtime/health_server.py` - -**Endpoints**: -- `/health/live` - Is process alive -- `/health/ready` - Can process envelopes -- `/health/handlers` - Per-handler status snapshot - -**Acceptance Criteria**: -- [ ] Optional (can be disabled in config) -- [ ] Configurable port -- [ ] Minimal dependencies (use `aiohttp` or built-in) -- [ ] Integrates with Docker healthchecks -- [ ] Integrates with K8s probes -- [ ] Unit tests -- [ ] mypy --strict passes - ---- - -#### Issue 3.17: Implement handler retry wrapper [BETA] - -**Title**: Create generic retry/rate-limit wrapper for handlers -**Type**: Feature -**Priority**: Medium -**Labels**: `infrastructure`, `handlers` -**Milestone**: v0.2.0 Beta - -**Description**: -Create a decorator/wrapper that adds retry and rate limiting to any handler. - -**File**: `src/omnibase_infra/handlers/handler_wrapper.py` - -**Features**: -- Retry with configurable backoff (from `ModelRetryPolicy`) -- Token bucket rate limiting -- Circuit breaker pattern - -**Acceptance Criteria**: -- [ ] Reads config from `ModelHandlerBindingConfig` -- [ ] Wraps `handler.execute()` transparently -- [ ] Logs retry attempts -- [ ] Rate limit enforced per handler instance -- [ ] Circuit breaker trips after threshold -- [ ] Unit tests for retry logic -- [ ] mypy --strict passes - ---- - -#### Issue 3.18: Add contract schema and linting [BETA] - -**Title**: Create JSON Schema for runtime host contracts -**Type**: Feature -**Priority**: Medium -**Labels**: `infrastructure`, `validation` -**Milestone**: v0.2.0 Beta - -**Description**: -Create validation tooling for runtime host contracts. - -**Files**: -- `src/omnibase_infra/contracts/schema/runtime_host_contract.schema.json` -- `src/omnibase_infra/cli/validate_contract.py` - -**Lint Rules**: -- No embedded secrets (require `secret_ref`) -- No `LOCAL` handler in production contracts -- Topic names follow naming schema - -**CLI**: `omnibase-runtime-validate-contract PATH` - -**Acceptance Criteria**: -- [ ] JSON Schema created -- [ ] CLI validates contracts -- [ ] Lint rules enforced -- [ ] CI integration -- [ ] Unit tests for validation -- [ ] mypy --strict passes - ---- - -## Phase 4: Integration & Testing - -**Priority**: HIGH/MEDIUM -**Dependencies**: Phase 3 complete -**Repository**: omnibase_infra - -**Testing Infrastructure Requirements**: - -**Deterministic Test Helpers** (MVP): -```python -# tests/helpers/deterministic.py -class DeterministicIdGenerator: - """Generates predictable UUIDs for testing.""" - def __init__(self, seed: int = 42): - self._counter = seed - - def next_uuid(self) -> UUID: - self._counter += 1 - return UUID(int=self._counter) - -class DeterministicClock: - """Provides controllable timestamps for testing.""" - def __init__(self, start: datetime = datetime(2024, 1, 1)): - self._now = start - - def now(self) -> datetime: - return self._now + def now(self) -> datetime: + return self._now def advance(self, seconds: int) -> None: self._now += timedelta(seconds=seconds) @@ -2011,10 +1155,9 @@ All test suites MUST assert: - No resource leaks (connections, file handles) - Deterministic ordering of results - --- -## Testing Without Docker +### Testing Without Docker Many contributors prefer local testing without Docker. Here's the pure-Python strategy: @@ -2085,9 +1228,7 @@ def mock_db_handler(): --- -### MVP Issues (v0.1.0) - -#### Issue 4.1: Unit tests for handlers (mocked) [MVP] +### Issue 4.1: Unit tests for handlers (mocked) [MVP] **Title**: Unit test coverage for MVP handlers **Type**: Testing @@ -2115,7 +1256,7 @@ Unit tests for HttpHandler and DbHandler with mocked dependencies. --- -#### Issue 4.2: Unit tests for InMemoryEventBus [MVP] +### Issue 4.2: Unit tests for InMemoryEventBus [MVP] **Title**: Unit tests for InMemoryEventBus **Type**: Testing @@ -2137,7 +1278,7 @@ Unit tests for InMemoryEventBus implementation. --- -#### Issue 4.3: Single E2E flow test with InMemoryEventBus [MVP] +### Issue 4.3: Single E2E flow test with InMemoryEventBus [MVP] **Title**: E2E test: InMemoryEventBus -> Runtime -> Handler **Type**: Testing @@ -2172,7 +1313,7 @@ Single end-to-end test verifying the complete flow using InMemoryEventBus. --- -#### Issue 4.4: Architecture compliance verification (SIMPLIFIED) [MVP] +### Issue 4.4: Architecture compliance verification (SIMPLIFIED) [MVP] **Title**: Verify architectural invariants **Type**: Task @@ -2200,230 +1341,15 @@ Create simplified verification that architectural invariants are maintained. **J --- -### Beta Issues (v0.2.0) - -#### Issue 4.5: Unit tests for KafkaEventBus [BETA] - -**Title**: Unit tests for KafkaEventBus -**Type**: Testing -**Priority**: High -**Labels**: `testing`, `event-bus` -**Milestone**: v0.2.0 Beta - -**Description**: -Unit tests for KafkaEventBus with mocked Kafka. - -**File**: `tests/unit/event_bus/test_kafka_event_bus.py` - -**Mock Requirements**: MockKafka MUST match aiokafka interface exactly. This prevents Beta teams from mocking the wrong subset of APIs. - -**Acceptance Criteria**: -- [ ] Test envelope publishing -- [ ] Test subscription and consumption -- [ ] Test backpressure behavior -- [ ] Test circuit breaker -- [ ] Test error handling -- [ ] Test health check -- [ ] >90% coverage -- [ ] MockKafka implements complete aiokafka interface -- [ ] No aiokafka methods missing from mock - ---- - -#### Issue 4.6: Integration tests with Docker [BETA] - -**Title**: Integration tests with real services -**Type**: Testing -**Priority**: High -**Labels**: `testing`, `integration`, `docker` -**Milestone**: v0.2.0 Beta - -**Description**: -Integration tests using docker-compose with real PostgreSQL, Kafka, Vault. - -**File**: `tests/integration/test_runtime_host_integration.py` - -**Tests**: -- DbHandler with real PostgreSQL -- VaultHandler with real Vault -- KafkaEventBus with real Kafka -- Full envelope flow - -**Test Performance Requirements**: -- Integration tests must complete within 3 seconds per test case. -- Tests MUST NOT contain sleeps >50ms (use proper async waiting). -- Flaky tests are not acceptable - deterministic behavior required. - -**Acceptance Criteria**: -- [ ] docker-compose.test.yaml with all services -- [ ] pytest fixtures for service setup -- [ ] Tests pass in CI -- [ ] Cleanup after tests -- [ ] Each test completes in <3 seconds -- [ ] No sleeps >50ms in test code -- [ ] Tests pass consistently (0 flaky tests) - ---- - -#### Issue 4.7: Graceful shutdown tests [BETA] - -**Title**: Test graceful shutdown behavior -**Type**: Testing -**Priority**: High -**Labels**: `testing`, `shutdown` -**Milestone**: v0.2.0 Beta - -**Description**: -Verify shutdown semantics under load. - -**File**: `tests/integration/test_graceful_shutdown.py` - -**Scenarios**: -- SIGTERM while 50 envelopes in progress -- Verify no envelopes dropped -- Verify handlers' `shutdown()` called exactly once -- Verify exit within grace period - -**Acceptance Criteria**: -- [ ] Simulate SIGTERM under load -- [ ] Assert no silent envelope drops -- [ ] Assert all handlers shutdown -- [ ] Assert exit within configured timeout - ---- - -#### Issue 4.8: Backpressure and overload tests [BETA] - -**Title**: Test backpressure under overload -**Type**: Testing -**Priority**: Medium -**Labels**: `testing`, `performance` -**Milestone**: v0.2.0 Beta - -**Description**: -Verify backpressure behavior when Kafka produces faster than handlers consume. - -**File**: `tests/integration/test_backpressure.py` - -**Acceptance Criteria**: -- [ ] Simulate high-volume Kafka production -- [ ] Verify event bus pauses consumption -- [ ] Verify no unbounded memory growth -- [ ] Verify recovery when backlog clears - ---- - -#### Issue 4.9: Topic naming validation tests [BETA] - -**Title**: Validate Kafka topic naming schema -**Type**: Testing -**Priority**: Medium -**Labels**: `testing`, `kafka` -**Milestone**: v0.2.0 Beta - -**Description**: -Verify all configured topics follow naming schema. - -**Schema**: `onex....v` -**Signals**: `cmd`, `evt`, `state`, `error`, `log` - -**Acceptance Criteria**: -- [ ] Parse topics from all runtime contracts -- [ ] Validate tenant segment exists -- [ ] Validate signal is in allowed set -- [ ] CI integration - ---- - -### Production Issues (v0.3.0) - -#### Issue 4.10: Chaos and failure-mode tests [PROD] - -**Title**: Create chaos test suite -**Type**: Testing -**Priority**: Medium -**Labels**: `testing`, `chaos` -**Milestone**: v0.3.0 Production - -**Description**: -Test failure modes and recovery behavior. - -**Directory**: `tests/chaos/` - -**Scenarios**: -- Kafka connection loss (drop broker) -- Postgres unavailable for 5 seconds -- Vault network flakiness -- Handler returning errors continuously - -**Acceptance Criteria**: -- [ ] BaseRuntimeHostProcess does not crash -- [ ] Retries within configured budgets -- [ ] Health endpoints reflect degraded state -- [ ] Circuit breaker trips appropriately -- [ ] Recovery after service restoration - ---- - -#### Issue 4.11: Performance benchmarks [PROD] - -**Title**: Establish performance baselines -**Type**: Testing -**Priority**: Medium -**Labels**: `testing`, `performance` -**Milestone**: v0.3.0 Production - -**Description**: -Create benchmark suite to verify performance targets. - -**Targets**: -- Memory per 10 nodes: <200MB -- Envelope throughput: >100/sec -- Handler latency (local): <1ms p99 -- Handler latency (http): <100ms p99 -- Handler latency (db): <50ms p99 - -**Acceptance Criteria**: -- [ ] Benchmark script created -- [ ] Results logged and tracked -- [ ] CI integration for regression detection -- [ ] Documentation of baselines - ---- - -#### Issue 4.12: Complete architecture compliance [PROD] - -**Title**: Full architectural invariant verification -**Type**: Task -**Priority**: High -**Labels**: `architecture`, `validation` -**Milestone**: v0.3.0 Production - -**Description**: -Complete verification that all architectural invariants are maintained. - -**Additional Checks**: -- [ ] All handlers return `EnumHandlerType` -- [ ] `wiring.py` is only handler registration location -- [ ] No `os.getenv` in handlers (uses SecretResolver) -- [ ] Single BaseRuntimeHostProcess per process enforcement - -**Acceptance Criteria**: -- [ ] Shell script or pytest for verification -- [ ] Runs in CI -- [ ] Clear error messages on failure - ---- - -## Phase 5: Deployment & Migration +## Phase 5: Deployment & Migration - MVP Issues **Priority**: MEDIUM **Dependencies**: Phase 4 complete **Repository**: omnibase_infra -### MVP Issues (v0.1.0) +--- -#### Issue 5.1: Create Dockerfile for runtime host (BASIC) [MVP] +### Issue 5.1: Create Dockerfile for runtime host (BASIC) [MVP] **Title**: Create basic Dockerfile **Type**: DevOps @@ -2472,7 +1398,7 @@ ENTRYPOINT ["omnibase-runtime-host"] --- -#### Issue 5.2: Create docker-compose for local development [MVP] +### Issue 5.2: Create docker-compose for local development [MVP] **Title**: docker-compose.yaml for local development **Type**: DevOps @@ -2502,7 +1428,7 @@ Create docker-compose.yaml with minimal services for local development. --- -#### Issue 5.3: Update CLAUDE.md with new architecture (MINIMAL) [MVP] +### Issue 5.3: Update CLAUDE.md with new architecture (MINIMAL) [MVP] **Title**: Update CLAUDE.md for Runtime Host **Type**: Documentation @@ -2531,132 +1457,7 @@ Update CLAUDE.md with basic Runtime Host architecture patterns. --- -### Beta Issues (v0.2.0) - -#### Issue 5.4: Expand docker-compose with full services [BETA] - -**Title**: Add Kafka, Vault, Consul to docker-compose -**Type**: DevOps -**Priority**: Medium -**Labels**: `deployment`, `docker`, `development` -**Milestone**: v0.2.0 Beta - -**Description**: -Expand docker-compose.yaml with all dependent services. - -**Additional Services**: -- kafka (redpanda) -- vault (dev mode) -- consul (dev mode) - -**Acceptance Criteria**: -- [ ] All services defined -- [ ] Health checks configured -- [ ] Volumes for persistence -- [ ] Service dependencies correct - ---- - -#### Issue 5.5: Document CLIs and environments [BETA] - -**Title**: Create CLIs and Environments documentation -**Type**: Documentation -**Priority**: Medium -**Labels**: `documentation` -**Milestone**: v0.2.0 Beta - -**Description**: -Document the separation between dev and prod CLIs. - -**File**: `docs/CLI_ENVIRONMENTS.md` - -**Content**: -- `omninode-runtime-host-dev` (core, dev only, LocalHandler) -- `omnibase-runtime-host` (infra, production, no LocalHandler) -- Environment variables -- When to use each - -**Acceptance Criteria**: -- [ ] Clear distinction documented -- [ ] Examples for each scenario -- [ ] Warning about prod CLI restrictions - ---- - -#### Issue 5.6: Create version compatibility documentation [BETA] - -**Title**: Document version compatibility matrix -**Type**: Documentation -**Priority**: Low -**Labels**: `documentation`, `versioning` -**Milestone**: v0.2.0 Beta - -**Description**: -Document which versions of core/spi work with which infra versions. - -**File**: `docs/VERSION_COMPATIBILITY.md` - -**Acceptance Criteria**: -- [ ] Compatibility matrix table -- [ ] Minimum version requirements -- [ ] How to check versions at runtime -- [ ] Upgrade path documentation - ---- - -### Production Issues (v0.3.0) - -#### Issue 5.7: Kubernetes manifests [PROD] - -**Title**: Create Kubernetes deployment manifests -**Type**: DevOps -**Priority**: Medium -**Labels**: `deployment`, `kubernetes` -**Milestone**: v0.3.0 Production - -**Description**: -Create Kubernetes manifests for production deployment. - -**Files**: -- `k8s/deployment.yaml` -- `k8s/service.yaml` -- `k8s/configmap.yaml` -- `k8s/secrets.yaml` (template) - -**Acceptance Criteria**: -- [ ] Deployment with resource limits -- [ ] Service for internal access -- [ ] ConfigMap for contract -- [ ] Secret references documented -- [ ] Liveness probe: `/health/live` -- [ ] Readiness probe: `/health/ready` - ---- - -#### Issue 5.8: Migration guide documentation [PROD] - -**Title**: Create migration guide from legacy architecture -**Type**: Documentation -**Priority**: Medium -**Labels**: `documentation`, `migration` -**Milestone**: v0.3.0 Production - -**Description**: -Document the migration path from 1-container-per-node to Runtime Host model. - -**File**: `docs/MIGRATION_GUIDE.md` - -**Acceptance Criteria**: -- [ ] Before/after architecture comparison -- [ ] Step-by-step migration process -- [ ] Rollback procedures -- [ ] Common issues and solutions - ---- - -## Execution Order - -### MVP (v0.1.0) - Sprint 1-2 +## MVP Execution Order ``` Phase 0 (CI Guardrails - MVP) @@ -2717,78 +1518,9 @@ Phase 5 (Deployment - MVP) v0.1.0 Release ``` -### Beta (v0.2.0) - Sprint 3-4 - -``` -Phase 0 (CI Guardrails - Beta) - | - +-- 0.3 Protocol ownership verification [BETA] - +-- 0.4 Version compatibility check [BETA] - | - v -Phase 1 (Core Types - Beta) - | - +-- 1.12 ModelHandlerBindingConfig [BETA] - +-- 1.13 Extended error taxonomy [BETA] - | - v -Phase 3 (Infra - Beta) - | - +-- 3.9 VaultHandler [BETA] - +-- 3.10 ConsulHandler [BETA] - +-- 3.11 KafkaEventBus [BETA] - +-- 3.12 SecretResolver [BETA] - +-- 3.13 HandlerConfigResolver [BETA] - +-- 3.14 Extended error hierarchy [BETA] - +-- 3.15 Observability layer [BETA] - +-- 3.16 Health HTTP endpoint [BETA] - +-- 3.17 Handler retry wrapper [BETA] - +-- 3.18 Contract schema/linting [BETA] - | - v -Phase 4 (Testing - Beta) - | - +-- 4.5 KafkaEventBus unit tests [BETA] - +-- 4.6 Integration tests with Docker [BETA] - +-- 4.7 Graceful shutdown tests [BETA] - +-- 4.8 Backpressure tests [BETA] - +-- 4.9 Topic naming validation [BETA] - | - v -Phase 5 (Deployment - Beta) - | - +-- 5.4 Expand docker-compose [BETA] - +-- 5.5 CLI environments doc [BETA] - +-- 5.6 Version compatibility doc [BETA] - | - v -v0.2.0 Release -``` - -### Production (v0.3.0) - Sprint 5 - -``` -Phase 4 (Testing - Production) - | - +-- 4.10 Chaos tests [PROD] - +-- 4.11 Performance benchmarks [PROD] - +-- 4.12 Complete architecture compliance [PROD] - | - v -Phase 5 (Deployment - Production) - | - +-- 5.7 Kubernetes manifests [PROD] - +-- 5.8 Migration guide [PROD] - | - v -v0.3.0 Release -``` - --- -## Success Metrics - -### MVP (v0.1.0) Success Metrics +## MVP Success Metrics | Metric | Target | Measurement | |--------|--------|-------------| @@ -2797,44 +1529,36 @@ v0.3.0 Release | Handler unit test coverage | >80% | pytest-cov | | MVP issue count | 24 | Linear tracking | -### Beta (v0.2.0) Success Metrics - -| Metric | Target | Measurement | -|--------|--------|-------------| -| Test coverage | >90% | pytest-cov | -| Integration tests pass | Yes | CI | -| Graceful shutdown drain | 100% | No dropped envelopes | -| Circuit breaker trigger | <10s | After threshold failures | -| Architecture violations | 0 | CI checks | +--- -### Production (v0.3.0) Success Metrics +## Expected First PRs (Contributor Onboarding) -| Metric | Target | Measurement | -|--------|--------|-------------| -| Memory per 10 nodes | <200MB | tracemalloc | -| Envelope throughput | >100/sec | Benchmark suite | -| Handler latency (local) | <1ms | p99 latency | -| Handler latency (http) | <100ms | p99 latency | -| Handler latency (db) | <50ms | p99 latency | -| Chaos test survival | 100% | No crashes | +New contributors should tackle issues in this order to build momentum: ---- +| PR # | Component | Repository | Complexity | Dependencies | +|------|-----------|------------|------------|--------------| +| 1 | `EnumHandlerType` | omnibase_core | S | None | +| 2 | `EnumNodeKind.RUNTIME_HOST` | omnibase_core | S | None | +| 3 | `ModelOnexEnvelope` | omnibase_core | M | Enums | +| 4 | `ProtocolHandler` | omnibase_core | M | Enums, Envelope | +| 5 | `ModelRuntimeHostContract` | omnibase_core | M | Enums | +| 6 | `ProtocolEventBus` export | omnibase_spi | S | Core protocols | +| 7 | `InMemoryEventBus` | omnibase_infra | M | SPI protocols | +| 8 | `HttpHandler` (skeleton) | omnibase_infra | M | ProtocolHandler | +| 9 | `DbHandler` (skeleton) | omnibase_infra | M | ProtocolHandler | +| 10 | `wiring.py` | omnibase_infra | M | Handlers | +| 11 | `BaseRuntimeHostProcess` (skeleton) | omnibase_infra | L | All above | +| 12 | E2E integration test | omnibase_infra | L | All above | -## Issue Creation Guidelines +**Complexity Legend**: S = Small (<50 LOC), M = Medium (50-200 LOC), L = Large (200+ LOC) -When creating these issues in Linear: +**First Week Goals**: +- Day 1-2: PRs 1-3 (enums and envelope) +- Day 3-4: PRs 4-6 (protocols) +- Day 5-7: PRs 7-9 (handlers) +- Week 2: PRs 10-12 (wiring and integration) -1. **Team**: Omninode -2. **Project**: MVP - ONEX Runtime Host Infrastructure -3. **Labels**: Apply as indicated + milestone tag (`mvp`, `beta`, `production`) -4. **Priority**: - - 1 = Urgent - - 2 = High - - 3 = Normal - - 4 = Low -5. **Dependencies**: Link related issues where indicated -6. **Repository**: Tag with appropriate repo (omnibase_core, omnibase_spi, omnibase_infra) -7. **Milestone**: v0.1.0 MVP, v0.2.0 Beta, or v0.3.0 Production +**Tip**: Each PR should include unit tests. Don't batch PRs - small, focused changes review faster. --- @@ -2977,60 +1701,6 @@ dependencies: [] --- -## How to Read This Document - -Different roles need different sections: - -| Role | Start Here | Focus On | -|------|------------|----------| -| **New Engineer** | Executive Summary -> Glossary -> Map of Abstractions | Phase 1 (Core Types) | -| **Infra Engineer** | Map of Abstractions -> Phase 3 (Handlers) | BaseRuntimeHostProcess, Handlers | -| **QA Engineer** | Testing Infrastructure -> Phase 4 | Test requirements, E2E flows | -| **DevOps** | Phase 5 (Deployment) | Dockerfile, K8s, docker-compose | -| **Architect** | Architecture Invariants -> Failure Examples | Invariants, Risk Items | -| **Product Manager** | Executive Summary -> Milestone Overview | Success Metrics | - -**Quick Navigation**: -- "What is this?" -> Executive Summary -- "What do the terms mean?" -> Glossary -- "What goes where?" -> Map of Abstractions -- "What MUST work?" -> MVP Constraints Checklist -- "What MUST NOT happen?" -> Failure Examples -- "What's the smallest working thing?" -> Minimum Reference Contract - ---- - -## Expected First PRs (Contributor Onboarding) - -New contributors should tackle issues in this order to build momentum: - -| PR # | Component | Repository | Complexity | Dependencies | -|------|-----------|------------|------------|--------------| -| 1 | `EnumHandlerType` | omnibase_core | S | None | -| 2 | `EnumNodeKind.RUNTIME_HOST` | omnibase_core | S | None | -| 3 | `ModelOnexEnvelope` | omnibase_core | M | Enums | -| 4 | `ProtocolHandler` | omnibase_core | M | Enums, Envelope | -| 5 | `ModelRuntimeHostContract` | omnibase_core | M | Enums | -| 6 | `ProtocolEventBus` export | omnibase_spi | S | Core protocols | -| 7 | `InMemoryEventBus` | omnibase_infra | M | SPI protocols | -| 8 | `HttpHandler` (skeleton) | omnibase_infra | M | ProtocolHandler | -| 9 | `DbHandler` (skeleton) | omnibase_infra | M | ProtocolHandler | -| 10 | `wiring.py` | omnibase_infra | M | Handlers | -| 11 | `BaseRuntimeHostProcess` (skeleton) | omnibase_infra | L | All above | -| 12 | E2E integration test | omnibase_infra | L | All above | - -**Complexity Legend**: S = Small (<50 LOC), M = Medium (50-200 LOC), L = Large (200+ LOC) - -**First Week Goals**: -- Day 1-2: PRs 1-3 (enums and envelope) -- Day 3-4: PRs 4-6 (protocols) -- Day 5-7: PRs 7-9 (handlers) -- Week 2: PRs 10-12 (wiring and integration) - -**Tip**: Each PR should include unit tests. Don't batch PRs - small, focused changes review faster. - ---- +> **Navigation**: [Back to Overview](../MVP_PLAN.md) | [Next: Beta Hardening](./BETA_v0.2.0_HARDENING.md) **Last Updated**: 2025-12-03 -**Document Owner**: OmniNode Architecture Team -**Linear Project URL**: TBD diff --git a/docs/milestones/PRODUCTION_v0.3.0.md b/docs/milestones/PRODUCTION_v0.3.0.md new file mode 100644 index 0000000000..1482270176 --- /dev/null +++ b/docs/milestones/PRODUCTION_v0.3.0.md @@ -0,0 +1,386 @@ +# Production (v0.3.0) - Milestone Details + +> **Navigation**: [Back to Overview](../MVP_PLAN.md) | [Previous: Beta Hardening](./BETA_v0.2.0_HARDENING.md) + +**Repository**: omnibase_infra +**Target Version**: v0.3.0 +**Timeline**: Sprint 5 +**Issue Count**: 8 +**Prerequisites**: MVP Core (v0.1.0) and Beta Hardening (v0.2.0) must be complete + +--- + +## Production Philosophy + +**Production (v0.3.0)**: Deploy and validate at scale +- Kubernetes manifests +- Chaos testing +- Performance benchmarks +- Complete documentation + +This milestone focuses on production deployment readiness: +- Kubernetes-native deployment configurations +- Chaos engineering to validate failure handling +- Performance benchmarking and baselines +- Complete migration documentation + +--- + +## Production Scope + +Production milestone adds the final pieces required for live deployment: + +1. **Chaos Testing** - Validate system behavior under failure conditions +2. **Performance Benchmarks** - Establish and enforce performance baselines +3. **Complete Architecture Compliance** - Full validation of all architectural invariants +4. **Kubernetes Manifests** - Production-ready deployment configurations +5. **Migration Documentation** - Complete guide for adopting Runtime Host architecture + +--- + +## Phase 4: Integration & Testing - Production Issues + +--- + +### Issue 4.10: Chaos and failure-mode tests [PROD] + +**Title**: Create chaos test suite +**Type**: Testing +**Priority**: Medium +**Labels**: `testing`, `chaos` +**Milestone**: v0.3.0 Production + +**Description**: +Test failure modes and recovery behavior. + +**Directory**: `tests/chaos/` + +**Scenarios**: +- Kafka connection loss (drop broker) +- Postgres unavailable for 5 seconds +- Vault network flakiness +- Handler returning errors continuously + +**Acceptance Criteria**: +- [ ] BaseRuntimeHostProcess does not crash +- [ ] Retries within configured budgets +- [ ] Health endpoints reflect degraded state +- [ ] Circuit breaker trips appropriately +- [ ] Recovery after service restoration + +--- + +### Issue 4.11: Performance benchmarks [PROD] + +**Title**: Establish performance baselines +**Type**: Testing +**Priority**: Medium +**Labels**: `testing`, `performance` +**Milestone**: v0.3.0 Production + +**Description**: +Create benchmark suite to verify performance targets. + +**Targets**: +- Memory per 10 nodes: <200MB +- Envelope throughput: >100/sec +- Handler latency (local): <1ms p99 +- Handler latency (http): <100ms p99 +- Handler latency (db): <50ms p99 + +**Acceptance Criteria**: +- [ ] Benchmark script created +- [ ] Results logged and tracked +- [ ] CI integration for regression detection +- [ ] Documentation of baselines + +--- + +### Issue 4.12: Complete architecture compliance [PROD] + +**Title**: Full architectural invariant verification +**Type**: Task +**Priority**: High +**Labels**: `architecture`, `validation` +**Milestone**: v0.3.0 Production + +**Description**: +Complete verification that all architectural invariants are maintained. + +**Additional Checks**: +- [ ] All handlers return `EnumHandlerType` +- [ ] `wiring.py` is only handler registration location +- [ ] No `os.getenv` in handlers (uses SecretResolver) +- [ ] Single BaseRuntimeHostProcess per process enforcement + +**Acceptance Criteria**: +- [ ] Shell script or pytest for verification +- [ ] Runs in CI +- [ ] Clear error messages on failure + +--- + +## Phase 5: Deployment & Migration - Production Issues + +--- + +### Issue 5.7: Kubernetes manifests [PROD] + +**Title**: Create Kubernetes deployment manifests +**Type**: DevOps +**Priority**: Medium +**Labels**: `deployment`, `kubernetes` +**Milestone**: v0.3.0 Production + +**Description**: +Create Kubernetes manifests for production deployment. + +**Files**: +- `k8s/deployment.yaml` +- `k8s/service.yaml` +- `k8s/configmap.yaml` +- `k8s/secrets.yaml` (template) + +**Acceptance Criteria**: +- [ ] Deployment with resource limits +- [ ] Service for internal access +- [ ] ConfigMap for contract +- [ ] Secret references documented +- [ ] Liveness probe: `/health/live` +- [ ] Readiness probe: `/health/ready` + +--- + +### Issue 5.8: Migration guide documentation [PROD] + +**Title**: Create migration guide from legacy architecture +**Type**: Documentation +**Priority**: Medium +**Labels**: `documentation`, `migration` +**Milestone**: v0.3.0 Production + +**Description**: +Document the migration path from 1-container-per-node to Runtime Host model. + +**File**: `docs/MIGRATION_GUIDE.md` + +**Acceptance Criteria**: +- [ ] Before/after architecture comparison +- [ ] Step-by-step migration process +- [ ] Rollback procedures +- [ ] Common issues and solutions + +--- + +## Production Execution Order + +``` +Phase 4 (Testing - Production) + | + +-- 4.10 Chaos tests [PROD] + +-- 4.11 Performance benchmarks [PROD] + +-- 4.12 Complete architecture compliance [PROD] + | + v +Phase 5 (Deployment - Production) + | + +-- 5.7 Kubernetes manifests [PROD] + +-- 5.8 Migration guide [PROD] + | + v +v0.3.0 Release +``` + +--- + +## Production Success Metrics + +| Metric | Target | Measurement | +|--------|--------|-------------| +| Memory per 10 nodes | <200MB | tracemalloc | +| Envelope throughput | >100/sec | Benchmark suite | +| Handler latency (local) | <1ms | p99 latency | +| Handler latency (http) | <100ms | p99 latency | +| Handler latency (db) | <50ms | p99 latency | +| Chaos test survival | 100% | No crashes | + +--- + +## Non-Goals (All Milestones) + +To prevent scope creep, the following are explicitly **NOT** in scope for any milestone: + +- **No multi-region failover** - Single cluster only +- **No automatic topic creation** - Assumes infra bootstrap handled separately +- **No dynamic handler discovery** - Static wiring from contract only +- **No auto-migration of legacy nodes** - Handled by higher-level repos +- **No LLM handler** - Deferred to future milestone +- **No Consul-based service mesh** - Basic discovery only (Beta) + +--- + +## Chaos Test Scenarios (Detailed) + +### Scenario 1: Kafka Broker Failure + +**Setup**: +1. Runtime Host connected to Kafka cluster +2. Processing envelopes at steady state +3. Kill primary Kafka broker + +**Expected Behavior**: +- BaseRuntimeHostProcess detects connection loss +- Health endpoint reports degraded state +- Retries connection with exponential backoff +- No data loss (at-least-once delivery) +- Automatic recovery when broker returns + +**Validation**: +- [ ] No crash during broker outage +- [ ] Logs show connection retry attempts +- [ ] Health endpoint shows `kafka_healthy: false` +- [ ] Envelopes resume processing after recovery + +### Scenario 2: Database Connection Pool Exhaustion + +**Setup**: +1. DbHandler with pool_size=5 +2. Send 20 concurrent database operations +3. Simulate slow queries (2s each) + +**Expected Behavior**: +- Pool blocks on acquisition +- Operations queue up +- Timeout after configured limit +- Error envelopes returned for timed-out operations +- Pool recovers when queries complete + +**Validation**: +- [ ] No connection leaks +- [ ] Proper timeout errors +- [ ] Pool metrics accurate +- [ ] Recovery without restart + +### Scenario 3: Handler Continuous Failure + +**Setup**: +1. HttpHandler configured with retry_policy +2. External service returns 500 errors +3. Run for 100 consecutive requests + +**Expected Behavior**: +- Retries exhausted per envelope +- Circuit breaker trips after threshold +- Error envelopes returned +- Health endpoint shows handler degraded +- Auto-recovery when service healthy + +**Validation**: +- [ ] Circuit breaker trip time <10s +- [ ] Health shows degraded state +- [ ] Proper error envelopes +- [ ] Recovery when errors stop + +### Scenario 4: Memory Pressure + +**Setup**: +1. Runtime Host with 10 nodes +2. Send large payload envelopes (1MB each) +3. Backpressure disabled for test + +**Expected Behavior**: +- Memory usage tracked +- GC pressure monitored +- Potential OOM if unconstrained +- Backpressure (when enabled) prevents OOM + +**Validation**: +- [ ] Memory stays under 200MB baseline +- [ ] No memory leaks over time +- [ ] Proper cleanup after processing + +--- + +## Kubernetes Deployment Specifications + +### Resource Requirements + +| Component | CPU Request | CPU Limit | Memory Request | Memory Limit | +|-----------|-------------|-----------|----------------|--------------| +| Runtime Host | 250m | 1000m | 256Mi | 512Mi | +| PostgreSQL | 500m | 2000m | 512Mi | 2Gi | +| Kafka | 500m | 2000m | 1Gi | 4Gi | +| Vault | 250m | 1000m | 256Mi | 512Mi | + +### Health Check Configuration + +```yaml +livenessProbe: + httpGet: + path: /health/live + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +readinessProbe: + httpGet: + path: /health/ready + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 2 +``` + +### Pod Disruption Budget + +```yaml +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: runtime-host-pdb +spec: + minAvailable: 1 + selector: + matchLabels: + app: runtime-host +``` + +--- + +## Migration Checklist + +When migrating from 1-container-per-node to Runtime Host: + +### Pre-Migration +- [ ] Inventory all existing nodes +- [ ] Document current resource usage per node +- [ ] Identify handler requirements per node +- [ ] Create runtime host contract + +### Migration Steps +- [ ] Deploy Runtime Host alongside existing nodes +- [ ] Route traffic percentage to Runtime Host +- [ ] Monitor performance and errors +- [ ] Increase traffic percentage gradually +- [ ] Decommission individual node containers + +### Post-Migration +- [ ] Verify all functionality preserved +- [ ] Compare resource usage (should be lower) +- [ ] Update monitoring dashboards +- [ ] Document any differences in behavior + +### Rollback Plan +- [ ] Keep old containers available for 7 days +- [ ] Traffic can be reverted instantly +- [ ] Database state unchanged +- [ ] Kafka topics unchanged + +--- + +> **Navigation**: [Back to Overview](../MVP_PLAN.md) | [Previous: Beta Hardening](./BETA_v0.2.0_HARDENING.md) + +**Last Updated**: 2025-12-03 diff --git a/poetry.lock b/poetry.lock index 592ebcd313..b414bd03a8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. [[package]] name = "aiofiles" @@ -1793,6 +1793,8 @@ files = [ {file = "greenlet-3.2.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c2ca18a03a8cfb5b25bc1cbe20f3d9a4c80d8c3b13ba3df49ac3961af0b1018d"}, {file = "greenlet-3.2.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9fe0a28a7b952a21e2c062cd5756d34354117796c6d9215a87f55e38d15402c5"}, {file = "greenlet-3.2.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8854167e06950ca75b898b104b63cc646573aa5fef1353d4508ecdd1ee76254f"}, + {file = "greenlet-3.2.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f47617f698838ba98f4ff4189aef02e7343952df3a615f847bb575c3feb177a7"}, + {file = "greenlet-3.2.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:af41be48a4f60429d5cad9d22175217805098a9ef7c40bfef44f7669fb9d74d8"}, {file = "greenlet-3.2.4-cp310-cp310-win_amd64.whl", hash = "sha256:73f49b5368b5359d04e18d15828eecc1806033db5233397748f4ca813ff1056c"}, {file = "greenlet-3.2.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:96378df1de302bc38e99c3a9aa311967b7dc80ced1dcc6f171e99842987882a2"}, {file = "greenlet-3.2.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1ee8fae0519a337f2329cb78bd7a8e128ec0f881073d43f023c7b8d4831d5246"}, @@ -1802,6 +1804,8 @@ files = [ {file = "greenlet-3.2.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2523e5246274f54fdadbce8494458a2ebdcdbc7b802318466ac5606d3cded1f8"}, {file = "greenlet-3.2.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1987de92fec508535687fb807a5cea1560f6196285a4cde35c100b8cd632cc52"}, {file = "greenlet-3.2.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:55e9c5affaa6775e2c6b67659f3a71684de4c549b3dd9afca3bc773533d284fa"}, + {file = "greenlet-3.2.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c9c6de1940a7d828635fbd254d69db79e54619f165ee7ce32fda763a9cb6a58c"}, + {file = "greenlet-3.2.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03c5136e7be905045160b1b9fdca93dd6727b180feeafda6818e6496434ed8c5"}, {file = "greenlet-3.2.4-cp311-cp311-win_amd64.whl", hash = "sha256:9c40adce87eaa9ddb593ccb0fa6a07caf34015a29bf8d344811665b573138db9"}, {file = "greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd"}, {file = "greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb"}, @@ -1811,6 +1815,8 @@ files = [ {file = "greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0"}, {file = "greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0"}, {file = "greenlet-3.2.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:20fb936b4652b6e307b8f347665e2c615540d4b42b3b4c8a321d8286da7e520f"}, + {file = "greenlet-3.2.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ee7a6ec486883397d70eec05059353b8e83eca9168b9f3f9a361971e77e0bcd0"}, + {file = "greenlet-3.2.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:326d234cbf337c9c3def0676412eb7040a35a768efc92504b947b3e9cfc7543d"}, {file = "greenlet-3.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:a7d4e128405eea3814a12cc2605e0e6aedb4035bf32697f72deca74de4105e02"}, {file = "greenlet-3.2.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:1a921e542453fe531144e91e1feedf12e07351b1cf6c9e8a3325ea600a715a31"}, {file = "greenlet-3.2.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd3c8e693bff0fff6ba55f140bf390fa92c994083f838fece0f63be121334945"}, @@ -1820,6 +1826,8 @@ files = [ {file = "greenlet-3.2.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23768528f2911bcd7e475210822ffb5254ed10d71f4028387e5a99b4c6699671"}, {file = "greenlet-3.2.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:00fadb3fedccc447f517ee0d3fd8fe49eae949e1cd0f6a611818f4f6fb7dc83b"}, {file = "greenlet-3.2.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d25c5091190f2dc0eaa3f950252122edbbadbb682aa7b1ef2f8af0f8c0afefae"}, + {file = "greenlet-3.2.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6e343822feb58ac4d0a1211bd9399de2b3a04963ddeec21530fc426cc121f19b"}, + {file = "greenlet-3.2.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ca7f6f1f2649b89ce02f6f229d7c19f680a6238af656f61e0115b24857917929"}, {file = "greenlet-3.2.4-cp313-cp313-win_amd64.whl", hash = "sha256:554b03b6e73aaabec3745364d6239e9e012d64c68ccd0b8430c64ccc14939a8b"}, {file = "greenlet-3.2.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:49a30d5fda2507ae77be16479bdb62a660fa51b1eb4928b524975b3bde77b3c0"}, {file = "greenlet-3.2.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:299fd615cd8fc86267b47597123e3f43ad79c9d8a22bebdce535e53550763e2f"}, @@ -1827,6 +1835,8 @@ files = [ {file = "greenlet-3.2.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b4a1870c51720687af7fa3e7cda6d08d801dae660f75a76f3845b642b4da6ee1"}, {file = "greenlet-3.2.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:061dc4cf2c34852b052a8620d40f36324554bc192be474b9e9770e8c042fd735"}, {file = "greenlet-3.2.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44358b9bf66c8576a9f57a590d5f5d6e72fa4228b763d0e43fee6d3b06d3a337"}, + {file = "greenlet-3.2.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2917bdf657f5859fbf3386b12d68ede4cf1f04c90c3a6bc1f013dd68a22e2269"}, + {file = "greenlet-3.2.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:015d48959d4add5d6c9f6c5210ee3803a830dce46356e3bc326d6776bde54681"}, {file = "greenlet-3.2.4-cp314-cp314-win_amd64.whl", hash = "sha256:e37ab26028f12dbb0ff65f29a8d3d44a765c61e729647bf2ddfbbed621726f01"}, {file = "greenlet-3.2.4-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:b6a7c19cf0d2742d0809a4c05975db036fdff50cd294a93632d6a310bf9ac02c"}, {file = "greenlet-3.2.4-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:27890167f55d2387576d1f41d9487ef171849ea0359ce1510ca6e06c8bece11d"}, @@ -1836,6 +1846,8 @@ files = [ {file = "greenlet-3.2.4-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c9913f1a30e4526f432991f89ae263459b1c64d1608c0d22a5c79c287b3c70df"}, {file = "greenlet-3.2.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b90654e092f928f110e0007f572007c9727b5265f7632c2fa7415b4689351594"}, {file = "greenlet-3.2.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:81701fd84f26330f0d5f4944d4e92e61afe6319dcd9775e39396e39d7c3e5f98"}, + {file = "greenlet-3.2.4-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:28a3c6b7cd72a96f61b0e4b2a36f681025b60ae4779cc73c1535eb5f29560b10"}, + {file = "greenlet-3.2.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:52206cd642670b0b320a1fd1cbfd95bca0e043179c1d8a045f2c6109dfe973be"}, {file = "greenlet-3.2.4-cp39-cp39-win32.whl", hash = "sha256:65458b409c1ed459ea899e939f0e1cdb14f58dbc803f2f93c5eab5694d32671b"}, {file = "greenlet-3.2.4-cp39-cp39-win_amd64.whl", hash = "sha256:d2e685ade4dafd447ede19c31277a224a239a0a1a4eca4e6390efedf20260cfb"}, {file = "greenlet-3.2.4.tar.gz", hash = "sha256:0dca0d95ff849f9a364385f36ab49f50065d76964944638be9691e1832e9f86d"}, @@ -6230,4 +6242,4 @@ testing = ["coverage[toml]", "zope.event", "zope.testing"] [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "e3b52a79ab7550343c8d09d3d2759ffc868684e7cf66772270054debeb572b65" +content-hash = "e793182cecd2a7bfe17f4f3d05c530047b4a53a368e6e6eded66e8c7d405ce48" diff --git a/pyproject.toml b/pyproject.toml index 704f08d249..9d0cefca0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ omni-infra = "omnibase_infra.cli.commands:cli" python = "^3.12" # ONEX dependencies - from PyPI -omnibase-core = "0.3.5" +omnibase-core = "^0.3.5" omnibase-spi = "0.2.0" # Core dependencies diff --git a/src/omnibase_infra/enums/__init__.py b/src/omnibase_infra/enums/__init__.py index e69de29bb2..ecd3cfcd04 100644 --- a/src/omnibase_infra/enums/__init__.py +++ b/src/omnibase_infra/enums/__init__.py @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2025 OmniNode Team +"""ONEX Infrastructure Enumerations Module. + +Provides infrastructure-specific enumerations for transport types, +protocol identification, and classification. + +Exports: + EnumInfraTransportType: Infrastructure transport type enumeration +""" + +from omnibase_infra.enums.enum_infra_transport_type import EnumInfraTransportType + +__all__ = ["EnumInfraTransportType"] diff --git a/src/omnibase_infra/enums/enum_infra_transport_type.py b/src/omnibase_infra/enums/enum_infra_transport_type.py new file mode 100644 index 0000000000..0e0acc1f78 --- /dev/null +++ b/src/omnibase_infra/enums/enum_infra_transport_type.py @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2025 OmniNode Team +"""Infrastructure Transport Type Enumeration. + +Defines the canonical transport types for infrastructure components. +Used for error context, protocol routing, and transport identification. +""" + +from enum import Enum + + +class EnumInfraTransportType(str, Enum): + """Infrastructure transport types for ONEX infrastructure components. + + These represent the transport/protocol layer types used in + omnibase_infra for external integration. + + Attributes: + HTTP: HTTP/REST API transport + DATABASE: Database connection transport (PostgreSQL, etc.) + KAFKA: Kafka message broker transport + CONSUL: Consul discovery transport + VAULT: HashiCorp Vault secret transport + REDIS: Redis cache/message transport + GRPC: gRPC protocol transport + """ + + HTTP = "http" + DATABASE = "db" + KAFKA = "kafka" + CONSUL = "consul" + VAULT = "vault" + REDIS = "redis" + GRPC = "grpc" + + +__all__ = ["EnumInfraTransportType"] diff --git a/src/omnibase_infra/errors/__init__.py b/src/omnibase_infra/errors/__init__.py new file mode 100644 index 0000000000..5feee08922 --- /dev/null +++ b/src/omnibase_infra/errors/__init__.py @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2025 OmniNode Team +"""ONEX Infrastructure Errors Module. + +This module provides infrastructure-specific error classes and error handling +utilities for the omnibase_infra package. All errors extend from OnexError +to maintain consistency with the ONEX error handling patterns. + +Exports: + ModelInfraErrorContext: Configuration model for bundled error context + RuntimeHostError: Base infrastructure error class + ProtocolConfigurationError: Protocol configuration validation errors + SecretResolutionError: Secret/credential resolution errors + InfraConnectionError: Infrastructure connection errors + InfraTimeoutError: Infrastructure timeout errors + InfraAuthenticationError: Infrastructure authentication errors + InfraUnavailableError: Infrastructure resource unavailable errors + +Correlation ID Assignment: + All infrastructure errors support correlation_id for distributed tracing. + Follow these rules when assigning correlation IDs: + + - Always propagate correlation_id from incoming requests to error context + - If no correlation_id exists in the request, generate one using uuid4() + - Use UUID4 format for all new correlation IDs (from uuid import uuid4) + - Include correlation_id in all error context for distributed tracing + - Preserve correlation_id as UUID objects throughout the system (strong typing) + + Example:: + + from uuid import UUID, uuid4 + from omnibase_infra.errors import InfraConnectionError, ModelInfraErrorContext + from omnibase_infra.enums import EnumInfraTransportType + + # Propagate from request or generate new + correlation_id = request.correlation_id or uuid4() + + context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.DATABASE, + operation="execute_query", + target_name="postgresql-primary", + correlation_id=correlation_id, + ) + raise InfraConnectionError("Failed to connect", context=context) from e + +Error Sanitization Guidelines: + NEVER include in error messages or context: + - Passwords, API keys, tokens, or secrets + - Full connection strings with credentials + - PII (names, emails, SSNs, phone numbers) + - Internal IP addresses (in production logs) + - Private keys or certificates + - Session tokens or cookies + + SAFE to include: + - Service names (e.g., "postgresql", "kafka") + - Operation names (e.g., "connect", "query", "authenticate") + - Correlation IDs (always include for tracing) + - Error codes (e.g., EnumCoreErrorCode.DATABASE_CONNECTION_ERROR) + - Sanitized hostnames (e.g., "db.example.com") + - Port numbers + - Retry counts and timeout values + - Resource identifiers (non-sensitive) + + Example - BAD (exposes credentials):: + + raise InfraConnectionError( + f"Failed to connect with password={password}", # NEVER DO THIS + context=context, + ) + + Example - GOOD (sanitized):: + + raise InfraConnectionError( + "Failed to connect to database", + context=context, + host="db.example.com", + port=5432, + retry_count=3, + ) +""" + +from omnibase_infra.errors.infra_errors import ( + InfraAuthenticationError, + InfraConnectionError, + InfraTimeoutError, + InfraUnavailableError, + ProtocolConfigurationError, + RuntimeHostError, + SecretResolutionError, +) +from omnibase_infra.errors.model_infra_error_context import ModelInfraErrorContext + +__all__: list[str] = [ + # Configuration model + "ModelInfraErrorContext", + # Error classes + "RuntimeHostError", + "ProtocolConfigurationError", + "SecretResolutionError", + "InfraConnectionError", + "InfraTimeoutError", + "InfraAuthenticationError", + "InfraUnavailableError", +] diff --git a/src/omnibase_infra/errors/infra_errors.py b/src/omnibase_infra/errors/infra_errors.py new file mode 100644 index 0000000000..ddf8b045fd --- /dev/null +++ b/src/omnibase_infra/errors/infra_errors.py @@ -0,0 +1,418 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2025 OmniNode Team +"""Infrastructure-Specific Error Classes. + +This module defines infrastructure-specific error classes for the +omnibase_infra package. All error classes extend from ModelOnexError +(from omnibase_core) to maintain consistency with ONEX error handling patterns. + +Error Hierarchy: + ModelOnexError (from omnibase_core) + └── RuntimeHostError (base infrastructure error) + β”œβ”€β”€ ProtocolConfigurationError + β”œβ”€β”€ SecretResolutionError + β”œβ”€β”€ InfraConnectionError + β”œβ”€β”€ InfraTimeoutError + β”œβ”€β”€ InfraAuthenticationError + └── InfraUnavailableError + +All errors: + - Extend ModelOnexError from omnibase_core + - Use EnumCoreErrorCode for error classification + - Support proper error chaining with `raise ... from e` + - Include structured context for debugging + - Support correlation IDs for request tracking + - Accept ModelInfraErrorContext for bundled context parameters +""" + +from typing import Optional + +from omnibase_core.enums.enum_core_error_code import EnumCoreErrorCode +from omnibase_core.models.errors.model_onex_error import ModelOnexError + +from omnibase_infra.enums import EnumInfraTransportType +from omnibase_infra.errors.model_infra_error_context import ModelInfraErrorContext + + +class RuntimeHostError(ModelOnexError): + """Base error class for runtime host infrastructure errors. + + All infrastructure-specific errors should inherit from this class. + Provides common structured fields for infrastructure operations. + + Structured Fields (via ModelInfraErrorContext): + transport_type: Type of transport (http, db, kafka, etc.) + operation: Operation being performed + correlation_id: Request correlation ID for tracking + target_name: Target resource/endpoint name + + Example: + >>> context = ModelInfraErrorContext( + ... transport_type=EnumInfraTransportType.HTTP, + ... operation="process_request", + ... target_name="api-gateway", + ... ) + >>> raise RuntimeHostError("Operation failed", context=context) + + # Or with extra context: + >>> raise RuntimeHostError( + ... "Operation failed", + ... context=context, + ... retry_count=3, + ... ) + """ + + def __init__( + self, + message: str, + error_code: Optional[EnumCoreErrorCode] = None, + context: Optional[ModelInfraErrorContext] = None, + **extra_context: object, + ) -> None: + """Initialize RuntimeHostError with structured fields. + + Args: + message: Human-readable error message + error_code: Error code (defaults to OPERATION_FAILED) + context: Bundled infrastructure context (transport_type, operation, etc.) + **extra_context: Additional context information + """ + # Build structured context from model and extra kwargs + structured_context: dict[str, object] = dict(extra_context) + + # Extract fields from context model if provided + correlation_id = None + if context is not None: + if context.transport_type is not None: + structured_context["transport_type"] = context.transport_type + if context.operation is not None: + structured_context["operation"] = context.operation + if context.target_name is not None: + structured_context["target_name"] = context.target_name + correlation_id = context.correlation_id + + # Initialize base error with default error code + super().__init__( + message=message, + error_code=error_code or EnumCoreErrorCode.OPERATION_FAILED, + correlation_id=correlation_id, + **structured_context, + ) + + +class ProtocolConfigurationError(RuntimeHostError): + """Raised when protocol configuration validation fails. + + Used for configuration parsing errors, missing required fields, + invalid configuration values, or schema validation failures. + + Example: + >>> context = ModelInfraErrorContext( + ... transport_type=EnumInfraTransportType.HTTP, + ... operation="validate_config", + ... ) + >>> raise ProtocolConfigurationError( + ... "Missing required field 'endpoint'", + ... context=context, + ... ) + """ + + def __init__( + self, + message: str, + context: Optional[ModelInfraErrorContext] = None, + **extra_context: object, + ) -> None: + """Initialize ProtocolConfigurationError. + + Args: + message: Human-readable error message + context: Bundled infrastructure context + **extra_context: Additional context information + """ + super().__init__( + message=message, + error_code=EnumCoreErrorCode.INVALID_CONFIGURATION, + context=context, + **extra_context, + ) + + +class SecretResolutionError(RuntimeHostError): + """Raised when secret or credential resolution fails. + + Used for Vault connection failures, missing secrets, expired credentials, + or permission issues accessing secret stores. + + Example: + >>> context = ModelInfraErrorContext( + ... transport_type=EnumInfraTransportType.VAULT, + ... operation="get_secret", + ... target_name="vault-primary", + ... ) + >>> raise SecretResolutionError( + ... "Secret not found in Vault", + ... context=context, + ... secret_key="database/postgres/password", + ... ) + """ + + def __init__( + self, + message: str, + context: Optional[ModelInfraErrorContext] = None, + **extra_context: object, + ) -> None: + """Initialize SecretResolutionError. + + Args: + message: Human-readable error message + context: Bundled infrastructure context + **extra_context: Additional context information (e.g., secret_key, vault_path) + """ + super().__init__( + message=message, + error_code=EnumCoreErrorCode.RESOURCE_NOT_FOUND, + context=context, + **extra_context, + ) + + +class InfraConnectionError(RuntimeHostError): + """Raised when infrastructure connection fails. + + Used for database connection failures, mesh connectivity issues, + message broker connection problems, or network-related errors. + + The error code is automatically selected based on the transport type + in the context: + - DATABASE -> DATABASE_CONNECTION_ERROR + - HTTP, GRPC -> NETWORK_ERROR + - KAFKA, CONSUL, VAULT, REDIS -> SERVICE_UNAVAILABLE + - None (no context) -> SERVICE_UNAVAILABLE + + Example: + >>> # Database connection with transport-specific error code + >>> context = ModelInfraErrorContext( + ... transport_type=EnumInfraTransportType.DATABASE, + ... operation="connect", + ... target_name="postgresql-primary", + ... ) + >>> raise InfraConnectionError( + ... "Failed to connect to PostgreSQL", + ... context=context, + ... host="db.example.com", + ... port=5432, + ... ) + + >>> # HTTP connection uses NETWORK_ERROR + >>> context = ModelInfraErrorContext( + ... transport_type=EnumInfraTransportType.HTTP, + ... operation="request", + ... target_name="api-gateway", + ... ) + >>> raise InfraConnectionError("API connection failed", context=context) + + >>> # Kafka connection uses SERVICE_UNAVAILABLE + >>> context = ModelInfraErrorContext( + ... transport_type=EnumInfraTransportType.KAFKA, + ... operation="produce", + ... target_name="kafka-broker", + ... ) + >>> raise InfraConnectionError("Kafka connection failed", context=context) + """ + + # Transport type to error code mapping + _TRANSPORT_ERROR_CODE_MAP: dict[ + Optional[EnumInfraTransportType], EnumCoreErrorCode + ] = { + EnumInfraTransportType.DATABASE: EnumCoreErrorCode.DATABASE_CONNECTION_ERROR, + EnumInfraTransportType.HTTP: EnumCoreErrorCode.NETWORK_ERROR, + EnumInfraTransportType.GRPC: EnumCoreErrorCode.NETWORK_ERROR, + EnumInfraTransportType.KAFKA: EnumCoreErrorCode.SERVICE_UNAVAILABLE, + EnumInfraTransportType.CONSUL: EnumCoreErrorCode.SERVICE_UNAVAILABLE, + EnumInfraTransportType.VAULT: EnumCoreErrorCode.SERVICE_UNAVAILABLE, + EnumInfraTransportType.REDIS: EnumCoreErrorCode.SERVICE_UNAVAILABLE, + None: EnumCoreErrorCode.SERVICE_UNAVAILABLE, + } + + @classmethod + def _resolve_connection_error_code( + cls, context: Optional[ModelInfraErrorContext] + ) -> EnumCoreErrorCode: + """Resolve the appropriate error code based on transport type. + + Args: + context: Infrastructure error context containing transport type + + Returns: + Appropriate EnumCoreErrorCode for the transport type: + - DATABASE -> DATABASE_CONNECTION_ERROR + - HTTP, GRPC -> NETWORK_ERROR + - KAFKA, CONSUL, VAULT, REDIS, None -> SERVICE_UNAVAILABLE + """ + if context is None: + return cls._TRANSPORT_ERROR_CODE_MAP[None] + return cls._TRANSPORT_ERROR_CODE_MAP.get( + context.transport_type, + EnumCoreErrorCode.SERVICE_UNAVAILABLE, + ) + + def __init__( + self, + message: str, + context: Optional[ModelInfraErrorContext] = None, + **extra_context: object, + ) -> None: + """Initialize InfraConnectionError with transport-aware error code. + + The error code is automatically selected based on context.transport_type: + - DATABASE -> DATABASE_CONNECTION_ERROR + - HTTP, GRPC -> NETWORK_ERROR + - KAFKA, CONSUL, VAULT, REDIS -> SERVICE_UNAVAILABLE + - None (no context) -> SERVICE_UNAVAILABLE + + Args: + message: Human-readable error message + context: Bundled infrastructure context (transport_type determines error code) + **extra_context: Additional context information (e.g., host, port, retry_count) + """ + super().__init__( + message=message, + error_code=self._resolve_connection_error_code(context), + context=context, + **extra_context, + ) + + +class InfraTimeoutError(RuntimeHostError): + """Raised when infrastructure operation exceeds timeout. + + Used for database query timeouts, HTTP request timeouts, + message broker operation timeouts, or call deadlines. + + Example: + >>> context = ModelInfraErrorContext( + ... transport_type=EnumInfraTransportType.DATABASE, + ... operation="execute_query", + ... target_name="postgresql-primary", + ... ) + >>> raise InfraTimeoutError( + ... "Database query exceeded timeout", + ... context=context, + ... timeout_seconds=30, + ... ) + """ + + def __init__( + self, + message: str, + context: Optional[ModelInfraErrorContext] = None, + **extra_context: object, + ) -> None: + """Initialize InfraTimeoutError. + + Args: + message: Human-readable error message + context: Bundled infrastructure context + **extra_context: Additional context information (e.g., timeout_seconds) + """ + super().__init__( + message=message, + error_code=EnumCoreErrorCode.TIMEOUT_ERROR, + context=context, + **extra_context, + ) + + +class InfraAuthenticationError(RuntimeHostError): + """Raised when infrastructure authentication or authorization fails. + + Used for invalid credentials, expired tokens, insufficient permissions, + or authentication failures. + + Example: + >>> context = ModelInfraErrorContext( + ... transport_type=EnumInfraTransportType.VAULT, + ... operation="authenticate", + ... target_name="vault-primary", + ... ) + >>> raise InfraAuthenticationError( + ... "Invalid Vault token", + ... context=context, + ... auth_method="token", + ... ) + """ + + def __init__( + self, + message: str, + context: Optional[ModelInfraErrorContext] = None, + **extra_context: object, + ) -> None: + """Initialize InfraAuthenticationError. + + Args: + message: Human-readable error message + context: Bundled infrastructure context + **extra_context: Additional context information (e.g., username, auth_method) + """ + super().__init__( + message=message, + error_code=EnumCoreErrorCode.AUTHENTICATION_ERROR, + context=context, + **extra_context, + ) + + +class InfraUnavailableError(RuntimeHostError): + """Raised when infrastructure resource is unavailable. + + Used for resource downtime, maintenance mode, circuit breaker states, + or health check failures. + + Example: + >>> context = ModelInfraErrorContext( + ... transport_type=EnumInfraTransportType.KAFKA, + ... operation="produce", + ... target_name="kafka-broker-1", + ... ) + >>> raise InfraUnavailableError( + ... "Kafka broker unavailable", + ... context=context, + ... host="kafka.example.com", + ... port=9092, + ... retry_count=3, + ... ) + """ + + def __init__( + self, + message: str, + context: Optional[ModelInfraErrorContext] = None, + **extra_context: object, + ) -> None: + """Initialize InfraUnavailableError. + + Args: + message: Human-readable error message + context: Bundled infrastructure context + **extra_context: Additional context information (e.g., host, port, retry_count) + """ + super().__init__( + message=message, + error_code=EnumCoreErrorCode.SERVICE_UNAVAILABLE, + context=context, + **extra_context, + ) + + +__all__ = [ + "RuntimeHostError", + "ProtocolConfigurationError", + "SecretResolutionError", + "InfraConnectionError", + "InfraTimeoutError", + "InfraAuthenticationError", + "InfraUnavailableError", +] diff --git a/src/omnibase_infra/errors/model_infra_error_context.py b/src/omnibase_infra/errors/model_infra_error_context.py new file mode 100644 index 0000000000..09b918853c --- /dev/null +++ b/src/omnibase_infra/errors/model_infra_error_context.py @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2025 OmniNode Team +"""Infrastructure Error Context Configuration Model. + +This module defines the configuration model for infrastructure error context, +encapsulating common structured fields to reduce __init__ parameter count +while maintaining strong typing per ONEX standards. +""" + +from typing import Optional +from uuid import UUID, uuid4 + +from pydantic import BaseModel, ConfigDict, Field + +from omnibase_infra.enums import EnumInfraTransportType + + +class ModelInfraErrorContext(BaseModel): + """Configuration model for infrastructure error context. + + Encapsulates common structured fields for infrastructure errors + to reduce __init__ parameter count while maintaining strong typing. + This follows the ONEX pattern of using configuration models to + bundle related parameters. + + Attributes: + transport_type: Type of infrastructure transport (HTTP, DATABASE, KAFKA, etc.) + operation: Operation being performed (connect, query, authenticate, etc.) + target_name: Target resource or endpoint name + correlation_id: Request correlation ID for distributed tracing + + Example: + >>> context = ModelInfraErrorContext( + ... transport_type=EnumInfraTransportType.HTTP, + ... operation="process_request", + ... target_name="api-gateway", + ... correlation_id=uuid4(), + ... ) + >>> raise RuntimeHostError("Operation failed", context=context) + """ + + model_config = ConfigDict( + frozen=True, # Immutable for thread safety + extra="forbid", # Strict validation - no extra fields + ) + + transport_type: Optional[EnumInfraTransportType] = Field( + default=None, + description="Type of infrastructure transport (HTTP, DATABASE, KAFKA, etc.)", + ) + operation: Optional[str] = Field( + default=None, + description="Operation being performed (connect, query, authenticate, etc.)", + ) + target_name: Optional[str] = Field( + default=None, + description="Target resource or endpoint name", + ) + correlation_id: Optional[UUID] = Field( + default=None, + description="Request correlation ID for distributed tracing", + ) + + @classmethod + def with_correlation( + cls, + correlation_id: Optional[UUID] = None, + **kwargs: object, + ) -> "ModelInfraErrorContext": + """Create context with auto-generated correlation_id if not provided. + + This factory method ensures a correlation_id is always present, + generating one if not explicitly provided. Useful for distributed + tracing scenarios where every error should be traceable. + + Args: + correlation_id: Optional correlation ID. If None, one is auto-generated. + **kwargs: Additional context fields (transport_type, operation, target_name). + + Returns: + ModelInfraErrorContext with guaranteed correlation_id. + + Example: + >>> context = ModelInfraErrorContext.with_correlation( + ... transport_type=EnumInfraTransportType.HTTP, + ... operation="process_request", + ... ) + >>> assert context.correlation_id is not None + """ + return cls(correlation_id=correlation_id or uuid4(), **kwargs) + + +__all__ = ["ModelInfraErrorContext"] diff --git a/tests/unit/errors/__init__.py b/tests/unit/errors/__init__.py new file mode 100644 index 0000000000..5b61d02b53 --- /dev/null +++ b/tests/unit/errors/__init__.py @@ -0,0 +1 @@ +"""Unit tests for infrastructure error classes.""" diff --git a/tests/unit/errors/test_infra_errors.py b/tests/unit/errors/test_infra_errors.py new file mode 100644 index 0000000000..83f817b4de --- /dev/null +++ b/tests/unit/errors/test_infra_errors.py @@ -0,0 +1,1103 @@ +""" +Comprehensive tests for infrastructure error classes. + +Tests follow TDD approach: +1. Write tests first (red phase) +2. Implement error classes (green phase) +3. Refactor if needed (refactor phase) + +All tests validate: +- Error class instantiation +- Inheritance chain +- Error chaining (raise ... from e) +- Structured context fields via ModelInfraErrorContext +- Error code mapping +- Required fields storage +""" + +from uuid import uuid4 + +import pytest +from omnibase_core.enums.enum_core_error_code import EnumCoreErrorCode +from omnibase_core.errors import ModelOnexError +from pydantic import ValidationError + +from omnibase_infra.enums import EnumInfraTransportType +from omnibase_infra.errors import ModelInfraErrorContext +from omnibase_infra.errors.infra_errors import ( + InfraAuthenticationError, + InfraConnectionError, + InfraTimeoutError, + InfraUnavailableError, + ProtocolConfigurationError, + RuntimeHostError, + SecretResolutionError, +) + + +class TestModelInfraErrorContextWithCorrelation: + """Tests for ModelInfraErrorContext.with_correlation() factory method.""" + + def test_with_correlation_generates_uuid_when_none(self) -> None: + """Test that with_correlation generates a UUID when none is provided.""" + context = ModelInfraErrorContext.with_correlation() + assert context.correlation_id is not None + + def test_with_correlation_uses_provided_uuid(self) -> None: + """Test that with_correlation uses the provided UUID when given.""" + provided_id = uuid4() + context = ModelInfraErrorContext.with_correlation(correlation_id=provided_id) + assert context.correlation_id == provided_id + + def test_with_correlation_with_other_fields(self) -> None: + """Test that with_correlation correctly passes through other kwargs.""" + context = ModelInfraErrorContext.with_correlation( + transport_type=EnumInfraTransportType.HTTP, + operation="process_request", + target_name="api-gateway", + ) + assert context.correlation_id is not None + assert context.transport_type == EnumInfraTransportType.HTTP + assert context.operation == "process_request" + assert context.target_name == "api-gateway" + + def test_with_correlation_uuid_is_valid(self) -> None: + """Test that the generated UUID is a valid UUID4.""" + from uuid import UUID + + context = ModelInfraErrorContext.with_correlation() + # Verify it's a valid UUID object + assert isinstance(context.correlation_id, UUID) + # Verify it's a valid UUID4 (version 4) + assert context.correlation_id.version == 4 + + +class TestModelInfraErrorContext: + """Tests for ModelInfraErrorContext configuration model.""" + + def test_basic_instantiation(self) -> None: + """Test basic context model instantiation.""" + context = ModelInfraErrorContext() + assert context.transport_type is None + assert context.operation is None + assert context.target_name is None + assert context.correlation_id is None + + def test_with_all_fields(self) -> None: + """Test context model with all fields populated.""" + correlation_id = uuid4() + context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.HTTP, + operation="process_request", + target_name="api-gateway", + correlation_id=correlation_id, + ) + assert context.transport_type == EnumInfraTransportType.HTTP + assert context.operation == "process_request" + assert context.target_name == "api-gateway" + assert context.correlation_id == correlation_id + + def test_immutability(self) -> None: + """Test that context model is immutable (frozen).""" + context = ModelInfraErrorContext(transport_type=EnumInfraTransportType.HTTP) + with pytest.raises(ValidationError): + context.transport_type = EnumInfraTransportType.DATABASE # type: ignore[misc] + + +class TestRuntimeHostError: + """Tests for RuntimeHostError base class.""" + + def test_basic_instantiation(self) -> None: + """Test basic error instantiation.""" + error = RuntimeHostError("Test error message") + assert "Test error message" in str(error) + assert isinstance(error, ModelOnexError) + + def test_with_context_model(self) -> None: + """Test error with context model.""" + correlation_id = uuid4() + context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.HTTP, + operation="process_request", + target_name="api-endpoint", + correlation_id=correlation_id, + ) + error = RuntimeHostError("Test error", context=context) + assert error.model.correlation_id == correlation_id + assert error.model.context["transport_type"] == EnumInfraTransportType.HTTP + assert error.model.context["operation"] == "process_request" + assert error.model.context["target_name"] == "api-endpoint" + + def test_with_error_code(self) -> None: + """Test error with explicit error code.""" + error = RuntimeHostError( + "Test error", error_code=EnumCoreErrorCode.OPERATION_FAILED + ) + assert error.model.error_code == EnumCoreErrorCode.OPERATION_FAILED + + def test_with_extra_context(self) -> None: + """Test error with extra context via kwargs.""" + context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.HTTP, + operation="process_request", + ) + error = RuntimeHostError( + "Test error", + context=context, + retry_count=3, + endpoint="/api/v1/users", + ) + assert error.model.context["transport_type"] == EnumInfraTransportType.HTTP + assert error.model.context["retry_count"] == 3 + assert error.model.context["endpoint"] == "/api/v1/users" + + def test_error_chaining(self) -> None: + """Test error chaining with 'raise ... from e' pattern.""" + original_error = ValueError("Original error") + try: + raise RuntimeHostError("Wrapped error") from original_error + except RuntimeHostError as e: + assert e.__cause__ == original_error + assert isinstance(e.__cause__, ValueError) + + def test_inheritance_chain(self) -> None: + """Test that RuntimeHostError properly inherits from ModelOnexError.""" + error = RuntimeHostError("Test error") + assert isinstance(error, RuntimeHostError) + assert isinstance(error, ModelOnexError) + assert isinstance(error, Exception) + + +class TestProtocolConfigurationError: + """Tests for ProtocolConfigurationError.""" + + def test_basic_instantiation(self) -> None: + """Test basic error instantiation.""" + error = ProtocolConfigurationError("Invalid config") + assert "Invalid config" in str(error) + assert isinstance(error, RuntimeHostError) + + def test_with_context_model(self) -> None: + """Test error with context model.""" + context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.HTTP, + operation="validate_config", + ) + error = ProtocolConfigurationError("Invalid config", context=context) + assert error.model.context["transport_type"] == EnumInfraTransportType.HTTP + assert error.model.context["operation"] == "validate_config" + + def test_error_code_mapping(self) -> None: + """Test that error uses appropriate CoreErrorCode.""" + error = ProtocolConfigurationError("Config error") + assert error.model.error_code == EnumCoreErrorCode.INVALID_CONFIGURATION + + def test_error_chaining(self) -> None: + """Test error chaining from original exception.""" + context = ModelInfraErrorContext(transport_type=EnumInfraTransportType.DATABASE) + config_error = KeyError("missing_key") + try: + raise ProtocolConfigurationError( + "Missing required config key", context=context + ) from config_error + except ProtocolConfigurationError as e: + assert e.__cause__ == config_error + assert e.model.context["transport_type"] == EnumInfraTransportType.DATABASE + + +class TestSecretResolutionError: + """Tests for SecretResolutionError.""" + + def test_basic_instantiation(self) -> None: + """Test basic error instantiation.""" + error = SecretResolutionError("Failed to resolve secret") + assert "Failed to resolve secret" in str(error) + assert isinstance(error, RuntimeHostError) + + def test_with_context_model(self) -> None: + """Test error with context model and extra context.""" + context = ModelInfraErrorContext( + target_name="vault", + operation="get_secret", + ) + error = SecretResolutionError( + "Secret not found", + context=context, + secret_key="db_password", # noqa: S106 + ) + assert error.model.context["target_name"] == "vault" + assert error.model.context["operation"] == "get_secret" + assert error.model.context["secret_key"] == "db_password" + + def test_error_code_mapping(self) -> None: + """Test that error uses appropriate CoreErrorCode.""" + error = SecretResolutionError("Secret error") + assert error.model.error_code == EnumCoreErrorCode.RESOURCE_NOT_FOUND + + def test_error_chaining(self) -> None: + """Test error chaining from vault client error.""" + context = ModelInfraErrorContext(target_name="vault") + vault_error = ConnectionError("Vault unreachable") + try: + raise SecretResolutionError( + "Cannot resolve secret", context=context + ) from vault_error + except SecretResolutionError as e: + assert e.__cause__ == vault_error + assert e.model.context["target_name"] == "vault" + + +class TestInfraConnectionError: + """Tests for InfraConnectionError.""" + + def test_basic_instantiation(self) -> None: + """Test basic error instantiation.""" + error = InfraConnectionError("Connection failed") + assert "Connection failed" in str(error) + assert isinstance(error, RuntimeHostError) + + def test_with_context_model(self) -> None: + """Test error with context model and connection details.""" + context = ModelInfraErrorContext(target_name="postgresql") + error = InfraConnectionError( + "Database connection failed", + context=context, + host="db.example.com", + port=5432, + ) + assert error.model.context["target_name"] == "postgresql" + assert error.model.context["host"] == "db.example.com" + assert error.model.context["port"] == 5432 + + def test_error_code_mapping_without_context(self) -> None: + """Test that error uses SERVICE_UNAVAILABLE when no context provided.""" + error = InfraConnectionError("Connection error") + # Without context, defaults to SERVICE_UNAVAILABLE + assert error.model.error_code == EnumCoreErrorCode.SERVICE_UNAVAILABLE + + def test_error_code_mapping_database_transport(self) -> None: + """Test DATABASE transport uses DATABASE_CONNECTION_ERROR.""" + context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.DATABASE, + target_name="postgresql", + ) + error = InfraConnectionError("Database connection failed", context=context) + assert error.model.error_code == EnumCoreErrorCode.DATABASE_CONNECTION_ERROR + + def test_error_code_mapping_http_transport(self) -> None: + """Test HTTP transport uses NETWORK_ERROR.""" + context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.HTTP, + target_name="api-gateway", + ) + error = InfraConnectionError("HTTP connection failed", context=context) + assert error.model.error_code == EnumCoreErrorCode.NETWORK_ERROR + + def test_error_code_mapping_grpc_transport(self) -> None: + """Test GRPC transport uses NETWORK_ERROR.""" + context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.GRPC, + target_name="grpc-service", + ) + error = InfraConnectionError("gRPC connection failed", context=context) + assert error.model.error_code == EnumCoreErrorCode.NETWORK_ERROR + + def test_error_code_mapping_kafka_transport(self) -> None: + """Test KAFKA transport uses SERVICE_UNAVAILABLE.""" + context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.KAFKA, + target_name="kafka-broker", + ) + error = InfraConnectionError("Kafka connection failed", context=context) + assert error.model.error_code == EnumCoreErrorCode.SERVICE_UNAVAILABLE + + def test_error_code_mapping_consul_transport(self) -> None: + """Test CONSUL transport uses SERVICE_UNAVAILABLE.""" + context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.CONSUL, + target_name="consul-server", + ) + error = InfraConnectionError("Consul connection failed", context=context) + assert error.model.error_code == EnumCoreErrorCode.SERVICE_UNAVAILABLE + + def test_error_code_mapping_vault_transport(self) -> None: + """Test VAULT transport uses SERVICE_UNAVAILABLE.""" + context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.VAULT, + target_name="vault-server", + ) + error = InfraConnectionError("Vault connection failed", context=context) + assert error.model.error_code == EnumCoreErrorCode.SERVICE_UNAVAILABLE + + def test_error_code_mapping_redis_transport(self) -> None: + """Test REDIS transport uses SERVICE_UNAVAILABLE.""" + context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.REDIS, + target_name="redis-cluster", + ) + error = InfraConnectionError("Redis connection failed", context=context) + assert error.model.error_code == EnumCoreErrorCode.SERVICE_UNAVAILABLE + + def test_error_code_mapping_context_without_transport(self) -> None: + """Test context with no transport_type uses SERVICE_UNAVAILABLE.""" + context = ModelInfraErrorContext( + operation="connect", + target_name="unknown-service", + ) + error = InfraConnectionError("Connection failed", context=context) + assert error.model.error_code == EnumCoreErrorCode.SERVICE_UNAVAILABLE + + def test_error_chaining(self) -> None: + """Test error chaining from connection exception.""" + context = ModelInfraErrorContext(target_name="redis") + conn_error = OSError("Connection refused") + try: + raise InfraConnectionError( + "Failed to connect", context=context, host="localhost", port=6379 + ) from conn_error + except InfraConnectionError as e: + assert e.__cause__ == conn_error + assert e.model.context["target_name"] == "redis" + assert e.model.context["port"] == 6379 + + +class TestInfraConnectionErrorTransportMapping: + """Comprehensive tests for InfraConnectionError transport-aware error code mapping. + + Validates that InfraConnectionError selects the correct EnumCoreErrorCode + based on the transport_type in ModelInfraErrorContext. + """ + + def test_resolve_connection_error_code_with_none_context(self) -> None: + """Test _resolve_connection_error_code with None context.""" + error_code = InfraConnectionError._resolve_connection_error_code(None) + assert error_code == EnumCoreErrorCode.SERVICE_UNAVAILABLE + + def test_resolve_connection_error_code_database(self) -> None: + """Test _resolve_connection_error_code for DATABASE transport.""" + context = ModelInfraErrorContext(transport_type=EnumInfraTransportType.DATABASE) + error_code = InfraConnectionError._resolve_connection_error_code(context) + assert error_code == EnumCoreErrorCode.DATABASE_CONNECTION_ERROR + + def test_resolve_connection_error_code_network_transports(self) -> None: + """Test _resolve_connection_error_code for network transports (HTTP, GRPC).""" + for transport in [EnumInfraTransportType.HTTP, EnumInfraTransportType.GRPC]: + context = ModelInfraErrorContext(transport_type=transport) + error_code = InfraConnectionError._resolve_connection_error_code(context) + assert ( + error_code == EnumCoreErrorCode.NETWORK_ERROR + ), f"Expected NETWORK_ERROR for {transport}, got {error_code}" + + def test_resolve_connection_error_code_service_transports(self) -> None: + """Test _resolve_connection_error_code for service transports.""" + service_transports = [ + EnumInfraTransportType.KAFKA, + EnumInfraTransportType.CONSUL, + EnumInfraTransportType.VAULT, + EnumInfraTransportType.REDIS, + ] + for transport in service_transports: + context = ModelInfraErrorContext(transport_type=transport) + error_code = InfraConnectionError._resolve_connection_error_code(context) + assert ( + error_code == EnumCoreErrorCode.SERVICE_UNAVAILABLE + ), f"Expected SERVICE_UNAVAILABLE for {transport}, got {error_code}" + + def test_all_transport_types_have_mapping(self) -> None: + """Test that all EnumInfraTransportType values have error code mappings.""" + for transport in EnumInfraTransportType: + context = ModelInfraErrorContext(transport_type=transport) + # Should not raise and should return a valid error code + error_code = InfraConnectionError._resolve_connection_error_code(context) + assert isinstance( + error_code, EnumCoreErrorCode + ), f"Transport {transport} returned invalid error code type: {type(error_code)}" + + def test_transport_error_code_map_completeness(self) -> None: + """Test that the transport error code map includes all transport types.""" + for transport in EnumInfraTransportType: + assert ( + transport in InfraConnectionError._TRANSPORT_ERROR_CODE_MAP + ), f"Transport {transport} missing from _TRANSPORT_ERROR_CODE_MAP" + # Also verify None is in the map + assert None in InfraConnectionError._TRANSPORT_ERROR_CODE_MAP + + def test_error_code_preserved_in_model(self) -> None: + """Test that resolved error code is correctly stored in the error model.""" + test_cases = [ + ( + EnumInfraTransportType.DATABASE, + EnumCoreErrorCode.DATABASE_CONNECTION_ERROR, + ), + (EnumInfraTransportType.HTTP, EnumCoreErrorCode.NETWORK_ERROR), + (EnumInfraTransportType.GRPC, EnumCoreErrorCode.NETWORK_ERROR), + (EnumInfraTransportType.KAFKA, EnumCoreErrorCode.SERVICE_UNAVAILABLE), + (EnumInfraTransportType.CONSUL, EnumCoreErrorCode.SERVICE_UNAVAILABLE), + (EnumInfraTransportType.VAULT, EnumCoreErrorCode.SERVICE_UNAVAILABLE), + (EnumInfraTransportType.REDIS, EnumCoreErrorCode.SERVICE_UNAVAILABLE), + ] + for transport, expected_code in test_cases: + context = ModelInfraErrorContext(transport_type=transport) + error = InfraConnectionError("Test error", context=context) + assert ( + error.model.error_code == expected_code + ), f"Transport {transport}: expected {expected_code}, got {error.model.error_code}" + + +class TestInfraTimeoutError: + """Tests for InfraTimeoutError.""" + + def test_basic_instantiation(self) -> None: + """Test basic error instantiation.""" + error = InfraTimeoutError("Operation timed out") + assert "Operation timed out" in str(error) + assert isinstance(error, RuntimeHostError) + + def test_with_context_model(self) -> None: + """Test error with context model and timeout details.""" + context = ModelInfraErrorContext( + operation="execute_query", + target_name="postgresql", + ) + error = InfraTimeoutError( + "Query timeout exceeded", + context=context, + timeout_seconds=30, + ) + assert error.model.context["operation"] == "execute_query" + assert error.model.context["timeout_seconds"] == 30 + assert error.model.context["target_name"] == "postgresql" + + def test_error_code_mapping(self) -> None: + """Test that error uses appropriate CoreErrorCode.""" + error = InfraTimeoutError("Timeout error") + assert error.model.error_code == EnumCoreErrorCode.TIMEOUT_ERROR + + def test_error_chaining(self) -> None: + """Test error chaining from timeout exception.""" + context = ModelInfraErrorContext(operation="select") + timeout = TimeoutError("Operation exceeded deadline") + try: + raise InfraTimeoutError( + "Database query timeout", context=context, timeout_seconds=10 + ) from timeout + except InfraTimeoutError as e: + assert e.__cause__ == timeout + assert e.model.context["operation"] == "select" + assert e.model.context["timeout_seconds"] == 10 + + +class TestInfraAuthenticationError: + """Tests for InfraAuthenticationError.""" + + def test_basic_instantiation(self) -> None: + """Test basic error instantiation.""" + error = InfraAuthenticationError("Authentication failed") + assert "Authentication failed" in str(error) + assert isinstance(error, RuntimeHostError) + + def test_with_context_model(self) -> None: + """Test error with context model and auth details.""" + context = ModelInfraErrorContext( + target_name="consul", + operation="authenticate", + ) + error = InfraAuthenticationError( + "Invalid credentials", + context=context, + username="admin", + ) + assert error.model.context["target_name"] == "consul" + assert error.model.context["operation"] == "authenticate" + assert error.model.context["username"] == "admin" + + def test_error_code_mapping(self) -> None: + """Test that error uses appropriate CoreErrorCode.""" + error = InfraAuthenticationError("Auth error") + assert error.model.error_code == EnumCoreErrorCode.AUTHENTICATION_ERROR + + def test_error_chaining(self) -> None: + """Test error chaining from auth exception.""" + context = ModelInfraErrorContext( + target_name="vault", + operation="login", + ) + auth_error = PermissionError("Access denied") + try: + raise InfraAuthenticationError( + "Vault authentication failed", context=context + ) from auth_error + except InfraAuthenticationError as e: + assert e.__cause__ == auth_error + assert e.model.context["target_name"] == "vault" + + +class TestInfraUnavailableError: + """Tests for InfraUnavailableError.""" + + def test_basic_instantiation(self) -> None: + """Test basic error instantiation.""" + error = InfraUnavailableError("Resource unavailable") + assert "Resource unavailable" in str(error) + assert isinstance(error, RuntimeHostError) + + def test_with_context_model(self) -> None: + """Test error with context model and details.""" + context = ModelInfraErrorContext(target_name="kafka") + error = InfraUnavailableError( + "Kafka broker unavailable", + context=context, + host="kafka.example.com", + port=9092, + retry_count=3, + ) + assert error.model.context["target_name"] == "kafka" + assert error.model.context["host"] == "kafka.example.com" + assert error.model.context["port"] == 9092 + assert error.model.context["retry_count"] == 3 + + def test_error_code_mapping(self) -> None: + """Test that error uses appropriate CoreErrorCode.""" + error = InfraUnavailableError("Resource error") + assert error.model.error_code == EnumCoreErrorCode.SERVICE_UNAVAILABLE + + def test_error_chaining(self) -> None: + """Test error chaining from exception.""" + context = ModelInfraErrorContext(target_name="consul") + resource_error = ConnectionRefusedError("Not responding") + try: + raise InfraUnavailableError( + "Consul unavailable", + context=context, + host="consul.local", + port=8500, + ) from resource_error + except InfraUnavailableError as e: + assert e.__cause__ == resource_error + assert e.model.context["target_name"] == "consul" + assert e.model.context["port"] == 8500 + + +class TestAllErrorsInheritance: + """Test that all infrastructure errors properly inherit from RuntimeHostError.""" + + def test_all_errors_inherit_from_runtime_host_error(self) -> None: + """Test inheritance chain for all error classes.""" + errors = [ + ProtocolConfigurationError("test"), + SecretResolutionError("test"), + InfraConnectionError("test"), + InfraTimeoutError("test"), + InfraAuthenticationError("test"), + InfraUnavailableError("test"), + ] + + for error in errors: + assert isinstance(error, RuntimeHostError) + assert isinstance(error, ModelOnexError) + assert isinstance(error, Exception) + + +class TestStructuredFieldsComprehensive: + """Comprehensive tests for structured field support across all errors.""" + + def test_all_errors_support_correlation_id(self) -> None: + """Test that all errors support correlation_id via context model.""" + correlation_id = uuid4() + context = ModelInfraErrorContext(correlation_id=correlation_id) + errors = [ + ProtocolConfigurationError("test", context=context), + SecretResolutionError("test", context=context), + InfraConnectionError("test", context=context), + InfraTimeoutError("test", context=context), + InfraAuthenticationError("test", context=context), + InfraUnavailableError("test", context=context), + ] + + for error in errors: + assert error.model.correlation_id == correlation_id + + def test_all_errors_support_transport_type(self) -> None: + """Test that all errors support transport_type via context model.""" + transport_types = [ + EnumInfraTransportType.HTTP, + EnumInfraTransportType.VAULT, + EnumInfraTransportType.DATABASE, + EnumInfraTransportType.KAFKA, + EnumInfraTransportType.CONSUL, + EnumInfraTransportType.REDIS, + ] + errors = [ + ProtocolConfigurationError( + "test", + context=ModelInfraErrorContext( + transport_type=EnumInfraTransportType.HTTP + ), + ), + SecretResolutionError( + "test", + context=ModelInfraErrorContext( + transport_type=EnumInfraTransportType.VAULT + ), + ), + InfraConnectionError( + "test", + context=ModelInfraErrorContext( + transport_type=EnumInfraTransportType.DATABASE + ), + ), + InfraTimeoutError( + "test", + context=ModelInfraErrorContext( + transport_type=EnumInfraTransportType.KAFKA + ), + ), + InfraAuthenticationError( + "test", + context=ModelInfraErrorContext( + transport_type=EnumInfraTransportType.CONSUL + ), + ), + InfraUnavailableError( + "test", + context=ModelInfraErrorContext( + transport_type=EnumInfraTransportType.REDIS + ), + ), + ] + + for error, expected_type in zip(errors, transport_types, strict=True): + assert error.model.context["transport_type"] == expected_type + + def test_all_errors_support_operation(self) -> None: + """Test that all errors support operation via context model.""" + operations = [ + "validate", + "resolve", + "connect", + "execute", + "authenticate", + "check_health", + ] + errors = [ + ProtocolConfigurationError( + "test", context=ModelInfraErrorContext(operation="validate") + ), + SecretResolutionError( + "test", context=ModelInfraErrorContext(operation="resolve") + ), + InfraConnectionError( + "test", context=ModelInfraErrorContext(operation="connect") + ), + InfraTimeoutError( + "test", context=ModelInfraErrorContext(operation="execute") + ), + InfraAuthenticationError( + "test", context=ModelInfraErrorContext(operation="authenticate") + ), + InfraUnavailableError( + "test", context=ModelInfraErrorContext(operation="check_health") + ), + ] + + for error, operation in zip(errors, operations, strict=True): + assert error.model.context["operation"] == operation + + def test_all_errors_support_target_name(self) -> None: + """Test that all errors support target_name via context model.""" + targets = ["api", "vault", "postgresql", "kafka", "consul", "redis"] + errors = [ + ProtocolConfigurationError( + "test", context=ModelInfraErrorContext(target_name="api") + ), + SecretResolutionError( + "test", context=ModelInfraErrorContext(target_name="vault") + ), + InfraConnectionError( + "test", context=ModelInfraErrorContext(target_name="postgresql") + ), + InfraTimeoutError( + "test", context=ModelInfraErrorContext(target_name="kafka") + ), + InfraAuthenticationError( + "test", context=ModelInfraErrorContext(target_name="consul") + ), + InfraUnavailableError( + "test", context=ModelInfraErrorContext(target_name="redis") + ), + ] + + for error, target in zip(errors, targets, strict=True): + assert error.model.context["target_name"] == target + + +class TestErrorChaining: + """Test error chaining across all infrastructure error classes. + + Validates that the `raise ... from e` pattern properly chains exceptions + and preserves the original error as __cause__ for all error classes. + """ + + def test_runtime_host_error_chaining_preserves_cause(self) -> None: + """Test RuntimeHostError properly chains and preserves original exception.""" + original = ValueError("Original value error") + try: + try: + raise original + except ValueError as e: + raise RuntimeHostError("Wrapped error") from e + except RuntimeHostError as wrapped: + assert wrapped.__cause__ is original + assert isinstance(wrapped.__cause__, ValueError) + assert str(wrapped.__cause__) == "Original value error" + + def test_protocol_configuration_error_chaining_preserves_cause(self) -> None: + """Test ProtocolConfigurationError properly chains and preserves original exception.""" + original = KeyError("missing_config_key") + try: + try: + raise original + except KeyError as e: + raise ProtocolConfigurationError("Configuration error") from e + except ProtocolConfigurationError as wrapped: + assert wrapped.__cause__ is original + assert isinstance(wrapped.__cause__, KeyError) + assert "missing_config_key" in str(wrapped.__cause__) + + def test_secret_resolution_error_chaining_preserves_cause(self) -> None: + """Test SecretResolutionError properly chains and preserves original exception.""" + original = ConnectionError("Vault connection failed") + try: + try: + raise original + except ConnectionError as e: + raise SecretResolutionError("Cannot resolve secret") from e + except SecretResolutionError as wrapped: + assert wrapped.__cause__ is original + assert isinstance(wrapped.__cause__, ConnectionError) + assert "Vault connection failed" in str(wrapped.__cause__) + + def test_infra_connection_error_chaining_preserves_cause(self) -> None: + """Test InfraConnectionError properly chains and preserves original exception.""" + original = OSError("Connection refused") + try: + try: + raise original + except OSError as e: + raise InfraConnectionError("Database connection failed") from e + except InfraConnectionError as wrapped: + assert wrapped.__cause__ is original + assert isinstance(wrapped.__cause__, OSError) + assert "Connection refused" in str(wrapped.__cause__) + + def test_infra_timeout_error_chaining_preserves_cause(self) -> None: + """Test InfraTimeoutError properly chains and preserves original exception.""" + original = TimeoutError("Operation timed out after 30s") + try: + try: + raise original + except TimeoutError as e: + raise InfraTimeoutError("Query timeout") from e + except InfraTimeoutError as wrapped: + assert wrapped.__cause__ is original + assert isinstance(wrapped.__cause__, TimeoutError) + assert "30s" in str(wrapped.__cause__) + + def test_infra_authentication_error_chaining_preserves_cause(self) -> None: + """Test InfraAuthenticationError properly chains and preserves original exception.""" + original = PermissionError("Access denied") + try: + try: + raise original + except PermissionError as e: + raise InfraAuthenticationError("Authentication failed") from e + except InfraAuthenticationError as wrapped: + assert wrapped.__cause__ is original + assert isinstance(wrapped.__cause__, PermissionError) + assert "Access denied" in str(wrapped.__cause__) + + def test_infra_unavailable_error_chaining_preserves_cause(self) -> None: + """Test InfraUnavailableError properly chains and preserves original exception.""" + original = ConnectionRefusedError("Service not responding") + try: + try: + raise original + except ConnectionRefusedError as e: + raise InfraUnavailableError("Resource unavailable") from e + except InfraUnavailableError as wrapped: + assert wrapped.__cause__ is original + assert isinstance(wrapped.__cause__, ConnectionRefusedError) + assert "Service not responding" in str(wrapped.__cause__) + + def test_chained_error_with_context_preserved(self) -> None: + """Test that context is preserved when chaining errors.""" + correlation_id = uuid4() + context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.DATABASE, + operation="execute_query", + target_name="postgresql", + correlation_id=correlation_id, + ) + original = TimeoutError("Query exceeded deadline") + try: + try: + raise original + except TimeoutError as e: + raise InfraTimeoutError( + "Database query timeout", + context=context, + timeout_seconds=30, + ) from e + except InfraTimeoutError as wrapped: + # Verify chaining + assert wrapped.__cause__ is original + # Verify context preserved + assert wrapped.model.correlation_id == correlation_id + assert ( + wrapped.model.context["transport_type"] + == EnumInfraTransportType.DATABASE + ) + assert wrapped.model.context["operation"] == "execute_query" + assert wrapped.model.context["target_name"] == "postgresql" + assert wrapped.model.context["timeout_seconds"] == 30 + + def test_multi_level_chaining(self) -> None: + """Test error chaining through multiple levels.""" + root_error = OSError("Network unreachable") + try: + try: + try: + raise root_error + except OSError as e: + raise InfraConnectionError("Connection layer error") from e + except InfraConnectionError as e: + raise InfraUnavailableError("Service unavailable") from e + except InfraUnavailableError as final: + # Verify immediate cause + assert isinstance(final.__cause__, InfraConnectionError) + # Verify root cause through chain + assert isinstance(final.__cause__.__cause__, OSError) + assert final.__cause__.__cause__ is root_error + + def test_correlation_id_propagates_through_chain(self) -> None: + """Test correlation_id preserved through multi-level error chaining.""" + correlation_id = uuid4() + context = ModelInfraErrorContext(correlation_id=correlation_id) + + try: + try: + raise InfraConnectionError("Connection failed", context=context) + except InfraConnectionError as e: + # Correlation ID should propagate + new_context = ModelInfraErrorContext( + correlation_id=e.model.correlation_id + ) + raise InfraUnavailableError("Service down", context=new_context) from e + except InfraUnavailableError as final: + # Same correlation ID throughout the chain + assert final.model.correlation_id == correlation_id + assert final.__cause__ is not None + assert isinstance(final.__cause__, InfraConnectionError) + + +class TestContextSerialization: + """Test ModelInfraErrorContext serialization and deserialization. + + Validates that the context model correctly serializes to dict and JSON, + handles UUID and enum fields properly, and supports roundtrip serialization. + """ + + def test_context_to_dict_empty(self) -> None: + """Test serialization of empty context to dict.""" + context = ModelInfraErrorContext() + data = context.model_dump() + assert data == { + "transport_type": None, + "operation": None, + "target_name": None, + "correlation_id": None, + } + + def test_context_to_dict_with_all_fields(self) -> None: + """Test serialization of fully populated context to dict.""" + correlation_id = uuid4() + context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.KAFKA, + operation="produce_message", + target_name="events-topic", + correlation_id=correlation_id, + ) + data = context.model_dump() + assert data["transport_type"] == EnumInfraTransportType.KAFKA + assert data["operation"] == "produce_message" + assert data["target_name"] == "events-topic" + assert data["correlation_id"] == correlation_id + + def test_context_to_dict_mode_json(self) -> None: + """Test serialization with mode='json' for JSON-compatible output.""" + correlation_id = uuid4() + context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.HTTP, + operation="request", + target_name="api-endpoint", + correlation_id=correlation_id, + ) + data = context.model_dump(mode="json") + # Enum should be serialized as string value + assert data["transport_type"] == "http" + # UUID should be serialized as string + assert data["correlation_id"] == str(correlation_id) + assert data["operation"] == "request" + assert data["target_name"] == "api-endpoint" + + def test_context_to_json_string(self) -> None: + """Test serialization to JSON string.""" + import json + + correlation_id = uuid4() + context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.DATABASE, + operation="connect", + target_name="postgresql", + correlation_id=correlation_id, + ) + json_str = context.model_dump_json() + # Verify valid JSON + parsed = json.loads(json_str) + # DATABASE enum value is "db" + assert parsed["transport_type"] == "db" + assert parsed["operation"] == "connect" + assert parsed["target_name"] == "postgresql" + assert parsed["correlation_id"] == str(correlation_id) + + def test_context_roundtrip_serialization(self) -> None: + """Test roundtrip serialization: model -> dict -> model.""" + correlation_id = uuid4() + original = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.VAULT, + operation="get_secret", + target_name="secrets/database", + correlation_id=correlation_id, + ) + # Serialize to dict + data = original.model_dump() + # Deserialize back to model + restored = ModelInfraErrorContext(**data) + # Verify equality + assert restored.transport_type == original.transport_type + assert restored.operation == original.operation + assert restored.target_name == original.target_name + assert restored.correlation_id == original.correlation_id + + def test_context_roundtrip_via_json(self) -> None: + """Test roundtrip serialization via JSON string.""" + import json + + correlation_id = uuid4() + original = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.CONSUL, + operation="register_service", + target_name="my-service", + correlation_id=correlation_id, + ) + # Serialize to JSON string + json_str = original.model_dump_json() + # Parse JSON + data = json.loads(json_str) + # Deserialize back to model + restored = ModelInfraErrorContext.model_validate(data) + # Verify equality + assert restored.transport_type == original.transport_type + assert restored.operation == original.operation + assert restored.target_name == original.target_name + assert restored.correlation_id == original.correlation_id + + def test_context_uuid_field_serialization(self) -> None: + """Test that UUID fields serialize and deserialize correctly.""" + from uuid import UUID + + correlation_id = uuid4() + context = ModelInfraErrorContext(correlation_id=correlation_id) + + # Verify internal type is UUID + assert isinstance(context.correlation_id, UUID) + + # Serialize with mode='json' converts to string + json_data = context.model_dump(mode="json") + assert isinstance(json_data["correlation_id"], str) + assert json_data["correlation_id"] == str(correlation_id) + + # Standard dump preserves UUID type + data = context.model_dump() + assert isinstance(data["correlation_id"], UUID) + assert data["correlation_id"] == correlation_id + + def test_context_enum_field_serialization(self) -> None: + """Test that enum fields serialize and deserialize correctly.""" + context = ModelInfraErrorContext(transport_type=EnumInfraTransportType.REDIS) + + # Verify internal type is enum + assert isinstance(context.transport_type, EnumInfraTransportType) + + # Serialize with mode='json' converts to string value + json_data = context.model_dump(mode="json") + assert isinstance(json_data["transport_type"], str) + assert json_data["transport_type"] == "redis" + + # Standard dump preserves enum type + data = context.model_dump() + assert isinstance(data["transport_type"], EnumInfraTransportType) + assert data["transport_type"] == EnumInfraTransportType.REDIS + + def test_context_none_fields_in_serialization(self) -> None: + """Test that None fields are properly handled in serialization.""" + context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.HTTP, + operation=None, + target_name="endpoint", + correlation_id=None, + ) + data = context.model_dump() + assert data["transport_type"] == EnumInfraTransportType.HTTP + assert data["operation"] is None + assert data["target_name"] == "endpoint" + assert data["correlation_id"] is None + + # JSON serialization + json_data = context.model_dump(mode="json") + assert json_data["operation"] is None + assert json_data["correlation_id"] is None + + def test_context_exclude_none_serialization(self) -> None: + """Test serialization with exclude_none option.""" + context = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.KAFKA, + operation="consume", + ) + data = context.model_dump(exclude_none=True) + assert "transport_type" in data + assert "operation" in data + assert "target_name" not in data + assert "correlation_id" not in data + + def test_context_all_transport_types_serialize(self) -> None: + """Test that all transport types serialize correctly.""" + transport_types = [ + EnumInfraTransportType.HTTP, + EnumInfraTransportType.VAULT, + EnumInfraTransportType.DATABASE, + EnumInfraTransportType.KAFKA, + EnumInfraTransportType.CONSUL, + EnumInfraTransportType.REDIS, + ] + for transport in transport_types: + context = ModelInfraErrorContext(transport_type=transport) + # Standard serialization + data = context.model_dump() + assert data["transport_type"] == transport + # JSON-mode serialization + json_data = context.model_dump(mode="json") + assert json_data["transport_type"] == transport.value + # Roundtrip + restored = ModelInfraErrorContext.model_validate(data) + assert restored.transport_type == transport