diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 55abbf2ddc..c53dd445a3 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -68,7 +68,10 @@ "mcp__linear-server__get_team", "mcp__linear-server__create_issue", "mcp__linear-server__get_issue", - "Bash(git init:*)" + "Bash(git init:*)", + "Bash(poetry run pytest:*)", + "Bash(poetry run python:*)", + "Bash(poetry add:*)" ], "deny": [], "ask": [] diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 696545931f..d1aa088b11 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ on: env: POETRY_VERSION: "2.2.1" PYTHON_VERSION: "3.12" - CACHE_VERSION: "0.1.0" + CACHE_VERSION: "0.2.0" jobs: # Quick smoke test - fails fast on basic issues diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..7b185f3ff5 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,139 @@ +# Changelog + +All notable changes to the ONEX Infrastructure (omnibase_infra) will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added + +#### Handlers +- **HttpHandler** (OMN-237, PR #26): HTTP REST protocol handler for MVP + - GET and POST operations using httpx async client + - Fixed 30s timeout (configurable timeout deferred to Beta) + - Returns `EnumHandlerType.HTTP` + - Error handling mapping to infrastructure errors (`InfraTimeoutError`, `InfraConnectionError`) + - Full lifecycle support (initialize, shutdown, health_check, describe) + - 46 unit tests with 97.93% coverage + +#### Event Bus +- **InMemoryEventBus** (OMN-239, PR #25): In-memory event bus for local development and testing + - Implements `ProtocolEventBus` from omnibase_core + - Topic-based pub/sub with `asyncio.Queue` per topic + - Thread-safe subscription management + - Automatic cleanup on unsubscribe + - Consumer groups with load balancing + - Graceful shutdown with message draining + - Comprehensive error handling + - 1336+ lines of test coverage + +#### Runtime +- **ProtocolBindingRegistry** (OMN-240, PR #24): Handler and event bus registration system + - Single source of truth for handler registration + - Thread-safe registration operations + - Support for handler type constants (HTTP, DATABASE, KAFKA, etc.) + - Event bus registry (InMemory, Kafka) + - Protocol resolution utilities + +#### Errors +- **Infrastructure Error Taxonomy** (OMN-290, PR #23): Structured error hierarchy + - `RuntimeHostError`: Base infrastructure error class + - `ProtocolConfigurationError`: Protocol configuration validation errors + - `SecretResolutionError`: Secret/credential resolution errors + - `InfraConnectionError`: Infrastructure connection errors (transport-aware) + - `InfraTimeoutError`: Infrastructure timeout errors + - `InfraAuthenticationError`: Infrastructure authentication errors + - `InfraUnavailableError`: Infrastructure resource unavailable errors + - `ModelInfraErrorContext`: Structured error context model + - `EnumInfraTransportType`: Transport type classification + +#### Infrastructure +- **Directory Structure** (OMN-236, PR #21): Initial MVP directory structure + - `handlers/`: Protocol handler implementations + - `event_bus/`: Event bus implementations + - `runtime/`: Runtime host components + - `errors/`: Infrastructure error classes + - `enums/`: Infrastructure enumerations + - `validation/`: Contract validation utilities + +### Changed + +#### CI/CD +- **Pre-commit Configuration**: Migrated to fix deprecated stage warnings (PR #25) + +## [0.1.0] - Unreleased + +### Planned + +This version represents the MVP (Minimum Viable Product) for ONEX Runtime Host Infrastructure. + +#### Core Components (MVP) +- **BaseRuntimeHostProcess** (OMN-249): Infrastructure wrapper owning event bus and driving NodeRuntime +- **DbHandler** (OMN-238): PostgreSQL database protocol handler +- **wiring.py** (OMN-240): Handler registration module ✅ (implemented as ProtocolBindingRegistry) + +#### Testing (MVP) +- **E2E Flow Test** (OMN-254): InMemoryEventBus -> Runtime -> Handler flow +- **Architecture Verification** (OMN-255): Architectural invariant checks + +#### Deployment (MVP) +- **Dockerfile** (OMN-256): Basic container image for runtime host +- **docker-compose** (OMN-264): Local development configuration + +#### Documentation (MVP) +- **CLAUDE.md Updates** (OMN-265): Runtime Host architecture documentation + +### MVP Philosophy + +> **MVP (v0.1.0)**: Prove the architecture works end-to-end with minimal scope +> - InMemoryEventBus only (no Kafka complexity) +> - HTTP + DB handlers only (no Vault, no Consul) +> - Simplified contract format +> - Basic error handling +> - Unit tests with mocks + +### Deferred to Beta (v0.2.0) + +- KafkaEventBus with backpressure +- VaultHandler and ConsulHandler +- Retry logic and rate limiting +- Full graceful shutdown with drain +- Integration tests with real services +- Observability layer (structured logging, metrics) + +--- + +## Architecture + +``` +omnibase_infra (YOU ARE HERE) + ├── handlers/ # Protocol handler implementations + │ ├── http_handler # HTTP REST handler (MVP) + │ └── db_handler # PostgreSQL handler (MVP) + ├── event_bus/ # Event bus implementations + │ ├── inmemory # InMemory bus (MVP) + │ └── kafka # Kafka bus (Beta) + ├── runtime/ # Runtime host components + │ ├── handler_registry + │ └── runtime_host_process + └── errors/ # Infrastructure errors + └── infra_errors + +DEPENDENCY RULE: infra -> spi -> core (never reverse) +``` + +## Contributing + +See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines. + +### ONEX Standards +- Zero tolerance for `Any` types +- Contract-driven development +- Protocol-based dependency injection +- Comprehensive test coverage (>80% target) + +## License + +MIT License - See LICENSE file for details diff --git a/docs/architecture/CURRENT_NODE_ARCHITECTURE.md b/docs/architecture/CURRENT_NODE_ARCHITECTURE.md new file mode 100644 index 0000000000..b744c793ea --- /dev/null +++ b/docs/architecture/CURRENT_NODE_ARCHITECTURE.md @@ -0,0 +1,927 @@ +# ONEX Current Node Architecture (Pre-Runtime Host Migration) + +This document describes the current ONEX node architecture that uses a **1-container-per-node** deployment model. This is the "before" state that will be migrated to the new Runtime Host model. + +--- + +## 1. Overview + +### Current Architecture: 1 Container Per Node + +In the current ONEX architecture, each node runs as an **independent container** with its own: + +- Python runtime environment +- Entry point (`node.py` with `if __name__ == "__main__"`) +- Container injection setup +- Kafka consumer/producer connections +- Health check endpoint + +**Deployment Model:** +``` +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ Container 1 │ │ Container 2 │ │ Container 3 │ +│ ┌───────────┐ │ │ ┌───────────┐ │ │ ┌───────────┐ │ +│ │ VaultNode │ │ │ │ConsulNode │ │ │ │ KafkaNode │ │ +│ └───────────┘ │ │ └───────────┘ │ │ └───────────┘ │ +│ asyncio.run │ │ asyncio.run │ │ asyncio.run │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ + │ │ │ + └───────────────────┼─────────────────────┘ + │ + ┌──────▼──────┐ + │ Kafka │ + │ Event Bus │ + └─────────────┘ +``` + +### The 4 Node Types + +ONEX follows a strict **4-node architecture** pattern: + +| Node Type | Base Class | Purpose | I/O Operations | +|-----------|------------|---------|----------------| +| **EFFECT** | `NodeEffectService` | External I/O (APIs, DB, files) | Yes (`io_operations`) | +| **COMPUTE** | `NodeComputeService` | Pure transforms/algorithms | No | +| **REDUCER** | `NodeReducerService` | Aggregation/persistence | No (DB via adapters) | +| **ORCHESTRATOR** | `NodeOrchestratorService` | Workflow coordination | No | + +**Communication Pattern:** +``` +Adapters (EFFECT) → Events → Reducer → Intents → Orchestrator → Workflows → Adapters +``` + +--- + +## 2. Node Directory Structure + +### Standard Node Structure + +``` +nodes//v1_0_0/ +├── __init__.py # Package initialization +├── node.py # Main node implementation with entry point +├── contract.yaml # Node contract definition +├── models/ # Node-specific models +│ ├── __init__.py +│ ├── model__input.py +│ └── model__output.py +└── registry/ # Dependency injection registry (optional) + └── __init__.py +``` + +### Naming Conventions + +- **Directory Name**: `node__` (e.g., `node_vault_adapter_effect`) +- **Node Class**: `Node` in CamelCase (e.g., `NodeVaultAdapterEffect`) +- **Models**: `ModelInput`, `ModelOutput` +- **Files**: All snake_case (`model_vault_adapter_input.py`) + +--- + +## 3. Full Example: Effect Node (Vault Adapter) + +### File Tree + +``` +nodes/node_vault_adapter_effect/v1_0_0/ +├── __init__.py +├── node.py # 706 lines - Main implementation +├── contract.yaml # 177 lines - Contract definition +├── models/ +│ ├── __init__.py +│ ├── model_vault_adapter_input.py # Input model for envelope payloads +│ └── model_vault_adapter_output.py # Output model for results +└── registry/ + └── __init__.py +``` + +### node.py - Key Sections + +**Imports and Base Class:** +```python +#!/usr/bin/env python3 + +import asyncio +import logging +import os +from typing import Any + +from omnibase_core.core.errors.onex_error import CoreErrorCode, OnexError +from omnibase_core.core.node_effect_service import NodeEffectService +from omnibase_core.core.onex_container import ModelONEXContainer +from omnibase_core.enums.enum_health_status import EnumHealthStatus +from omnibase_core.models.core.model_health_status import ModelHealthStatus + +from omnibase_infra.models.vault import ( + ModelVaultSecretRequest, + ModelVaultSecretResponse, + ModelVaultTokenRequest, +) +``` + +**Node Class Definition:** +```python +class NodeVaultAdapterEffect(NodeEffectService): + """ + Vault Adapter - Event-Driven Secret Management Effect + + NodeEffect that processes event envelopes to perform Vault operations. + Integrates with event bus for secret management, token lifecycle, + and encryption services. Provides health check HTTP endpoint for monitoring. + """ + + def __init__(self, container: ModelONEXContainer): + # Use proper base class - no more boilerplate! + super().__init__(container) + + self.node_type = "effect" + self.domain = "infrastructure" + + # ONEX logger initialization with fallback + try: + self.logger = getattr(container, "get_tool", lambda x: None)( + "LOGGER", + ) or logging.getLogger(__name__) + except (AttributeError, Exception): + self.logger = logging.getLogger(__name__) + + # Vault client configuration - all environment variables required + vault_addr = os.getenv("VAULT_ADDR") + vault_token = os.getenv("VAULT_TOKEN") + vault_namespace = os.getenv("VAULT_NAMESPACE", "") + + if not vault_addr: + raise OnexError( + message="VAULT_ADDR environment variable is required but not set", + error_code=CoreErrorCode.MISSING_REQUIRED_PARAMETER, + ) + # ... configuration continues +``` + +**Resource Lifecycle Methods:** +```python + async def _initialize_node_resources(self) -> None: + """Override to initialize vault client.""" + await super()._initialize_node_resources() + await self.initialize_vault_client() + + async def _cleanup_node_resources(self) -> None: + """Override to cleanup vault connection pool resources.""" + if self.vault_connection_pool: + await self.vault_connection_pool.close_all() + await super()._cleanup_node_resources() +``` + +**Health Check Method:** +```python + def health_check(self) -> ModelHealthStatus: + """Check Vault service health and connectivity.""" + try: + client = self._get_vault_client() + + if client is None: + return ModelHealthStatus( + status=EnumHealthStatus.UNREACHABLE, + message="Vault client is not initialized", + ) + + health = client.sys.read_health_status(method="GET") + + if not health.get("initialized", False): + return ModelHealthStatus( + status=EnumHealthStatus.UNHEALTHY, + message="Vault is not initialized", + details=health, + ) + + if health.get("sealed", True): + return ModelHealthStatus( + status=EnumHealthStatus.UNHEALTHY, + message="Vault is sealed", + details=health, + ) + + return ModelHealthStatus( + status=EnumHealthStatus.HEALTHY, + message=f"Vault is healthy (version: {health.get('version', 'unknown')})", + details={...}, + ) + except Exception as e: + return ModelHealthStatus( + status=EnumHealthStatus.UNREACHABLE, + message=f"Vault health check failed: {str(e)}", + ) +``` + +**Entry Point (1-Container-Per-Node Pattern):** +```python +# Entry point for running the node +if __name__ == "__main__": + import sys + + # Create container (simplified for standalone operation) + container = ModelONEXContainer() + + # Create and run the node + node = NodeVaultAdapterEffect(container) + + # Run the node with asyncio + try: + asyncio.run(node.run()) + except KeyboardInterrupt: + print("\nVault adapter shutting down...") + sys.exit(0) +``` + +### contract.yaml - Full Structure + +```yaml +name: "vault_adapter" +contract_name: "vault_adapter" +node_name: "vault_adapter" +version: + major: 1 + minor: 0 + patch: 0 +contract_version: "1.0.0" +node_version: "1.0.0" + +node_type: "EFFECT" + +description: > + HashiCorp Vault secret management adapter for secure credential storage and retrieval. + Message bus bridge pattern for Vault operations including secret management, + token operations, and encryption services. + +capabilities: + - name: "secret_management" + description: "Read, write, delete, and list secrets in Vault" + - name: "token_management" + description: "Create, renew, revoke, and lookup Vault tokens" + - name: "encryption_services" + description: "Encrypt and decrypt data using Vault transit engine" + - name: "lease_management" + description: "Manage secret leases and renewals" + - name: "health_monitoring" + description: "Monitor Vault health and seal status" + +input_model: "ModelVaultAdapterInput" +output_model: "ModelVaultAdapterOutput" + +io_operations: + - operation: "get_secret" + description: "Retrieve secret from Vault" + input_fields: + - path + - version + - mount_path + output_fields: + - data + - metadata + - version + + - operation: "set_secret" + description: "Store secret in Vault" + input_fields: + - path + - data + - mount_path + output_fields: + - version + - created_time + + - operation: "delete_secret" + description: "Delete secret from Vault" + input_fields: + - path + - mount_path + output_fields: + - success + + - operation: "health_check" + description: "Check Vault health and seal status" + input_fields: [] + output_fields: + - initialized + - sealed + - standby + - version + +dependencies: + - name: "protocol_event_bus" + type: "protocol" + class_name: "ProtocolEventBus" + module: "omnibase_spi.protocols.event_bus" + + - name: "model_vault_secret_request" + type: "model" + class_name: "ModelVaultSecretRequest" + module: "omnibase_infra.models.vault.model_vault_secret_request" + + - name: "model_vault_secret_response" + type: "model" + class_name: "ModelVaultSecretResponse" + module: "omnibase_infra.models.vault.model_vault_secret_response" + +definitions: + ModelVaultAdapterInput: + type: object + description: "Input model for Vault adapter operations" + properties: + operation: + type: string + description: "Operation to perform (get_secret, set_secret, etc.)" + path: + type: string + description: "Vault secret path" + data: + type: object + description: "Secret data for write operations" + correlation_id: + type: string + description: "Request correlation ID" + required: + - operation + - correlation_id + + ModelVaultAdapterOutput: + type: object + description: "Output model for Vault adapter operations" + properties: + success: + type: boolean + description: "Whether operation succeeded" + data: + type: object + description: "Response data" + error: + type: string + description: "Error message if failed" + correlation_id: + type: string + description: "Request correlation ID" + required: + - success + - correlation_id + +metadata: + author: "ONEX Infrastructure Team" + created: "2025-11-14" + tags: + - vault + - secrets + - security + - adapter + - effect +``` + +### Model Files + +**model_vault_adapter_input.py:** +```python +#!/usr/bin/env python3 + +from typing import Literal +from pydantic import BaseModel, Field + + +class ModelVaultAdapterInput(BaseModel): + """Input model for Vault adapter operations from event envelopes. + + Node-specific model for processing event envelope payloads into Vault operations. + """ + + action: Literal[ + "vault_get_secret", + "vault_set_secret", + "vault_delete_secret", + "vault_list_secrets", + "vault_create_token", + "vault_renew_token", + "vault_revoke_token", + "vault_health_check", + ] = Field(description="Vault operation to perform") + + # Secret operation parameters + path: str | None = Field(default=None, description="Secret path in Vault") + mount_path: str = Field(default="secret", description="Vault mount path") + secret_data: dict | None = Field(default=None, description="Secret data for write operations") + version: int | None = Field(default=None, description="Secret version to retrieve") + + # Token operation parameters + token: str | None = Field(default=None, description="Token for renew/revoke operations") + policies: list[str] | None = Field(default=None, description="Policies for token creation") + ttl: str | None = Field(default=None, description="Token TTL (e.g., '768h')") + renewable: bool = Field(default=True, description="Whether token is renewable") + + # Common fields + correlation_id: str = Field(description="Correlation ID for request tracking") +``` + +**model_vault_adapter_output.py:** +```python +#!/usr/bin/env python3 + +from pydantic import BaseModel, Field +from omnibase_infra.models.vault.model_vault_secret_response import ( + ModelVaultSecretResponse, +) + + +class ModelVaultAdapterOutput(BaseModel): + """Output model for Vault adapter operation results. + + Node-specific model for returning Vault operation results through effect outputs. + """ + + vault_operation_result: ( + ModelVaultSecretResponse + | dict[str, str | int | bool | list | None] + | str + | bool + ) = Field(description="Result of Vault operation") + + success: bool = Field(description="Whether the operation succeeded") + operation_type: str = Field(description="Type of Vault operation performed") + correlation_id: str = Field(description="Correlation ID from request") +``` + +--- + +## 4. Full Example: Effect Node (Consul Projector) + +### File Tree + +``` +nodes/node_consul_projector_effect/v1_0_0/ +├── __init__.py +├── node.py # Main implementation +├── contract.yaml # 342 lines - Contract definition +├── models/ +│ ├── __init__.py +│ ├── model_consul_cache_entry.py +│ ├── model_consul_health_projection.py +│ ├── model_consul_kv_details.py +│ ├── model_consul_kv_projection.py +│ ├── model_consul_kv_summary.py +│ ├── model_consul_projection_type.py +│ ├── model_consul_projections.py +│ ├── model_consul_projector_input.py +│ ├── model_consul_projector_output.py +│ ├── model_consul_service_projection.py +│ ├── model_consul_topology_graph.py +│ ├── model_consul_topology_metrics.py +│ └── model_consul_topology_projection.py +└── registry/ + └── __init__.py +``` + +### node.py - Key Sections + +```python +#!/usr/bin/env python3 + +import asyncio +import logging +from datetime import UTC, datetime + +from omnibase_core.core.errors.onex_error import CoreErrorCode, OnexError +from omnibase_core.core.node_effect_service import NodeEffectService +from omnibase_core.core.onex_container import ModelONEXContainer +from omnibase_core.enums.enum_health_status import EnumHealthStatus +from omnibase_core.models.core.model_health_status import ModelHealthStatus + +# Import node-specific models +from .models import ( + ModelConsulHealthCacheEntry, + ModelConsulHealthProjection, + ModelConsulKVCacheEntry, + ModelConsulKVProjection, + ModelConsulProjectorInput, + ModelConsulProjectorOutput, + ModelConsulServiceCacheEntry, + ModelConsulServiceProjection, + ModelConsulTopologyProjection, +) + + +class NodeConsulProjectorEffect(NodeEffectService): + """ + Consul Projector - Event-Driven Infrastructure State Projector + + NodeEffect that processes Consul state data to create projected views and aggregations. + Integrates with event bus for event-driven state projection and monitoring. + Provides comprehensive state views for service discovery, health monitoring, and topology analysis. + """ + + def __init__(self, container: ModelONEXContainer): + super().__init__(container) + + self.node_type = "effect" + self.domain = "infrastructure" + + # ONEX logger initialization with fallback + try: + self.logger = getattr(container, "get_tool", lambda x: None)( + "LOGGER", + ) or logging.getLogger(__name__) + except (AttributeError, Exception): + self.logger = logging.getLogger(__name__) + + # State cache for projection optimization with strong typing + self._service_cache: dict[str, ModelConsulServiceCacheEntry] = {} + self._health_cache: dict[str, ModelConsulHealthCacheEntry] = {} + self._kv_cache: dict[str, ModelConsulKVCacheEntry] = {} + self._cache_ttl: int = 300 # 5 minutes + + self._initialized = False + + async def project_service_state(self, input_data: ModelConsulProjectorInput) -> ModelConsulServiceProjection: + """Project current service state from Consul data.""" + # ... projection logic + + async def project_health_state(self, input_data: ModelConsulProjectorInput) -> ModelConsulHealthProjection: + """Project health state aggregation from Consul data.""" + # ... projection logic + + async def project_kv_state(self, input_data: ModelConsulProjectorInput) -> ModelConsulKVProjection: + """Project KV store state changes from Consul data.""" + # ... projection logic + + async def project_topology(self, input_data: ModelConsulProjectorInput) -> ModelConsulTopologyProjection: + """Project service topology view from Consul data.""" + # ... topology generation logic + + def health_check(self) -> ModelHealthStatus: + """Single comprehensive health check for Consul projector.""" + try: + if not self._initialized: + return ModelHealthStatus( + status=EnumHealthStatus.UNHEALTHY, + message="Consul projector not initialized", + ) + + cache_health = len(self._service_cache) + len(self._health_cache) + len(self._kv_cache) + + if cache_health == 0: + return ModelHealthStatus( + status=EnumHealthStatus.DEGRADED, + message="Consul projector operational but caches empty", + ) + + return ModelHealthStatus( + status=EnumHealthStatus.HEALTHY, + message=f"Consul projector healthy - cache entries: {cache_health}", + ) + except Exception as e: + return ModelHealthStatus( + status=EnumHealthStatus.UNREACHABLE, + message=f"Consul projector health check failed: {e!s}", + ) + + +# Entry point for running the node +if __name__ == "__main__": + import sys + + container = ModelONEXContainer() + node = NodeConsulProjectorEffect(container) + + try: + asyncio.run(node.run()) + except KeyboardInterrupt: + print("\nConsul projector shutting down...") + sys.exit(0) +``` + +--- + +## 5. Contract YAML Structure + +### Required Fields + +| Field | Type | Description | +|-------|------|-------------| +| `name` | string | Short identifier for the node | +| `contract_name` | string | Full contract identifier | +| `node_name` | string | Node identifier | +| `version` | object | Semantic version `{major, minor, patch}` | +| `contract_version` | string | Contract version string | +| `node_version` | string | Node implementation version | +| `node_type` | enum | One of: `EFFECT`, `COMPUTE`, `REDUCER`, `ORCHESTRATOR` | +| `description` | string | Human-readable description | +| `input_model` | string | Name of input model class | +| `output_model` | string | Name of output model class | +| `dependencies` | array | List of required protocols and models | +| `definitions` | object | Model definitions for input/output | + +### Effect Node Specific Fields + +```yaml +io_operations: + - operation: "operation_name" + description: "What this operation does" + input_fields: + - field1 + - field2 + output_fields: + - result_field +``` + +### Orchestrator Specific Fields + +```yaml +workflows: + workflow_name: + name: "WorkflowClassName" + description: "What this workflow does" + trigger: "event_type_that_triggers" + steps: + - step: "step_name" + action: "action_to_perform" + depends_on: ["previous_step"] + output: "output_variable" + +intent_consumption: + subscribed_intents: + - "intent_type_1" + - "intent_type_2" + intent_routing_table: + intent_type_1: "workflow_to_trigger" +``` + +### Reducer Specific Fields + +```yaml +event_consumption: + subscribed_topics: + - "topic-name-1" + - "topic-name-2" + consumer_group: "reducer_consumer_group" + consumed_event_types: + - "EVENT_TYPE_1" + - "EVENT_TYPE_2" + +intent_emission: + published_intents: + - "intent_type_1" + - "intent_type_2" + intent_routing: "orchestrator_name" + +state_schema: + tables: + - name: "table_name" + columns: + - name: "column_name" + type: "uuid" + primary_key: true +``` + +--- + +## 6. Node Base Classes + +### Import Patterns + +All nodes import from `omnibase_core`: + +```python +# Base classes (pick one based on node type) +from omnibase_core.core.node_effect_service import NodeEffectService +from omnibase_core.base.node_compute_service import NodeComputeService +from omnibase_core.core.node_reducer_service import NodeReducerService +from omnibase_core.core.node_orchestrator_service import NodeOrchestratorService + +# Common imports +from omnibase_core.core.errors.onex_error import CoreErrorCode, OnexError +from omnibase_core.core.onex_container import ModelONEXContainer +from omnibase_core.enums.enum_health_status import EnumHealthStatus +from omnibase_core.models.core.model_health_status import ModelHealthStatus +``` + +### Base Class Methods + +**NodeEffectService:** +```python +class NodeEffectService: + def __init__(self, container: ModelONEXContainer): ... + async def _initialize_node_resources(self) -> None: ... + async def _cleanup_node_resources(self) -> None: ... + async def run(self) -> None: ... # Main event loop + def health_check(self) -> ModelHealthStatus: ... +``` + +**NodeComputeService[TInput, TOutput]:** +```python +class NodeComputeService(Generic[TInput, TOutput]): + def __init__(self, container: ModelONEXContainer): ... + async def initialize(self) -> None: ... + async def compute(self, input_data: TInput) -> TOutput: ... # Pure transformation +``` + +**NodeReducerService:** +```python +class NodeReducerService: + def __init__(self, container: ModelONEXContainer): ... + async def reduce(self, input_data: TInput) -> TOutput: ... # Aggregation + async def initialize(self) -> None: ... + async def cleanup(self) -> None: ... +``` + +**NodeOrchestratorService:** +```python +class NodeOrchestratorService: + def __init__(self, container: ModelONEXContainer): ... + async def orchestrate(self, input_data: TInput) -> TOutput: ... # Workflow coordination + async def initialize(self) -> None: ... + async def cleanup(self) -> None: ... + async def health_check(self) -> dict: ... +``` + +--- + +## 7. Deployment Model + +### Current: 1 Container Per Node + +Each node is deployed as an independent Docker container: + +```dockerfile +# Example Dockerfile for a node +FROM python:3.11-slim + +WORKDIR /app +COPY . . +RUN pip install -e . + +# Each node has its own entry point +CMD ["python", "-m", "omnibase_infra.nodes.node_vault_adapter_effect.v1_0_0.node"] +``` + +**Docker Compose Example:** +```yaml +services: + vault-adapter: + build: . + command: python -m omnibase_infra.nodes.node_vault_adapter_effect.v1_0_0.node + environment: + - VAULT_ADDR=http://vault:8200 + - VAULT_TOKEN=${VAULT_TOKEN} + - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 + depends_on: + - kafka + - vault + + consul-projector: + build: . + command: python -m omnibase_infra.nodes.node_consul_projector_effect.v1_0_0.node + environment: + - CONSUL_ADDR=http://consul:8500 + - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 + depends_on: + - kafka + - consul + + kafka-adapter: + build: . + command: python -m omnibase_infra.nodes.kafka_adapter.v1_0_0.node + environment: + - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 + depends_on: + - kafka +``` + +### Node Communication + +Nodes communicate exclusively via **Kafka topics**: + +``` +┌─────────────────┐ publish ┌─────────────────┐ +│ Effect Node │ ─────────────────► │ Kafka │ +│ (vault_adapter) │ │ Event Bus │ +└─────────────────┘ └────────┬────────┘ + │ + │ subscribe + ▼ + ┌─────────────────┐ + │ Reducer Node │ + │ (omni_reducer) │ + └────────┬────────┘ + │ + │ emit intent + ▼ + ┌─────────────────┐ + │ Orchestrator │ + │ (omni_orchestr) │ + └─────────────────┘ +``` + +### Entry Point Pattern + +Every node has the same entry point pattern: + +```python +# Entry point for running the node +if __name__ == "__main__": + import sys + + # Create container (simplified for standalone operation) + container = ModelONEXContainer() + + # Create and run the node + node = NodeClassName(container) + + # Run the node with asyncio + try: + asyncio.run(node.run()) + except KeyboardInterrupt: + print("\nNode shutting down...") + sys.exit(0) +``` + +--- + +## 8. Limitations (Why We're Migrating) + +### Resource Overhead + +**Problem:** Each node requires its own container with: +- Full Python runtime (~150MB base image) +- Separate Kafka consumer connections +- Independent health check endpoints +- Duplicate dependency installations + +**Impact:** With 15+ nodes, this results in significant resource waste: +``` +15 nodes × 150MB = 2.25GB+ for container images alone +15 nodes × 1 Kafka connection = 15 Kafka connections +``` + +### Complex Deployment + +**Problem:** Each node needs separate: +- Docker image builds +- Kubernetes deployments/pods +- Service definitions +- ConfigMaps/Secrets +- Health check probes + +**Impact:** Managing 15+ separate deployments becomes unwieldy: +- Difficult to coordinate rolling updates +- Complex dependency management +- Increased Kubernetes resource definitions + +### No Shared Handler Infrastructure + +**Problem:** Common functionality is duplicated: +- Kafka consumer setup in every node +- Health check endpoints repeated +- Error handling patterns duplicated +- Logging configuration repeated + +**Impact:** +- Code duplication across nodes +- Inconsistent error handling +- Difficult to add cross-cutting concerns + +### Scaling Limitations + +**Problem:** Cannot scale node types independently: +- Must scale entire container for one node +- Cannot collocate related nodes efficiently +- Memory-intensive nodes affect all + +### Migration Path: Runtime Host Model + +The new **Runtime Host** model addresses these limitations by: +- Running multiple nodes in a single container +- Sharing Kafka connections and infrastructure +- Providing unified health check endpoints +- Enabling efficient resource sharing +- Simplifying deployment to a single host + +``` +┌─────────────────────────────────────────────────┐ +│ Runtime Host │ +│ ┌───────────┐ ┌───────────┐ ┌───────────────┐ │ +│ │VaultNode │ │ConsulNode │ │OrchestratorNd │ │ +│ └───────────┘ └───────────┘ └───────────────┘ │ +│ │ +│ Shared: Kafka, Health, Logging, Error Handling │ +└─────────────────────────────────────────────────┘ +``` + +--- + +## Summary + +This document captures the current ONEX node architecture with: + +1. **1-container-per-node deployment** - Each node runs independently +2. **4 node types** - EFFECT, COMPUTE, REDUCER, ORCHESTRATOR +3. **Standard directory structure** - `nodes//v1_0_0/` +4. **Contract-driven configuration** - `contract.yaml` defines everything +5. **Base classes from omnibase_core** - Consistent inheritance patterns +6. **Kafka-based communication** - Event bus for all inter-node messaging + +The migration to Runtime Host will preserve the node contracts and logic while fundamentally changing the deployment model for improved efficiency. diff --git a/docs/architecture/DECLARATIVE_EFFECT_NODES_PLAN.md b/docs/architecture/DECLARATIVE_EFFECT_NODES_PLAN.md new file mode 100644 index 0000000000..64583bed3f --- /dev/null +++ b/docs/architecture/DECLARATIVE_EFFECT_NODES_PLAN.md @@ -0,0 +1,1623 @@ +# Effect Nodes Plan for omnibase_infra + +**Created**: December 2, 2025 +**Status**: Planning +**Dependencies**: MVP_EXECUTION_PLAN.md, EFFECT_NODES_SPEC.md (omniintelligence) +**Target**: omnibase-core ^0.4.0 (post contract-driven effect release) + +--- + +## Executive Summary + +This document outlines the integration of contract-driven effect nodes into `omnibase_infra`, building on the existing MVP execution plan. The contract-driven pattern will eliminate boilerplate code, standardize resilience patterns, and enable runtime configuration changes without code deployment. + +**Naming Convention Change (December 2025)**: Contract-driven nodes are now the DEFAULT implementation and use standard names (no suffix). Previous imperative implementations are renamed with a `Legacy` suffix during the migration period. + +**Key Benefits**: +- Reduce legacy imperative effect node code by ~60% +- Standardize retry, circuit breaker, and DLQ patterns +- Enable infrastructure-as-code for all effect nodes +- Simplify testing through protocol handler mocking + +--- + +## Naming Convention + +### Standard Names (Contract-Driven - DEFAULT) + +Contract-driven effect nodes use standard names without any suffix. These are the default implementation: + +| Class Name | Description | +|------------|-------------| +| `NodeEffect` | Base class for contract-driven effect nodes | +| `NodePostgresAdapter` | PostgreSQL adapter (contract-driven) | +| `NodeKafkaAdapter` | Kafka adapter (contract-driven) | +| `NodeConsulAdapter` | Consul adapter (contract-driven) | +| `NodeVaultAdapterEffect` | Vault adapter (contract-driven) | +| `NodeKeycloakAdapterEffect` | Keycloak adapter (contract-driven) | + +### Legacy Names (Imperative - Deprecated) + +Previous imperative implementations are renamed with a `Legacy` suffix during the migration period: + +| Legacy Class Name | Original Name | Deprecation Timeline | +|-------------------|---------------|----------------------| +| `NodeEffectLegacy` | `NodeEffect` | Remove in v0.5.0 | +| `NodePostgresAdapterLegacy` | `NodePostgresAdapter` | Remove in v0.5.0 | +| `NodeKafkaAdapterLegacy` | `NodeKafkaAdapter` | Remove in v0.5.0 | +| `NodeConsulAdapterLegacy` | `NodeConsulAdapter` | Remove in v0.5.0 | +| `NodeVaultAdapterEffectLegacy` | `NodeVaultAdapterEffect` | Remove in v0.5.0 | +| `NodeKeycloakAdapterEffectLegacy` | `NodeKeycloakAdapterEffect` | Remove in v0.5.0 | + +### Migration Path + +1. **Phase 1 (v0.4.0)**: Legacy suffix added to imperative nodes, contract-driven nodes take standard names +2. **Phase 2 (v0.4.x)**: Deprecation warnings on Legacy imports +3. **Phase 3 (v0.5.0)**: Legacy classes removed entirely + +### Import Examples + +```python +# NEW (v0.4.0+): Contract-driven effect nodes (DEFAULT) +from omnibase_core.nodes import NodeEffect # Base class +from omnibase_infra.nodes import NodePostgresAdapter # Contract-driven + +# DEPRECATED (v0.4.0-v0.4.x): Legacy imperative nodes +from omnibase_core.nodes import NodeEffectLegacy # Will be removed in v0.5.0 +from omnibase_infra.nodes import NodePostgresAdapterLegacy # Will be removed in v0.5.0 +``` + +--- + +## Core Design Invariants + +These invariants apply to all contract-driven effect nodes: + +1. **All behavior is contract-driven** - YAML contracts define everything +2. **NodeRuntime is the only executable event loop** - Effect nodes don't run their own loops +3. **Node logic is pure: no I/O, no mixins, no inheritance** +4. **Core never depends on SPI or infra** - Dependency: infra -> spi -> core +5. **SPI only defines protocols, never implementations** +6. **Infra owns all I/O and real system integrations** - Handlers live in infra + +**Critical Invariant**: +``` +No code in omnibase_core may initiate network I/O, database I/O, +file I/O, or external process execution. +``` + +This means: +- Handler implementations (HTTP, Postgres, Kafka, Bolt) MUST be in omnibase_infra +- Protocol interfaces are defined in omnibase_spi +- NodeRuntime and NodeInstance are in omnibase_core + +--- + +## Handler Placement Rule + +**Rule**: Any code that touches the network, filesystem, sockets, or external systems +MUST live in `omnibase_infra`. Core and SPI must remain pure and dependency-free. + +**Correct Structure**: +``` +omnibase_infra/ + handlers/ + http/ + http_rest_handler.py + http_retry_policy.py + http_circuit_breaker.py + db/ + postgres_handler.py + connection_pool.py + graph/ + bolt_handler.py + event/ + kafka_handler.py + resilience/ + retry_policy.py + circuit_breaker.py + rate_limiter.py + runtime_host/ + entrypoint.py + wiring.py +``` + +--- + +## Integration with Runtime Host Model + +Contract-driven effect nodes are designed to work with the Runtime Host model: + +``` +RuntimeHostProcess (omnibase_infra) + +-- NodeRuntime (omnibase_core) + |-- NodeInstance(contract-driven effect 1) + |-- NodeInstance(contract-driven effect 2) + +-- handlers: + postgres_handler (infra) + kafka_handler (infra) + http_rest_handler (infra) +``` + +**Key Points**: +- Contract-driven effects are loaded as NodeInstances +- Handlers are registered with the runtime at startup +- Effect contracts declare which handler types they need +- Runtime routes operations to appropriate handlers + +--- + +## 1. Integration with Existing MVP Plan + +### 1.1 Timeline Integration + +The contract-driven effect work slots into the MVP as a **Phase 2.5** enhancement: + +| Phase | Description | Status | Contract-Driven Impact | +|-------|-------------|--------|------------------------| +| Phase 0 | Pre-flight | Existing | No change | +| Phase 1 | Foundation | Existing | No change | +| Phase 2 | Effect Nodes (Legacy) | Existing | **Baseline, renamed to Legacy** | +| **Phase 2.5** | **Contract-Driven Effect Migration** | **NEW** | **New nodes take standard names** | +| Phase 3 | Stamping Service | Existing | Can use contract-driven Postgres | +| Phase 4 | Contract-Driven Nodes (Reducer/Orchestrator) | Existing | Foundation for effect contracts | +| Phase 5 | Compute Nodes | Existing | No change | +| Phase 6 | Testing & Validation | Existing | Extended for contracts | + +### 1.2 Dependency Chain + +``` +omnibase-spi ^0.2.0 (protocol definitions) + | + +-- ProtocolHandler protocol (abstract, no I/O) + +-- Protocol contracts and interfaces + | + v +omnibase-core ^0.4.0 (contract-driven effects release) + | + +-- NodeEffect base class (contract-driven, DEFAULT) + +-- NodeEffectLegacy base class (imperative, DEPRECATED) + +-- NodeRuntime (event loop, pure orchestration) + +-- NodeInstance (pure node logic) + +-- Effect contract JSON schema + | + v +omnibase_infra ^0.2.0 (post-migration) + | + +-- YAML effect contracts + +-- Protocol handlers (HTTP, Bolt, Postgres, Kafka) - ALL I/O HERE + +-- ProtocolHandlerRegistry (manages handler lifecycle) + +-- Resilience utilities (RetryPolicy, CircuitBreaker) + +-- RuntimeHostProcess (wiring and entrypoint) + +-- NodePostgresAdapter, NodeKafkaAdapter, etc. (contract-driven) + +-- NodePostgresAdapterLegacy, etc. (deprecated, for fallback) + +-- Infrastructure-specific operations +``` + +### 1.3 Parallel vs Sequential Work + +**Can Run in Parallel with MVP**: +- Contract YAML design (no code dependency) +- Protocol handler implementation in omnibase_infra +- Documentation and examples + +**Must Wait for omnibase-core 0.4.0**: +- Actual node conversion +- Integration testing +- Runtime validation + +--- + +## 2. Effect Nodes Inventory + +### 2.1 Current Effect Nodes in omnibase_infra + +| Node | Location | Protocol | Priority | Effort | +|------|----------|----------|----------|--------| +| `postgres_adapter` | `nodes/postgres_adapter/v1_0_0/` | postgres | P0 | Medium | +| `kafka_adapter` | `nodes/kafka_adapter/v1_0_0/` | kafka | P0 | Medium | +| `consul_adapter` | `nodes/consul_adapter/v1_0_0/` | http_rest | P1 | Low | +| `node_vault_adapter_effect` | `nodes/node_vault_adapter_effect/v1_0_0/` | http_rest | P1 | Low | +| `node_keycloak_adapter_effect` | `nodes/node_keycloak_adapter_effect/v1_0_0/` | http_rest | P2 | Medium | +| `hook_node` (webhook) | `nodes/hook_node/v1_0_0/` | http_rest | P2 | Low | +| `node_consul_projector_effect` | `nodes/node_consul_projector_effect/v1_0_0/` | http_rest | P3 | Low | + +### 2.2 Priority Justification + +**P0 - Foundation (Week 1)**: +- **Postgres**: Workflow state storage, FSM transitions - core to all operations +- **Kafka**: Event publishing - enables loose coupling between services + +**P1 - Service Integration (Week 2)**: +- **Consul**: Service discovery, health checks - required for deployment +- **Vault**: Secret management - security critical but can use env vars initially + +**P2 - Extended Features (Week 3)**: +- **Keycloak**: Authentication flows - can defer to later if using API keys +- **Webhook**: Notification delivery - enhancement, not critical path + +**P3 - Projections (Week 4)**: +- **Consul Projector**: Read-model projection - optimization, not core + +### 2.3 Effort Estimation + +| Effort Level | Hours | Description | +|--------------|-------|-------------| +| Low | 2-4h | Simple HTTP REST, 3-5 operations | +| Medium | 4-8h | Complex protocol, many operations, custom validation | +| High | 8-16h | New protocol handler, extensive testing | + +--- + +## 3. YAML Contracts to Create + +### 3.1 Contract Directory Structure + +``` +omnibase_infra/ +├── contracts/ +│ └── effects/ +│ ├── _schema/ +│ │ └── effect_contract_schema.json # JSON Schema for validation +│ ├── postgres_workflow.yaml # Workflow state persistence +│ ├── postgres_fsm.yaml # FSM transition storage +│ ├── kafka_event.yaml # Event publishing +│ ├── consul_registry.yaml # Service discovery +│ ├── consul_kv.yaml # KV store operations +│ ├── vault_secret.yaml # Secret management +│ ├── keycloak_auth.yaml # Authentication +│ ├── webhook_delivery.yaml # Webhook notifications +│ └── valkey_cache.yaml # Caching (future) +└── src/omnibase_infra/ + ├── handlers/ # At package root, NOT under nodes + │ ├── __init__.py + │ ├── http/ + │ │ ├── http_rest_handler.py # HTTP REST protocol handler + │ │ ├── http_retry_policy.py # HTTP-specific retry logic + │ │ └── http_circuit_breaker.py # HTTP circuit breaker + │ ├── db/ + │ │ ├── postgres_handler.py # PostgreSQL protocol handler + │ │ └── connection_pool.py # Connection pool management + │ ├── graph/ + │ │ └── bolt_handler.py # Neo4j Bolt protocol handler + │ ├── event/ + │ │ └── kafka_handler.py # Kafka protocol handler + │ └── cache/ + │ └── valkey_handler.py # Valkey (Redis-compatible) handler + ├── resilience/ # Shared resilience patterns + │ ├── __init__.py + │ ├── retry_policy.py # Generic retry policy + │ ├── circuit_breaker.py # Generic circuit breaker + │ └── rate_limiter.py # Rate limiting utilities + ├── runtime_host/ # Runtime host wiring + │ ├── entrypoint.py # Process entrypoint + │ └── wiring.py # Handler registration + └── nodes/ # Nodes are separate from handlers + ├── effect_nodes/ # Contract-driven effect nodes (DEFAULT) + │ ├── __init__.py + │ └── loader.py # Contract loader utility + └── effect_nodes_legacy/ # Legacy imperative nodes (DEPRECATED) + ├── __init__.py + └── ... # To be removed in v0.5.0 +``` + +**Important**: Handlers live at the package root (`src/omnibase_infra/handlers/`), NOT under +`nodes/`. This follows the architectural invariant that handlers contain I/O code which must +remain separate from pure node logic. + +### 3.2 postgres_workflow.yaml + +```yaml +# Workflow state persistence for infrastructure orchestration +name: postgres_workflow_effect +version: + major: 1 + minor: 0 + patch: 0 + +description: | + PostgreSQL effect for workflow state persistence. + Stores workflow execution states, checkpoints, and audit logs. + +protocol: + type: postgres + version: "14" + +connection: + host: ${POSTGRES_HOST} + port: ${POSTGRES_PORT} + database: ${POSTGRES_DB} + pool: + min_size: 2 + max_size: 10 + max_idle_time_ms: 300000 + timeout_ms: 30000 + tls: + enabled: ${POSTGRES_TLS_ENABLED} + verify: true + +authentication: + type: basic + basic: + username: ${POSTGRES_USER} + password: ${POSTGRES_PASSWORD} + +operations: + save_workflow_state: + description: "Persist workflow execution state" + request: + sql: | + INSERT INTO workflow_states ( + workflow_id, execution_id, state, payload, checkpoint, created_at + ) VALUES ($1, $2, $3, $4, $5, NOW()) + ON CONFLICT (workflow_id, execution_id) + DO UPDATE SET state = $3, payload = $4, checkpoint = $5, updated_at = NOW() + RETURNING id, workflow_id, execution_id, state + sql_params: + - ${input.workflow_id} + - ${input.execution_id} + - ${input.state} + - ${input.payload} + - ${input.checkpoint} + response: + mapping: + record_id: "$.rows[0].id" + workflow_id: "$.rows[0].workflow_id" + validation: + required_fields: + - workflow_id + - execution_id + - state + + get_workflow_state: + description: "Retrieve workflow state by execution ID" + request: + sql: | + SELECT id, workflow_id, execution_id, state, payload, checkpoint, created_at, updated_at + FROM workflow_states + WHERE workflow_id = $1 AND execution_id = $2 + sql_params: + - ${input.workflow_id} + - ${input.execution_id} + response: + mapping: + state: "$.rows[0].state" + payload: "$.rows[0].payload" + checkpoint: "$.rows[0].checkpoint" + + list_workflow_history: + description: "List workflow execution history" + request: + sql: | + SELECT id, workflow_id, execution_id, state, created_at, updated_at + FROM workflow_states + WHERE workflow_id = $1 + ORDER BY created_at DESC + LIMIT $2 OFFSET $3 + sql_params: + - ${input.workflow_id} + - ${input.limit} + - ${input.offset} + response: + mapping: + executions: "$.rows" + count: "$.row_count" + + delete_workflow_state: + description: "Delete workflow state (for cleanup)" + request: + sql: | + DELETE FROM workflow_states + WHERE workflow_id = $1 AND execution_id = $2 + RETURNING id + sql_params: + - ${input.workflow_id} + - ${input.execution_id} + response: + mapping: + deleted: "$.affected_rows" + +resilience: + retry: + enabled: true + max_attempts: 3 + initial_delay_ms: 100 + max_delay_ms: 2000 + backoff_multiplier: 2.0 + jitter: true + circuit_breaker: + enabled: true + failure_threshold: 5 + success_threshold: 2 + timeout_ms: 30000 + timeout: + request_ms: 5000 + operation_ms: 30000 + +events: + consume: + topic: dev.omnibase-infra.effect.postgres-workflow.request.v1 + group_id: postgres-workflow-effect-consumer + produce: + success_topic: dev.omnibase-infra.effect.postgres-workflow.response.v1 + failure_topic: dev.omnibase-infra.effect.postgres-workflow.failure.v1 + dlq_topic: dev.omnibase-infra.effect.postgres-workflow.request.v1.dlq + +observability: + metrics: + enabled: true + prefix: omnibase_infra_postgres_workflow + labels: + service: omnibase_infra + component: postgres_workflow_effect + logging: + level: INFO + sanitize_secrets: true + secret_patterns: + - password + - secret + - token + +metadata: + author: OmniInfra Team + created_at: "2025-12-02" + tags: + - infrastructure + - persistence + - workflow + documentation: https://docs.omninode.ai/infra/effects/postgres-workflow +``` + +### 3.3 postgres_fsm.yaml + +```yaml +# FSM state transition storage +name: postgres_fsm_effect +version: + major: 1 + minor: 0 + patch: 0 + +description: | + PostgreSQL effect for FSM state transition persistence. + Stores state machine states, transitions, and audit history. + +protocol: + type: postgres + version: "14" + +connection: + host: ${POSTGRES_HOST} + port: ${POSTGRES_PORT} + database: ${POSTGRES_DB} + pool: + min_size: 2 + max_size: 10 + timeout_ms: 30000 + +authentication: + type: basic + basic: + username: ${POSTGRES_USER} + password: ${POSTGRES_PASSWORD} + +operations: + record_transition: + description: "Record an FSM state transition" + request: + sql: | + INSERT INTO fsm_transitions ( + entity_id, entity_type, fsm_type, from_state, to_state, + event, context, correlation_id, created_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW()) + RETURNING id, entity_id, from_state, to_state + sql_params: + - ${input.entity_id} + - ${input.entity_type} + - ${input.fsm_type} + - ${input.from_state} + - ${input.to_state} + - ${input.event} + - ${input.context} + - ${context.correlation_id} + response: + mapping: + transition_id: "$.rows[0].id" + entity_id: "$.rows[0].entity_id" + validation: + required_fields: + - entity_id + - entity_type + - fsm_type + - from_state + - to_state + - event + + get_current_state: + description: "Get current FSM state for an entity" + request: + sql: | + SELECT to_state as current_state, event as last_event, created_at as last_transition + FROM fsm_transitions + WHERE entity_id = $1 AND entity_type = $2 AND fsm_type = $3 + ORDER BY created_at DESC + LIMIT 1 + sql_params: + - ${input.entity_id} + - ${input.entity_type} + - ${input.fsm_type} + response: + mapping: + current_state: "$.rows[0].current_state" + last_event: "$.rows[0].last_event" + last_transition: "$.rows[0].last_transition" + + get_transition_history: + description: "Get FSM transition history for an entity" + request: + sql: | + SELECT id, from_state, to_state, event, context, correlation_id, created_at + FROM fsm_transitions + WHERE entity_id = $1 AND entity_type = $2 AND fsm_type = $3 + ORDER BY created_at DESC + LIMIT $4 OFFSET $5 + sql_params: + - ${input.entity_id} + - ${input.entity_type} + - ${input.fsm_type} + - ${input.limit} + - ${input.offset} + response: + mapping: + transitions: "$.rows" + count: "$.row_count" + +resilience: + retry: + enabled: true + max_attempts: 3 + initial_delay_ms: 100 + max_delay_ms: 2000 + circuit_breaker: + enabled: true + failure_threshold: 5 + timeout_ms: 30000 + +events: + consume: + topic: dev.omnibase-infra.effect.postgres-fsm.request.v1 + group_id: postgres-fsm-effect-consumer + produce: + success_topic: dev.omnibase-infra.effect.postgres-fsm.response.v1 + dlq_topic: dev.omnibase-infra.effect.postgres-fsm.request.v1.dlq + +observability: + metrics: + enabled: true + prefix: omnibase_infra_postgres_fsm + logging: + level: INFO + sanitize_secrets: true + +metadata: + author: OmniInfra Team + created_at: "2025-12-02" + tags: + - infrastructure + - fsm + - state-management +``` + +### 3.4 kafka_event.yaml + +```yaml +# Kafka event publishing for infrastructure events +name: kafka_event_effect +version: + major: 1 + minor: 0 + patch: 0 + +description: | + Kafka effect for event publishing. + Publishes infrastructure events with delivery confirmation. + +protocol: + type: kafka + version: "3.0" + +connection: + url: ${KAFKA_BOOTSTRAP_SERVERS} + timeout_ms: 30000 + pool: + max_size: 5 + +authentication: + type: none # SASL config can be added if needed + +operations: + publish_event: + description: "Publish an event to a Kafka topic" + request: + topic: ${input.topic} + key: ${input.key} + payload: ${input.payload} + response: + mapping: + partition: "$.partition" + offset: "$.offset" + topic: "$.topic" + validation: + required_fields: + - topic + - payload + + publish_batch: + description: "Publish multiple events to a topic" + request: + topic: ${input.topic} + messages: ${input.messages} + response: + mapping: + published_count: "$.count" + partitions: "$.partitions" + validation: + required_fields: + - topic + - messages + + publish_workflow_event: + description: "Publish a workflow lifecycle event" + request: + topic: dev.omnibase-infra.workflow.${input.event_type}.v1 + key: ${input.workflow_id} + payload: + event_id: ${context.event_id} + event_type: ${input.event_type} + workflow_id: ${input.workflow_id} + execution_id: ${input.execution_id} + state: ${input.state} + timestamp: ${context.timestamp} + correlation_id: ${context.correlation_id} + response: + mapping: + partition: "$.partition" + offset: "$.offset" + + publish_fsm_transition: + description: "Publish an FSM transition event" + request: + topic: dev.omnibase-infra.fsm.transition.v1 + key: ${input.entity_id} + payload: + event_id: ${context.event_id} + entity_id: ${input.entity_id} + entity_type: ${input.entity_type} + fsm_type: ${input.fsm_type} + from_state: ${input.from_state} + to_state: ${input.to_state} + event: ${input.event} + timestamp: ${context.timestamp} + correlation_id: ${context.correlation_id} + response: + mapping: + partition: "$.partition" + offset: "$.offset" + +resilience: + retry: + enabled: true + max_attempts: 5 + initial_delay_ms: 500 + max_delay_ms: 10000 + backoff_multiplier: 2.0 + circuit_breaker: + enabled: true + failure_threshold: 10 + success_threshold: 3 + timeout_ms: 60000 + timeout: + request_ms: 10000 + operation_ms: 60000 + +observability: + metrics: + enabled: true + prefix: omnibase_infra_kafka_event + labels: + service: omnibase_infra + logging: + level: INFO + include_request_body: false + +metadata: + author: OmniInfra Team + created_at: "2025-12-02" + tags: + - infrastructure + - events + - kafka +``` + +### 3.5 consul_registry.yaml + +```yaml +# Consul service registry operations +name: consul_registry_effect +version: + major: 1 + minor: 0 + patch: 0 + +description: | + Consul effect for service registration and discovery. + Manages service lifecycle in the Consul catalog. + +protocol: + type: http_rest + version: "HTTP/1.1" + content_type: application/json + +connection: + url: http://${CONSUL_HOST}:${CONSUL_PORT} + timeout_ms: 10000 + pool: + max_size: 5 + +authentication: + type: api_key + api_key: + header: X-Consul-Token + prefix: "" + value: ${CONSUL_ACL_TOKEN} + +operations: + register_service: + description: "Register a service with Consul" + request: + method: PUT + path: /v1/agent/service/register + body: + ID: ${input.service_id} + Name: ${input.service_name} + Tags: ${input.tags} + Address: ${input.address} + Port: ${input.port} + Check: + HTTP: http://${input.address}:${input.port}${input.health_path} + Interval: ${input.check_interval} + Timeout: ${input.check_timeout} + response: + success_codes: [200] + mapping: + registered: "true" + validation: + required_fields: + - service_id + - service_name + - address + - port + + deregister_service: + description: "Deregister a service from Consul" + request: + method: PUT + path: /v1/agent/service/deregister/${input.service_id} + response: + success_codes: [200] + mapping: + deregistered: "true" + validation: + required_fields: + - service_id + + get_service: + description: "Get service details by ID" + request: + method: GET + path: /v1/catalog/service/${input.service_name} + response: + success_codes: [200] + mapping: + services: "$[*]" + count: "$.length" + + list_services: + description: "List all registered services" + request: + method: GET + path: /v1/catalog/services + response: + success_codes: [200] + mapping: + services: "$" + + health_check: + description: "Get health status of a service" + request: + method: GET + path: /v1/health/service/${input.service_name} + query: + passing: ${input.passing_only} + response: + success_codes: [200] + mapping: + instances: "$[*]" + healthy_count: "$.length" + +resilience: + retry: + enabled: true + max_attempts: 3 + initial_delay_ms: 500 + circuit_breaker: + enabled: true + failure_threshold: 5 + timeout_ms: 30000 + +events: + consume: + topic: dev.omnibase-infra.effect.consul-registry.request.v1 + group_id: consul-registry-effect-consumer + produce: + success_topic: dev.omnibase-infra.effect.consul-registry.response.v1 + dlq_topic: dev.omnibase-infra.effect.consul-registry.request.v1.dlq + +observability: + metrics: + enabled: true + prefix: omnibase_infra_consul_registry + +metadata: + author: OmniInfra Team + created_at: "2025-12-02" + tags: + - infrastructure + - service-discovery + - consul +``` + +### 3.6 valkey_cache.yaml + +```yaml +# Valkey (Redis-compatible) caching operations +name: valkey_cache_effect +version: + major: 1 + minor: 0 + patch: 0 + +description: | + Valkey effect for caching operations. + Provides high-performance key-value caching with TTL support. + +protocol: + type: valkey # Custom handler required + version: "7.0" + +connection: + host: ${VALKEY_HOST} + port: ${VALKEY_PORT} + database: ${VALKEY_DB} + pool: + min_size: 2 + max_size: 20 + timeout_ms: 5000 + tls: + enabled: ${VALKEY_TLS_ENABLED} + +authentication: + type: basic + basic: + password: ${VALKEY_PASSWORD} + +operations: + get: + description: "Get a value by key" + request: + command: GET + key: ${input.key} + response: + mapping: + value: "$.value" + exists: "$.exists" + validation: + required_fields: + - key + + set: + description: "Set a value with optional TTL" + request: + command: SET + key: ${input.key} + value: ${input.value} + ttl_seconds: ${input.ttl} + response: + mapping: + success: "$.ok" + validation: + required_fields: + - key + - value + + delete: + description: "Delete a key" + request: + command: DEL + key: ${input.key} + response: + mapping: + deleted: "$.count" + + get_many: + description: "Get multiple values by keys" + request: + command: MGET + keys: ${input.keys} + response: + mapping: + values: "$.values" + + set_many: + description: "Set multiple key-value pairs" + request: + command: MSET + pairs: ${input.pairs} + response: + mapping: + success: "$.ok" + + increment: + description: "Increment a numeric value" + request: + command: INCR + key: ${input.key} + amount: ${input.amount} + response: + mapping: + new_value: "$.value" + + expire: + description: "Set TTL on an existing key" + request: + command: EXPIRE + key: ${input.key} + ttl_seconds: ${input.ttl} + response: + mapping: + success: "$.ok" + +resilience: + retry: + enabled: true + max_attempts: 2 + initial_delay_ms: 50 + max_delay_ms: 500 + circuit_breaker: + enabled: true + failure_threshold: 10 + timeout_ms: 10000 + timeout: + request_ms: 1000 + operation_ms: 5000 + +events: + consume: + topic: dev.omnibase-infra.effect.valkey-cache.request.v1 + group_id: valkey-cache-effect-consumer + produce: + success_topic: dev.omnibase-infra.effect.valkey-cache.response.v1 + dlq_topic: dev.omnibase-infra.effect.valkey-cache.request.v1.dlq + +observability: + metrics: + enabled: true + prefix: omnibase_infra_valkey_cache + histograms: + buckets: [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0] + logging: + level: DEBUG + include_request_body: false + +metadata: + author: OmniInfra Team + created_at: "2025-12-02" + tags: + - infrastructure + - caching + - valkey + dependencies: + - valkey_handler # Custom protocol handler +``` + +--- + +## 4. Integration with omnibase_core + +### 4.1 Importing NodeEffect (Contract-Driven) + +```python +"""Example of using contract-driven effect nodes in omnibase_infra.""" + +from pathlib import Path +from omnibase_core.nodes import NodeEffect # Contract-driven base class (DEFAULT) +from omnibase_core.models import ModelEffectInput + +# Path to contracts directory +CONTRACTS_DIR = Path(__file__).parent.parent / "contracts" / "effects" + +async def create_postgres_workflow_effect() -> NodeEffect: + """Create and initialize Postgres workflow effect node.""" + node = NodeEffect( + contract_path=CONTRACTS_DIR / "postgres_workflow.yaml", + config_overrides={ + # Override env vars if needed for testing + "POSTGRES_HOST": "localhost", + "POSTGRES_PORT": "5432", + } + ) + await node.initialize() + return node + +async def save_workflow_state( + node: NodeEffect, + workflow_id: str, + execution_id: str, + state: str, + payload: dict +) -> dict: + """Save workflow state using contract-driven effect.""" + result = await node.execute_effect( + ModelEffectInput( + operation="save_workflow_state", + params={ + "workflow_id": workflow_id, + "execution_id": execution_id, + "state": state, + "payload": payload, + "checkpoint": None, + } + ) + ) + + if not result.success: + raise RuntimeError(f"Failed to save workflow state: {result.error}") + + return result.data +``` + +### 4.1.1 Legacy Fallback (Deprecated) + +For migration purposes, you can still use the legacy imperative nodes: + +```python +"""Legacy imperative nodes - DEPRECATED, will be removed in v0.5.0.""" + +import warnings +from omnibase_core.nodes import NodeEffectLegacy # DEPRECATED + +# This will emit a deprecation warning +warnings.warn( + "NodeEffectLegacy is deprecated and will be removed in v0.5.0. " + "Use NodeEffect with YAML contracts instead.", + DeprecationWarning, + stacklevel=2 +) +``` + +### 4.2 Registering Custom Protocol Handlers + +**Architectural Note**: All handler implementations live in `omnibase_infra`, never in +`omnibase_core`. Core only contains pure logic (NodeRuntime, NodeInstance). Protocol +interfaces (abstract base classes) are defined in `omnibase_spi`. + +For custom protocols (e.g., Valkey): + +```python +"""Custom Valkey protocol handler for omnibase_infra. + +Location: src/omnibase_infra/handlers/cache/valkey_handler.py + +This handler implements the ProtocolHandler protocol from omnibase_spi +and provides the actual I/O operations for Valkey/Redis. +""" + +from typing import Any +# Protocol interface from SPI (abstract, no I/O) +from omnibase_spi.protocols import ProtocolHandler +# Models can be shared or defined in infra +from omnibase_infra.handlers.models import ModelProtocolRequest, ModelProtocolResponse +# Handler registry lives in infra (manages I/O components) +from omnibase_infra.handlers.registry import ProtocolHandlerRegistry +import valkey # or redis-py with valkey compatibility + +class ValkeyHandler(ProtocolHandler): + """Protocol handler for Valkey (Redis-compatible) operations.""" + + def __init__(self): + self.client: valkey.Redis | None = None + + async def initialize(self, config: dict[str, Any]) -> None: + """Initialize Valkey connection pool.""" + self.client = valkey.Redis( + host=config.get("host", "localhost"), + port=config.get("port", 6379), + db=config.get("database", 0), + password=config.get("authentication", {}).get("basic", {}).get("password"), + decode_responses=True, + max_connections=config.get("pool", {}).get("max_size", 10), + ) + + async def shutdown(self) -> None: + """Close Valkey connection.""" + if self.client: + await self.client.close() + + async def execute( + self, + request: ModelProtocolRequest, + operation_config: dict[str, Any] + ) -> ModelProtocolResponse: + """Execute Valkey command.""" + import time + start_time = time.perf_counter() + + try: + req_config = operation_config.get("request", {}) + command = req_config.get("command", "").upper() + + # Route to appropriate Valkey command + if command == "GET": + value = await self.client.get(request.params.get("input", {}).get("key")) + data = {"value": value, "exists": value is not None} + elif command == "SET": + params = request.params.get("input", {}) + ttl = params.get("ttl") + if ttl: + result = await self.client.setex(params["key"], ttl, params["value"]) + else: + result = await self.client.set(params["key"], params["value"]) + data = {"ok": result} + elif command == "DEL": + count = await self.client.delete(request.params.get("input", {}).get("key")) + data = {"count": count} + elif command == "MGET": + keys = request.params.get("input", {}).get("keys", []) + values = await self.client.mget(keys) + data = {"values": dict(zip(keys, values))} + elif command == "INCR": + params = request.params.get("input", {}) + amount = params.get("amount", 1) + if amount == 1: + value = await self.client.incr(params["key"]) + else: + value = await self.client.incrby(params["key"], amount) + data = {"value": value} + else: + raise ValueError(f"Unsupported Valkey command: {command}") + + duration_ms = (time.perf_counter() - start_time) * 1000 + + return ModelProtocolResponse( + success=True, + data=data, + duration_ms=duration_ms + ) + + except Exception as e: + duration_ms = (time.perf_counter() - start_time) * 1000 + return ModelProtocolResponse( + success=False, + error=str(e), + duration_ms=duration_ms + ) + + async def health_check(self) -> bool: + """Check Valkey connectivity.""" + try: + return await self.client.ping() + except Exception: + return False + +# Register custom handler at module load +def register_custom_handlers(): + """Register omnibase_infra custom protocol handlers.""" + ProtocolHandlerRegistry.register("valkey", ValkeyHandler) +``` + +### 4.3 Version Pinning Strategy + +In `pyproject.toml`: + +```toml +[tool.poetry.dependencies] +# Pin to specific minor version for stability +omnibase-core = ">=0.4.0,<0.5.0" + +# Allow patch updates +omnibase-spi = "^0.2.0" +``` + +**Upgrade Policy**: +1. **Patch versions** (0.4.x): Auto-update, run tests +2. **Minor versions** (0.5.0): Review changelog, test in staging +3. **Major versions** (1.0.0): Full compatibility review, migration plan + +--- + +## 5. Migration Path + +### 5.1 Phased Migration Strategy + +``` +Phase A: Naming Transition (Week 1) +├── Rename current imperative nodes to *Legacy suffix +├── Create YAML contracts for new contract-driven nodes +├── New contract-driven nodes take standard names (no suffix) +├── Feature flag: USE_CONTRACT_DRIVEN_EFFECTS=false +└── No production impact, import aliases maintained + +Phase B: Parallel Implementation (Week 2) +├── Implement contract-driven nodes alongside Legacy code +├── Implement wrapper that can use either +├── Deprecation warnings on Legacy imports +└── Feature flag: USE_CONTRACT_DRIVEN_EFFECTS=false + +Phase C: Shadow Mode (Week 3) +├── Run both implementations in parallel +├── Compare results for consistency +├── Log discrepancies +└── Feature flag: USE_CONTRACT_DRIVEN_EFFECTS=shadow + +Phase D: Gradual Rollout (Week 4) +├── Enable contract-driven for non-critical paths +├── Monitor metrics and error rates +├── Quick rollback to Legacy capability +└── Feature flag: USE_CONTRACT_DRIVEN_EFFECTS=true (per-node) + +Phase E: Full Migration (Week 5-6) +├── All nodes using contract-driven implementation +├── Legacy nodes deprecated but available +├── Plan removal for v0.5.0 +└── Feature flag: USE_CONTRACT_DRIVEN_EFFECTS=true (default) + +Phase F: Legacy Removal (v0.5.0) +├── Remove all *Legacy classes +├── Remove feature flags +├── Contract-driven is the only implementation +└── Clean up deprecated imports +``` + +### 5.1.1 Naming Transition Details + +| Current Name | v0.4.0 Name | v0.5.0 Name | +|--------------|-------------|-------------| +| `NodeEffect` (imperative) | `NodeEffectLegacy` | REMOVED | +| N/A | `NodeEffect` (contract-driven) | `NodeEffect` (contract-driven) | +| `NodePostgresAdapter` (imperative) | `NodePostgresAdapterLegacy` | REMOVED | +| N/A | `NodePostgresAdapter` (contract-driven) | `NodePostgresAdapter` (contract-driven) | +| `NodeKafkaAdapter` (imperative) | `NodeKafkaAdapterLegacy` | REMOVED | +| N/A | `NodeKafkaAdapter` (contract-driven) | `NodeKafkaAdapter` (contract-driven) | + +### 5.2 Feature Flag Implementation + +```python +"""Feature flags for contract-driven effect migration.""" + +import os +from enum import Enum +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from omnibase_core.nodes import NodeEffect + +class EffectNodeMode(str, Enum): + """Migration mode for effect nodes.""" + LEGACY = "legacy" # Use Legacy imperative only (deprecated) + SHADOW = "shadow" # Run both, compare, use Legacy + CONTRACT = "contract" # Use contract-driven only (DEFAULT in v0.4.0+) + PER_NODE = "per_node" # Check per-node flags + +def get_effect_mode() -> EffectNodeMode: + """Get current effect node mode from environment.""" + mode = os.getenv("USE_CONTRACT_DRIVEN_EFFECTS", "contract").lower() + return EffectNodeMode(mode) + +def should_use_contract_driven(node_name: str) -> bool: + """Check if a specific node should use contract-driven implementation.""" + mode = get_effect_mode() + + if mode == EffectNodeMode.LEGACY: + return False + elif mode == EffectNodeMode.CONTRACT: + return True + elif mode == EffectNodeMode.PER_NODE: + # Check node-specific flag + flag = os.getenv(f"USE_CONTRACT_DRIVEN_{node_name.upper()}", "true") + return flag.lower() == "true" + elif mode == EffectNodeMode.SHADOW: + # Shadow mode handled separately + return False + + return True # Default to contract-driven +``` + +### 5.3 Rollback Strategy + +```python +"""Rollback utilities for contract-driven effect migration.""" + +import logging +from typing import TypeVar, Generic +from omnibase_core.nodes import NodeEffect, NodeEffectLegacy +from omnibase_core.models import ModelEffectInput + +logger = logging.getLogger(__name__) + +T = TypeVar("T", bound=NodeEffectLegacy) + +class EffectNodeFallback(Generic[T]): + """ + Fallback wrapper that tries contract-driven first, + falls back to Legacy imperative on failure. + + Note: This is a temporary migration utility. Once Legacy + nodes are removed in v0.5.0, this class will be deprecated. + """ + + def __init__( + self, + contract_node: NodeEffect, + legacy_fallback: T, + fallback_threshold: int = 3, + ): + self.contract = contract_node + self.legacy = legacy_fallback # Deprecated, for fallback only + self.threshold = fallback_threshold + self.consecutive_failures = 0 + self._using_legacy = False + + async def execute(self, operation: str, params: dict) -> dict: + """Execute with automatic fallback to Legacy.""" + if self._using_legacy: + return await self._execute_legacy(operation, params) + + try: + result = await self.contract.execute_effect( + ModelEffectInput(operation=operation, params=params) + ) + + if result.success: + self.consecutive_failures = 0 + return result.data + else: + raise RuntimeError(result.error) + + except Exception as e: + self.consecutive_failures += 1 + logger.warning( + f"Contract-driven effect failed | " + f"operation={operation} | " + f"failures={self.consecutive_failures} | " + f"error={e}" + ) + + if self.consecutive_failures >= self.threshold: + logger.error( + f"Switching to Legacy fallback | " + f"node={self.contract.contract.get('name')}" + ) + self._using_legacy = True + + return await self._execute_legacy(operation, params) + + async def _execute_legacy(self, operation: str, params: dict) -> dict: + """Execute using Legacy imperative implementation (deprecated).""" + # Call the appropriate method on the Legacy node + method = getattr(self.legacy, f"execute_{operation}", None) + if method: + return await method(**params) + raise ValueError(f"Unknown operation: {operation}") + + def reset_fallback(self) -> None: + """Reset to try contract-driven again.""" + self._using_legacy = False + self.consecutive_failures = 0 +``` + +--- + +## 6. Timeline Integration + +### 6.1 Detailed Timeline + +| Week | Phase | Tasks | Deliverables | +|------|-------|-------|--------------| +| W1 | MVP Phase 2 | Complete Legacy effect nodes, rename with suffix | Working *Legacy adapters | +| W2 | MVP Phase 3-4 | Stamping service, contract-driven reducer | Stamping node, FSM contracts | +| W3 | **Phase 2.5a** | Design YAML contracts | 5 contract files, schema validation | +| W4 | **Phase 2.5b** | Wait for omnibase-core 0.4.0 | Dependency available (NodeEffect, NodeEffectLegacy) | +| W5 | **Phase 2.5c** | Implement contract-driven nodes | New nodes with standard names, feature flags | +| W6 | **Phase 2.5d** | Shadow mode testing | Comparison logs, discrepancy fixes | +| W7 | MVP Phase 6 | Full testing, validation | Test coverage, quality gates | +| W8 | Release | Production deployment | omnibase-infra 0.2.0 | +| Future | v0.5.0 | Remove Legacy nodes | Clean codebase, only contract-driven | + +### 6.2 Dependencies and Blockers + +**Hard Dependencies**: +1. omnibase-core 0.4.0 release with `NodeEffect` (contract-driven) and `NodeEffectLegacy` +2. Protocol handlers (HTTP, Bolt, Postgres, Kafka) implemented in omnibase_infra +3. `ProtocolHandler` protocol defined in omnibase_spi +4. `ProtocolHandlerRegistry` API stable (in omnibase_infra) + +**Soft Dependencies**: +1. MVP Phase 2 Legacy effect nodes (for fallback baseline) +2. MVP Phase 4 FSM contracts (shared patterns) +3. Integration test infrastructure + +**Potential Blockers**: +1. **omnibase-core 0.4.0 delay**: Proceed with contract design only +2. **Protocol handler gaps**: Implement custom handlers in infra +3. **Schema incompatibility**: Version contracts, maintain backward compat +4. **Legacy removal timing**: Ensure sufficient deprecation period before v0.5.0 + +### 6.3 Risk Mitigation + +| Risk | Likelihood | Impact | Mitigation | +|------|------------|--------|------------| +| Core release delay | Medium | High | Parallel contract design, fallback to Legacy | +| Contract schema changes | Medium | Medium | Version contracts, maintain v1 compatibility | +| Performance regression | Low | High | Shadow mode comparison, benchmarking | +| Protocol handler bugs | Medium | Medium | Custom handler overrides, rapid patching | +| Premature Legacy removal | Low | High | Clear deprecation timeline, warning logs | + +--- + +## 7. Success Criteria + +### 7.1 Technical Criteria + +- [ ] All 5 core contracts (postgres x2, kafka, consul, valkey) validated +- [ ] Custom Valkey handler working +- [ ] Feature flag system operational +- [ ] Shadow mode comparison passing (>99% consistency) +- [ ] Performance within 5% of Legacy baseline +- [ ] All existing tests passing with contract-driven nodes +- [ ] Legacy nodes renamed and deprecation warnings active + +### 7.2 Quality Criteria + +- [ ] Contract schema validation passing +- [ ] mypy --strict on new code +- [ ] Test coverage >85% for contract-driven wrappers +- [ ] Documentation for contract authoring +- [ ] Runbook for rollback to Legacy procedures +- [ ] Clear deprecation timeline documented + +### 7.3 Operational Criteria + +- [ ] Metrics dashboards for contract-driven vs Legacy comparison +- [ ] Alerting for circuit breaker opens +- [ ] DLQ monitoring configured +- [ ] Health check endpoints including contract validation +- [ ] Deprecation warning logs for Legacy usage + +--- + +## Appendix A: Contract Validation + +```python +"""Utility for validating effect contracts.""" + +import json +import yaml +from pathlib import Path +from jsonschema import validate, ValidationError + +SCHEMA_PATH = Path(__file__).parent / "_schema" / "effect_contract_schema.json" + +def load_schema() -> dict: + """Load the effect contract JSON schema.""" + with open(SCHEMA_PATH) as f: + return json.load(f) + +def validate_contract(contract_path: Path) -> list[str]: + """ + Validate an effect contract against the schema. + + Returns: + List of validation errors (empty if valid) + """ + errors = [] + + try: + with open(contract_path) as f: + contract = yaml.safe_load(f) + + schema = load_schema() + validate(instance=contract, schema=schema) + + except ValidationError as e: + errors.append(f"Schema validation failed: {e.message}") + errors.append(f" Path: {'.'.join(str(p) for p in e.path)}") + + except yaml.YAMLError as e: + errors.append(f"YAML parsing failed: {e}") + + except Exception as e: + errors.append(f"Unexpected error: {e}") + + return errors + +def validate_all_contracts(contracts_dir: Path) -> dict[str, list[str]]: + """Validate all contracts in a directory.""" + results = {} + + for contract_file in contracts_dir.glob("*.yaml"): + if contract_file.name.startswith("_"): + continue # Skip schema/internal files + + errors = validate_contract(contract_file) + results[contract_file.name] = errors + + return results +``` + +--- + +## Appendix B: Quick Reference + +### Environment Variables + +| Variable | Purpose | Default | +|----------|---------|---------| +| `USE_CONTRACT_DRIVEN_EFFECTS` | Migration mode | `contract` | +| `USE_CONTRACT_DRIVEN_{NODE_NAME}` | Per-node override | `true` | +| `POSTGRES_HOST` | PostgreSQL host | `localhost` | +| `POSTGRES_PORT` | PostgreSQL port | `5432` | +| `POSTGRES_DB` | Database name | `omnibase_infra` | +| `POSTGRES_USER` | Database user | `omni` | +| `POSTGRES_PASSWORD` | Database password | (required) | +| `KAFKA_BOOTSTRAP_SERVERS` | Kafka brokers | `localhost:9092` | +| `CONSUL_HOST` | Consul host | `localhost` | +| `CONSUL_PORT` | Consul port | `8500` | +| `CONSUL_ACL_TOKEN` | Consul ACL token | (optional) | +| `VALKEY_HOST` | Valkey host | `localhost` | +| `VALKEY_PORT` | Valkey port | `6379` | + +### CLI Commands + +```bash +# Validate all contracts +poetry run python -m omnibase_infra.contracts.validate + +# Run with contract-driven effects (DEFAULT in v0.4.0+) +USE_CONTRACT_DRIVEN_EFFECTS=contract poetry run pytest + +# Run with Legacy effects (deprecated) +USE_CONTRACT_DRIVEN_EFFECTS=legacy poetry run pytest + +# Run shadow mode comparison +USE_CONTRACT_DRIVEN_EFFECTS=shadow poetry run pytest tests/integration/ + +# Enable specific node for contract-driven +USE_CONTRACT_DRIVEN_POSTGRES_WORKFLOW=true poetry run pytest + +# Force Legacy for specific node (deprecated) +USE_CONTRACT_DRIVEN_POSTGRES_WORKFLOW=false poetry run pytest +``` + +### Class Names Quick Reference + +| Standard Name (v0.4.0+) | Legacy Name (Deprecated) | Removal | +|-------------------------|--------------------------|---------| +| `NodeEffect` | `NodeEffectLegacy` | v0.5.0 | +| `NodePostgresAdapter` | `NodePostgresAdapterLegacy` | v0.5.0 | +| `NodeKafkaAdapter` | `NodeKafkaAdapterLegacy` | v0.5.0 | +| `NodeConsulAdapter` | `NodeConsulAdapterLegacy` | v0.5.0 | +| `NodeVaultAdapterEffect` | `NodeVaultAdapterEffectLegacy` | v0.5.0 | + +--- + +*This plan was created on December 2, 2025 and should be reviewed when omnibase-core 0.4.0 is released. Naming convention updated to make contract-driven the default (December 2025).* diff --git a/docs/architecture/RUNTIME_HOST_IMPLEMENTATION_PLAN.md b/docs/architecture/RUNTIME_HOST_IMPLEMENTATION_PLAN.md new file mode 100644 index 0000000000..2842d46017 --- /dev/null +++ b/docs/architecture/RUNTIME_HOST_IMPLEMENTATION_PLAN.md @@ -0,0 +1,2204 @@ +# Runtime Host Architecture Implementation Plan + +**Created**: December 3, 2025 +**Updated**: December 3, 2025 (Architectural Refinements) +**Status**: Ready for Execution +**Estimated Duration**: 6-8 weeks +**Dependencies**: omnibase_core ^0.3.5, omnibase_spi ^0.2.0 + +--- + +## Executive Summary + +This plan details the complete implementation of the ONEX Runtime Host architecture, transitioning from a 1-container-per-node model to a unified Runtime Host model. The implementation spans three repositories (omnibase_core, omnibase_spi, omnibase_infra) with strict dependency ordering. + +### Key Architectural Invariants + +These invariants MUST be maintained throughout the implementation: + +1. **Core is transport-agnostic**: `omnibase_core` has NO Kafka/HTTP/DB/Vault imports anywhere +2. **NodeRuntime is a pure in-memory orchestrator**: It does NOT own any event loop or bus consumer +3. **Event bus consumption is an infra concern**: `RuntimeHostProcess` + `ProtocolEventBus` drive the runtime +4. **Handlers use strong typing**: `handler_type` returns `EnumHandlerType`, not `str` +5. **LocalHandler is dev/test-only**: Never used in production contracts +6. **Single source of truth for handler registry**: No duplicate registration logic + +### Architecture Overview + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ RuntimeHostProcess (omnibase_infra) │ +│ ┌────────────────────────────────────────────────────────────┐ │ +│ │ KafkaEventBus (ProtocolEventBus impl) │ │ +│ │ Consumes envelopes → calls runtime.route_envelope(...) │ │ +│ └────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌────────────────────────────────────────────────────────────┐ │ +│ │ NodeRuntime (omnibase_core) │ │ +│ │ Pure in-memory orchestrator - NO event loop, NO I/O │ │ +│ │ │ │ +│ │ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ │ +│ │ │NodeInstance │ │NodeInstance │ │NodeInstance │ ... │ │ +│ │ │ vault_ │ │ consul_ │ │ postgres_ │ │ │ +│ │ │ adapter │ │ projector │ │ adapter │ │ │ +│ │ └─────────────┘ └─────────────┘ └─────────────┘ │ │ +│ │ │ │ +│ │ ┌──────────────────────────────────────────────────────┐ │ │ +│ │ │ Handlers: [local*] [http] [db] [vault] [consul] │ │ │ +│ │ │ *local is dev/test only │ │ │ +│ │ └──────────────────────────────────────────────────────┘ │ │ +│ └────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌────────────────────────────────────────────────────────────┐ │ +│ │ FileRegistry | HealthEndpoint | MetricsCollector │ │ +│ └────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +### Handler vs Event Bus Separation + +| Abstraction | Purpose | Location | Example | +|-------------|---------|----------|---------| +| `ProtocolHandler` | Per-request operations (db/http/vault/etc.) | SPI protocol, infra impl | `HttpHandler`, `DbHandler` | +| `ProtocolEventBus` | Message transport feeding envelopes into runtime | SPI protocol, infra impl | `KafkaEventBus` | + +**Key distinction**: +- `ProtocolHandler` = "do this specific thing for me" (synchronous request/response) +- `ProtocolEventBus` = "deliver messages to me" (async event consumption) + +--- + +## Phase 0: Prerequisites & Validation + +**Duration**: 1-2 days +**Repository**: All + +### 0.1 Dependency Validation + +```bash +# Verify PyPI packages +pip index versions omnibase-core # Should show ≥0.3.5 +pip index versions omnibase-spi # Should show ≥0.2.0 + +# Test current imports +python -c "from omnibase_core.nodes import NodeEffect; print('Core OK')" +python -c "from omnibase_spi.protocols import ProtocolEventBus; print('SPI OK')" +``` + +### 0.2 Repository State Verification + +| Repository | Branch | State Required | +|------------|--------|----------------| +| omnibase_core | main | Clean, passing CI | +| omnibase_spi | main | Clean, passing CI | +| omnibase_infra | main | Fresh setup complete | + +### 0.3 Success Criteria +- [ ] All dependencies installable from PyPI +- [ ] Import tests pass +- [ ] All repositories have clean main branches +- [ ] Development environments configured + +--- + +## Phase 1: Core Types (omnibase_core) + +**Duration**: 1 week +**Repository**: omnibase_core +**Priority**: CRITICAL - Must complete before all other phases + +### 1.1 New Enum Values + +**File**: `src/omnibase_core/enums/enum_node_kind.py` + +```python +from enum import Enum + +class EnumNodeKind(str, Enum): + """Types of ONEX nodes.""" + EFFECT = "effect" + COMPUTE = "compute" + REDUCER = "reducer" + ORCHESTRATOR = "orchestrator" + RUNTIME_HOST = "runtime_host" # NEW +``` + +**File**: `src/omnibase_core/enums/enum_handler_type.py` (NEW) + +```python +from enum import Enum + +class EnumHandlerType(str, Enum): + """Types of protocol handlers for per-request operations. + + Note: Kafka is NOT in this enum. Kafka is an event bus (ProtocolEventBus), + not a per-request handler (ProtocolHandler). Use ProtocolEventBus for + message consumption and delivery. + """ + LOCAL = "local" # Echo/test - dev/test only, no external deps + HTTP = "http" # HTTP REST calls + DB = "db" # Database operations + LLM = "llm" # LLM API calls + VAULT = "vault" # Vault secret management + CONSUL = "consul" # Consul service discovery +``` + +### 1.2 OnexEnvelope Model + +**File**: `src/omnibase_core/models/runtime/model_onex_envelope.py` (NEW) + +```python +"""Unified message envelope for all node communication.""" +from __future__ import annotations + +from datetime import datetime, UTC +from typing import Any +from uuid import UUID, uuid4 + +from pydantic import BaseModel, Field + +from omnibase_core.enums.enum_handler_type import EnumHandlerType + + +class ModelOnexEnvelope(BaseModel): + """Unified message envelope for Runtime Host communication. + + All communication between nodes, handlers, and external systems + flows through this envelope format. + """ + + # Identity + envelope_id: UUID = Field(default_factory=uuid4, description="Unique envelope identifier") + envelope_version: str = Field(default="1.0.0", description="Envelope schema version") + + # Correlation + correlation_id: UUID = Field(default_factory=uuid4, description="Request correlation ID") + causation_id: UUID | None = Field(default=None, description="ID of envelope that caused this one") + + # Routing + source_node: str = Field(description="Source node slug") + target_node: str | None = Field(default=None, description="Target node slug (None for broadcast)") + handler_type: EnumHandlerType = Field(description="Handler type for processing") + operation: str = Field(description="Operation to perform") + + # Payload + payload: dict[str, Any] = Field(default_factory=dict, description="Operation payload") + metadata: dict[str, Any] = Field(default_factory=dict, description="Additional metadata") + + # Timing + timestamp: datetime = Field(default_factory=lambda: datetime.now(UTC), description="Creation timestamp") + ttl_seconds: int | None = Field(default=None, description="Time-to-live in seconds") + + # Response (for reply envelopes) + is_response: bool = Field(default=False, description="Whether this is a response envelope") + success: bool | None = Field(default=None, description="Operation success (for responses)") + error: str | None = Field(default=None, description="Error message (for failures)") + + class Config: + json_encoders = { + datetime: lambda v: v.isoformat(), + UUID: str, + } +``` + +### 1.3 RuntimeHostContract Model + +**File**: `src/omnibase_core/models/contracts/model_runtime_host_contract.py` (NEW) + +```python +"""Runtime Host contract definition.""" +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from pydantic import BaseModel, Field + +from omnibase_core.enums.enum_handler_type import EnumHandlerType + + +class ModelHandlerConfig(BaseModel): + """Configuration for a protocol handler. + + TODO (post-MVP): Introduce per-handler config models (e.g. ModelHttpHandlerConfig, + ModelDbHandlerConfig) and validate `config` against them, instead of using + free-form `dict[str, Any]`. + """ + handler_type: EnumHandlerType + enabled: bool = True + config: dict[str, Any] = Field(default_factory=dict) + + +class ModelEventBusConfig(BaseModel): + """Configuration for the event bus. + + The event bus is separate from handlers - it's the transport that + feeds envelopes into NodeRuntime via RuntimeHostProcess. + """ + enabled: bool = True + # Transport-agnostic config - infra provides concrete implementation + config: dict[str, Any] = Field(default_factory=dict) + + +class ModelNodeRef(BaseModel): + """Reference to a node to be loaded.""" + slug: str = Field(description="Node slug identifier") + contract_path: str = Field(description="Path to node contract YAML") + enabled: bool = True + config_overrides: dict[str, Any] = Field(default_factory=dict) + + +class ModelRuntimeHostContract(BaseModel): + """Contract defining a Runtime Host configuration. + + The Runtime Host contract specifies which nodes to load, + which handlers to enable, and how to configure them. + + Note: Event bus configuration is separate from handlers. + Handlers are for per-request operations; the event bus is + for message transport. + """ + + # Identity + name: str = Field(description="Runtime host name") + version: str = Field(default="1.0.0", description="Contract version") + description: str = Field(default="", description="Human-readable description") + + # Node Configuration + nodes: list[ModelNodeRef] = Field(default_factory=list, description="Nodes to load") + contracts_directory: str = Field(default="contracts", description="Base directory for contracts") + + # Handler Configuration (for per-request operations) + handlers: list[ModelHandlerConfig] = Field(default_factory=list, description="Handler configurations") + + # Event Bus Configuration (transport-agnostic) + event_bus: ModelEventBusConfig = Field( + default_factory=ModelEventBusConfig, + description="Event bus config (transport implemented in infra)" + ) + + # Health & Metrics + health_endpoint: dict[str, Any] = Field( + default_factory=lambda: {"enabled": True, "port": 8080, "path": "/health"} + ) + metrics_endpoint: dict[str, Any] = Field( + default_factory=lambda: {"enabled": True, "port": 9090, "path": "/metrics"} + ) + + # Runtime Settings + max_concurrent_operations: int = Field(default=100, description="Max concurrent operations") + shutdown_timeout_seconds: int = Field(default=30, description="Graceful shutdown timeout") + + @classmethod + def from_yaml(cls, path: Path) -> "ModelRuntimeHostContract": + """Load contract from YAML file.""" + from omnibase_core.utils.util_safe_yaml_loader import load_and_validate_yaml_model + return load_and_validate_yaml_model(path, cls) +``` + +### 1.4 Handler Protocol Interface (Strongly Typed) + +**File**: `src/omnibase_core/protocols/protocol_handler.py` (NEW) + +```python +"""Protocol interface for Runtime Host handlers.""" +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +from omnibase_core.enums.enum_handler_type import EnumHandlerType + +if TYPE_CHECKING: + from omnibase_core.models.runtime.model_onex_envelope import ModelOnexEnvelope + + +class ProtocolHandler(ABC): + """Abstract protocol for Runtime Host handlers. + + Handlers are responsible for executing per-request operations defined in + OnexEnvelopes. Each handler type (http, db, vault, etc.) implements + this protocol to provide its specific functionality. + + Note: This is for synchronous request/response operations. + For message transport (event consumption/production), use ProtocolEventBus. + """ + + @property + @abstractmethod + def handler_type(self) -> EnumHandlerType: + """Return the handler type identifier. + + Returns: + EnumHandlerType - strongly typed, not a string + """ + ... + + @abstractmethod + async def initialize(self, config: dict) -> None: + """Initialize handler with configuration. + + Args: + config: Handler-specific configuration dictionary + """ + ... + + @abstractmethod + async def shutdown(self) -> None: + """Gracefully shutdown the handler.""" + ... + + @abstractmethod + async def execute(self, envelope: "ModelOnexEnvelope") -> "ModelOnexEnvelope": + """Execute an operation from an envelope. + + Args: + envelope: Input envelope with operation details + + Returns: + Response envelope with operation results + """ + ... + + @abstractmethod + async def health_check(self) -> dict: + """Check handler health. + + Returns: + Dict with 'healthy' bool and optional details + """ + ... +``` + +### 1.5 NodeInstance Class + +**File**: `src/omnibase_core/runtime/node_instance.py` (NEW) + +```python +"""Lightweight node instance wrapper for Runtime Host.""" +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any, Callable + +from omnibase_core.models.runtime.model_onex_envelope import ModelOnexEnvelope + +if TYPE_CHECKING: + from omnibase_core.models.contracts.model_node_contract import ModelNodeContract + from omnibase_core.runtime.node_runtime import NodeRuntime + + +class NodeInstance: + """Lightweight wrapper around node business logic. + + NodeInstance represents a single node within the Runtime Host. + It contains no event loop or I/O code - all operations are + delegated to handlers through the parent runtime. + + Implementation Note: + Currently, NodeInstance delegates all operation execution to NodeRuntime, + which in turn routes to the appropriate ProtocolHandler based on + `envelope.handler_type`. This keeps NodeInstance free of I/O details + and maintains the architectural invariant that nodes are pure logic. + """ + + def __init__( + self, + contract: "ModelNodeContract", + runtime: "NodeRuntime", + ) -> None: + self._contract = contract + self._runtime = runtime + self._logger = logging.getLogger(f"node.{contract.name}") + self._initialized = False + self._operation_handlers: dict[str, Callable] = {} + + @property + def slug(self) -> str: + """Return node slug identifier.""" + return self._contract.name + + @property + def node_type(self) -> str: + """Return node type (effect, compute, reducer, orchestrator).""" + return self._contract.node_type + + @property + def contract(self) -> "ModelNodeContract": + """Return the node contract.""" + return self._contract + + async def initialize(self) -> None: + """Initialize the node instance.""" + self._logger.info(f"Initializing node instance: {self.slug}") + + # Register operation handlers from contract + for op in self._contract.io_operations or []: + self._operation_handlers[op.operation] = self._create_operation_handler(op) + + self._initialized = True + self._logger.info(f"Node instance initialized: {self.slug}") + + async def shutdown(self) -> None: + """Shutdown the node instance.""" + self._logger.info(f"Shutting down node instance: {self.slug}") + self._initialized = False + + async def handle(self, envelope: ModelOnexEnvelope) -> ModelOnexEnvelope: + """Handle an incoming envelope. + + Delegates execution to NodeRuntime which routes to the appropriate + ProtocolHandler based on envelope.handler_type. + + Args: + envelope: Input envelope to process + + Returns: + Response envelope with operation results + """ + if not self._initialized: + return self._error_response(envelope, "Node not initialized") + + operation = envelope.operation + + # Validate operation is supported + if operation not in self._operation_handlers: + return self._error_response( + envelope, + f"Unknown operation: {operation}. Available: {list(self._operation_handlers.keys())}" + ) + + # Delegate to runtime for handler execution + try: + return await self._runtime.execute_with_handler(envelope) + except Exception as e: + self._logger.exception(f"Error handling operation {operation}") + return self._error_response(envelope, str(e)) + + def _create_operation_handler(self, operation_config: Any) -> Callable: + """Create a handler function for an operation.""" + async def handler(envelope: ModelOnexEnvelope) -> ModelOnexEnvelope: + return await self._runtime.execute_with_handler(envelope) + return handler + + def _error_response(self, envelope: ModelOnexEnvelope, error: str) -> ModelOnexEnvelope: + """Create an error response envelope.""" + return ModelOnexEnvelope( + correlation_id=envelope.correlation_id, + causation_id=envelope.envelope_id, + source_node=self.slug, + target_node=envelope.source_node, + handler_type=envelope.handler_type, + operation=envelope.operation, + is_response=True, + success=False, + error=error, + ) +``` + +### 1.6 NodeRuntime Class (Transport-Agnostic) + +**File**: `src/omnibase_core/runtime/node_runtime.py` (NEW) + +```python +"""Core NodeRuntime implementation for Runtime Host.""" +from __future__ import annotations + +import logging +from pathlib import Path +from typing import TYPE_CHECKING + +from omnibase_core.enums.enum_handler_type import EnumHandlerType +from omnibase_core.models.runtime.model_onex_envelope import ModelOnexEnvelope +from omnibase_core.runtime.node_instance import NodeInstance + +if TYPE_CHECKING: + from omnibase_core.models.contracts.model_runtime_host_contract import ModelRuntimeHostContract + from omnibase_core.protocols.protocol_handler import ProtocolHandler + + +class NodeRuntime: + """Core runtime that hosts multiple node instances. + + NodeRuntime is a pure in-memory orchestrator. It manages node instances, + routes operations to handlers, and provides shared infrastructure for + all hosted nodes. + + IMPORTANT ARCHITECTURAL INVARIANT: + NodeRuntime does NOT own any event loop or message consumer. + Event bus consumption is implemented in RuntimeHostProcess (omnibase_infra) + using ProtocolEventBus. RuntimeHostProcess calls route_envelope() for + each received message. + + This separation ensures: + - Core remains transport-agnostic (no Kafka/NATS/etc. imports) + - Testing is simplified (no real message bus needed) + - Different transports can be swapped in infra without touching core + """ + + def __init__(self, contract: "ModelRuntimeHostContract") -> None: + self._contract = contract + self._logger = logging.getLogger(f"runtime.{contract.name}") + + # Node instances + self._nodes: dict[str, NodeInstance] = {} + + # Handlers (for per-request operations) + self._handlers: dict[EnumHandlerType, "ProtocolHandler"] = {} + + # State + self._running = False + + @property + def name(self) -> str: + """Return runtime name.""" + return self._contract.name + + @property + def nodes(self) -> dict[str, NodeInstance]: + """Return registered node instances.""" + return self._nodes + + @property + def handlers(self) -> dict[EnumHandlerType, "ProtocolHandler"]: + """Return registered handlers.""" + return self._handlers + + @property + def is_running(self) -> bool: + """Return whether runtime is running.""" + return self._running + + def register_handler(self, handler: "ProtocolHandler") -> None: + """Register a protocol handler. + + Args: + handler: Handler instance to register (must return EnumHandlerType) + """ + # handler.handler_type is now EnumHandlerType, no conversion needed + handler_type = handler.handler_type + self._handlers[handler_type] = handler + self._logger.info(f"Registered handler: {handler_type}") + + def register_node(self, node: NodeInstance) -> None: + """Register a node instance. + + Args: + node: Node instance to register + """ + self._nodes[node.slug] = node + self._logger.info(f"Registered node: {node.slug}") + + async def load_nodes_from_directory(self, contracts_dir: Path) -> None: + """Load all node contracts from a directory. + + Args: + contracts_dir: Directory containing node contract YAML files + """ + from omnibase_core.runtime.file_registry import FileRegistry + + registry = FileRegistry(contracts_dir) + contracts = registry.load_all() + + for contract in contracts: + node = NodeInstance(contract, self) + self.register_node(node) + + async def initialize(self) -> None: + """Initialize the runtime and all components.""" + self._logger.info(f"Initializing runtime: {self.name}") + + # Initialize handlers + for handler_type, handler in self._handlers.items(): + config = self._get_handler_config(handler_type) + await handler.initialize(config) + self._logger.info(f"Handler initialized: {handler_type}") + + # Initialize nodes + for slug, node in self._nodes.items(): + await node.initialize() + self._logger.info(f"Node initialized: {slug}") + + self._running = True + self._logger.info(f"Runtime initialized with {len(self._nodes)} nodes and {len(self._handlers)} handlers") + + async def shutdown(self) -> None: + """Shutdown the runtime gracefully.""" + self._logger.info(f"Shutting down runtime: {self.name}") + self._running = False + + # Shutdown nodes + for slug, node in self._nodes.items(): + await node.shutdown() + self._logger.info(f"Node shutdown: {slug}") + + # Shutdown handlers + for handler_type, handler in self._handlers.items(): + await handler.shutdown() + self._logger.info(f"Handler shutdown: {handler_type}") + + self._logger.info(f"Runtime stopped: {self.name}") + + async def execute_with_handler(self, envelope: ModelOnexEnvelope) -> ModelOnexEnvelope: + """Execute an envelope using the appropriate handler. + + Args: + envelope: Envelope to execute + + Returns: + Response envelope + """ + handler_type = envelope.handler_type + + if handler_type not in self._handlers: + return ModelOnexEnvelope( + correlation_id=envelope.correlation_id, + causation_id=envelope.envelope_id, + source_node="runtime", + handler_type=handler_type, + operation=envelope.operation, + is_response=True, + success=False, + error=f"No handler registered for type: {handler_type}", + ) + + handler = self._handlers[handler_type] + return await handler.execute(envelope) + + async def route_envelope(self, envelope: ModelOnexEnvelope) -> ModelOnexEnvelope: + """Route an envelope to the appropriate node. + + This is the main entry point for envelopes coming from the event bus. + RuntimeHostProcess calls this method for each message received. + + Args: + envelope: Envelope to route + + Returns: + Response envelope + """ + target = envelope.target_node + + if target is None: + # Broadcast to all nodes (not typical) + return await self._broadcast_envelope(envelope) + + if target not in self._nodes: + return ModelOnexEnvelope( + correlation_id=envelope.correlation_id, + causation_id=envelope.envelope_id, + source_node="runtime", + handler_type=envelope.handler_type, + operation=envelope.operation, + is_response=True, + success=False, + error=f"Unknown target node: {target}", + ) + + node = self._nodes[target] + return await node.handle(envelope) + + async def _broadcast_envelope(self, envelope: ModelOnexEnvelope) -> ModelOnexEnvelope: + """Broadcast envelope to all nodes.""" + results = [] + for node in self._nodes.values(): + result = await node.handle(envelope) + results.append(result) + + return ModelOnexEnvelope( + correlation_id=envelope.correlation_id, + causation_id=envelope.envelope_id, + source_node="runtime", + handler_type=envelope.handler_type, + operation=envelope.operation, + is_response=True, + success=all(r.success for r in results if r.success is not None), + payload={"results": [r.model_dump() for r in results]}, + ) + + def _get_handler_config(self, handler_type: EnumHandlerType) -> dict: + """Get configuration for a handler type.""" + for handler_config in self._contract.handlers: + if handler_config.handler_type == handler_type: + return handler_config.config + return {} + + async def health_check(self) -> dict: + """Check health of runtime and all components.""" + handler_health = {} + for handler_type, handler in self._handlers.items(): + handler_health[handler_type.value] = await handler.health_check() + + node_health = {} + for slug, node in self._nodes.items(): + node_health[slug] = {"initialized": node._initialized} + + return { + "runtime": self.name, + "running": self._running, + "handlers": handler_health, + "nodes": node_health, + "healthy": self._running and all( + h.get("healthy", False) for h in handler_health.values() + ), + } +``` + +### 1.7 FileRegistry Class + +**File**: `src/omnibase_core/runtime/file_registry.py` (NEW) + +```python +"""File-based contract registry for Runtime Host.""" +from __future__ import annotations + +import logging +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from omnibase_core.models.contracts.model_node_contract import ModelNodeContract + + +class FileRegistry: + """Registry that loads node contracts from filesystem. + + FileRegistry scans a directory for YAML contract files and + loads them into ModelNodeContract instances. + """ + + def __init__(self, contracts_dir: Path) -> None: + self._contracts_dir = contracts_dir + self._contracts: dict[str, "ModelNodeContract"] = {} + self._logger = logging.getLogger("file_registry") + + def load_all(self) -> list["ModelNodeContract"]: + """Load all contracts from the directory. + + Returns: + List of loaded node contracts + """ + from omnibase_core.models.contracts.model_node_contract import ModelNodeContract + from omnibase_core.utils.util_safe_yaml_loader import load_and_validate_yaml_model + + contracts = [] + + if not self._contracts_dir.exists(): + self._logger.warning(f"Contracts directory does not exist: {self._contracts_dir}") + return contracts + + # Find all YAML files + for yaml_file in self._contracts_dir.glob("**/*.yaml"): + if yaml_file.name.startswith("_"): + continue # Skip private/schema files + + try: + contract = load_and_validate_yaml_model(yaml_file, ModelNodeContract) + self._contracts[contract.name] = contract + contracts.append(contract) + self._logger.info(f"Loaded contract: {contract.name} from {yaml_file}") + except Exception as e: + self._logger.error(f"Failed to load contract from {yaml_file}: {e}") + + return contracts + + def get(self, slug: str) -> "ModelNodeContract | None": + """Get a contract by slug. + + Args: + slug: Node slug identifier + + Returns: + Contract if found, None otherwise + """ + return self._contracts.get(slug) + + def list_all(self) -> list[str]: + """List all loaded contract slugs.""" + return list(self._contracts.keys()) +``` + +### 1.8 Local Handler (Dev/Test Only) + +**File**: `src/omnibase_core/runtime/handlers/local_handler.py` (NEW) + +```python +"""Local echo handler for testing - no external dependencies.""" +from __future__ import annotations + +import logging +from typing import Any + +from omnibase_core.enums.enum_handler_type import EnumHandlerType +from omnibase_core.models.runtime.model_onex_envelope import ModelOnexEnvelope +from omnibase_core.protocols.protocol_handler import ProtocolHandler + + +class LocalHandler(ProtocolHandler): + """Local echo handler for testing and development only. + + WARNING: + This handler is not intended for production use. It exists to + validate Runtime Host wiring without external dependencies. + Production handlers (http/db/vault/etc.) are all implemented + in `omnibase_infra`. + + This handler has no external dependencies and simply echoes + back the input payload. Used for testing the runtime infrastructure. + """ + + def __init__(self) -> None: + self._logger = logging.getLogger("handler.local") + self._initialized = False + self._config: dict[str, Any] = {} + + @property + def handler_type(self) -> EnumHandlerType: + """Return handler type as EnumHandlerType (not str).""" + return EnumHandlerType.LOCAL + + async def initialize(self, config: dict) -> None: + """Initialize the local handler.""" + self._config = config + self._initialized = True + self._logger.info("Local handler initialized (dev/test only)") + + async def shutdown(self) -> None: + """Shutdown the local handler.""" + self._initialized = False + self._logger.info("Local handler shutdown") + + async def execute(self, envelope: ModelOnexEnvelope) -> ModelOnexEnvelope: + """Echo back the envelope payload. + + Supports operations: + - echo: Return payload as-is + - transform: Apply simple transformations + - error: Simulate an error response + """ + operation = envelope.operation + + if operation == "echo": + return self._success_response(envelope, envelope.payload) + + elif operation == "transform": + # Simple transformation: uppercase all string values + transformed = self._transform_payload(envelope.payload) + return self._success_response(envelope, transformed) + + elif operation == "error": + error_msg = envelope.payload.get("error_message", "Simulated error") + return self._error_response(envelope, error_msg) + + else: + return self._success_response(envelope, { + "operation": operation, + "payload": envelope.payload, + "message": "Unknown operation, echoing back", + }) + + async def health_check(self) -> dict: + """Check handler health.""" + return { + "healthy": self._initialized, + "handler_type": self.handler_type.value, + "dev_test_only": True, + } + + def _success_response(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: + """Create a success response envelope.""" + return ModelOnexEnvelope( + correlation_id=envelope.correlation_id, + causation_id=envelope.envelope_id, + source_node="local_handler", + target_node=envelope.source_node, + handler_type=envelope.handler_type, + operation=envelope.operation, + payload=payload, + is_response=True, + success=True, + ) + + def _error_response(self, envelope: ModelOnexEnvelope, error: str) -> ModelOnexEnvelope: + """Create an error response envelope.""" + return ModelOnexEnvelope( + correlation_id=envelope.correlation_id, + causation_id=envelope.envelope_id, + source_node="local_handler", + target_node=envelope.source_node, + handler_type=envelope.handler_type, + operation=envelope.operation, + is_response=True, + success=False, + error=error, + ) + + def _transform_payload(self, payload: dict) -> dict: + """Transform payload values.""" + result = {} + for key, value in payload.items(): + if isinstance(value, str): + result[key] = value.upper() + elif isinstance(value, dict): + result[key] = self._transform_payload(value) + else: + result[key] = value + return result +``` + +### 1.9 CLI Entry Point (Dev/Test) + +**File**: `src/omnibase_core/cli/runtime_host_cli.py` (NEW) + +```python +"""CLI for running the Runtime Host (dev/test mode - local handler only). + +For production use, use omnibase_infra.runtime.runtime_host_process which +provides real handlers and event bus integration. +""" +from __future__ import annotations + +import argparse +import asyncio +import logging +import sys +from pathlib import Path + +from omnibase_core.models.contracts.model_runtime_host_contract import ModelRuntimeHostContract +from omnibase_core.runtime.node_runtime import NodeRuntime +from omnibase_core.runtime.handlers.local_handler import LocalHandler + + +def setup_logging(level: str = "INFO") -> None: + """Configure logging.""" + logging.basicConfig( + level=getattr(logging, level.upper()), + format="%(asctime)s | %(levelname)-8s | %(name)s | %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + + +async def run_runtime(contract_path: Path) -> None: + """Run the runtime host with the given contract (dev/test mode). + + This CLI only registers LocalHandler for testing purposes. + For production, use RuntimeHostProcess from omnibase_infra. + """ + # Load contract + contract = ModelRuntimeHostContract.from_yaml(contract_path) + + # Create runtime + runtime = NodeRuntime(contract) + + # Register local handler only (dev/test) + runtime.register_handler(LocalHandler()) + + # Load nodes + contracts_dir = contract_path.parent / contract.contracts_directory + if contracts_dir.exists(): + await runtime.load_nodes_from_directory(contracts_dir) + + # Initialize + await runtime.initialize() + + logging.info("Runtime initialized (dev/test mode - LocalHandler only)") + logging.info("For production, use omnibase_infra.runtime.runtime_host_process") + + # Keep running until interrupted + try: + while runtime.is_running: + await asyncio.sleep(1) + except KeyboardInterrupt: + pass + finally: + await runtime.shutdown() + + +def main() -> None: + """Main entry point.""" + parser = argparse.ArgumentParser( + description="ONEX Runtime Host (dev/test mode)", + epilog="For production use, see omnibase_infra.runtime.runtime_host_process", + ) + parser.add_argument( + "contract", + type=Path, + help="Path to runtime host contract YAML", + ) + parser.add_argument( + "--log-level", + default="INFO", + choices=["DEBUG", "INFO", "WARNING", "ERROR"], + help="Logging level", + ) + + args = parser.parse_args() + + setup_logging(args.log_level) + + if not args.contract.exists(): + print(f"Contract file not found: {args.contract}", file=sys.stderr) + sys.exit(1) + + asyncio.run(run_runtime(args.contract)) + + +if __name__ == "__main__": + main() +``` + +### 1.10 Phase 1 Success Criteria + +- [ ] `EnumNodeKind.RUNTIME_HOST` added and exported +- [ ] `EnumHandlerType` enum created (WITHOUT Kafka - that's an event bus) +- [ ] `ModelOnexEnvelope` model complete with serialization +- [ ] `ModelRuntimeHostContract` model with separate event_bus config +- [ ] `ProtocolHandler` abstract class with `EnumHandlerType` return type +- [ ] `NodeInstance` class with envelope handling (pass-through documented) +- [ ] `NodeRuntime` class - NO event loop, NO bus consumer +- [ ] `FileRegistry` class for contract loading +- [ ] `LocalHandler` working with dev/test warnings +- [ ] CLI entry point clearly marked as dev/test only +- [ ] Unit tests for all new classes (>90% coverage) +- [ ] mypy --strict passes +- [ ] **VERIFICATION**: No Kafka/HTTP/DB/Vault imports in omnibase_core + +--- + +## Phase 2: SPI Protocol Updates (omnibase_spi) + +**Duration**: 3-4 days +**Repository**: omnibase_spi +**Depends On**: Phase 1 complete + +### 2.1 Handler Protocol Export + +**File**: `src/omnibase_spi/protocols/__init__.py` (UPDATE) + +```python +# Add to existing exports +from omnibase_core.protocols.protocol_handler import ProtocolHandler + +__all__ = [ + # ... existing exports + "ProtocolHandler", +] +``` + +### 2.2 Event Bus Protocol Updates + +**File**: `src/omnibase_spi/protocols/protocol_event_bus.py` (UPDATE) + +The event bus protocol handles message transport - separate from handlers: + +```python +"""Protocol for event bus implementations.""" +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Callable, Awaitable + +if TYPE_CHECKING: + from omnibase_core.models.runtime.model_onex_envelope import ModelOnexEnvelope + + +class ProtocolEventBus(ABC): + """Abstract protocol for event bus implementations. + + The event bus is responsible for message transport - delivering envelopes + to NodeRuntime via RuntimeHostProcess. This is separate from ProtocolHandler + which handles per-request operations. + + Implementation Note: + - KafkaEventBus implements this protocol in omnibase_infra + - RuntimeHostProcess owns the event bus and calls runtime.route_envelope() + - NodeRuntime is transport-agnostic and only knows about envelopes + """ + + @abstractmethod + async def initialize(self, config: dict) -> None: + """Initialize the event bus connection.""" + ... + + @abstractmethod + async def shutdown(self) -> None: + """Gracefully shutdown the event bus.""" + ... + + @abstractmethod + async def publish_envelope(self, envelope: "ModelOnexEnvelope", topic: str) -> None: + """Publish an OnexEnvelope to a topic. + + Args: + envelope: Envelope to publish + topic: Target topic name + """ + ... + + @abstractmethod + async def subscribe( + self, + topic: str, + handler: Callable[["ModelOnexEnvelope"], Awaitable["ModelOnexEnvelope"]], + ) -> None: + """Subscribe to envelopes on a topic. + + Args: + topic: Topic to subscribe to + handler: Async callback for each received envelope + """ + ... + + @abstractmethod + async def start_consuming(self) -> None: + """Start the consumer loop. + + This runs until shutdown() is called. + """ + ... + + @abstractmethod + async def health_check(self) -> dict: + """Check event bus health.""" + ... +``` + +### 2.3 Phase 2 Success Criteria + +- [ ] `ProtocolHandler` exported from SPI +- [ ] `ProtocolEventBus` updated with envelope methods +- [ ] Clear separation documented: Handler vs EventBus +- [ ] Backward compatibility maintained +- [ ] Tests passing + +--- + +## Phase 3: Infrastructure Handlers (omnibase_infra) + +**Duration**: 2 weeks +**Repository**: omnibase_infra +**Depends On**: Phase 1 and Phase 2 complete + +### 3.1 Directory Structure + +``` +src/omnibase_infra/ +├── handlers/ # Protocol handlers (per-request ops) +│ ├── __init__.py +│ ├── http_handler.py # HTTP REST handler +│ ├── db_handler.py # PostgreSQL handler +│ ├── vault_handler.py # Vault secrets handler +│ ├── consul_handler.py # Consul service discovery handler +│ └── llm_handler.py # LLM API handler (Phase 4) +├── event_bus/ # Event bus implementations (transport) +│ ├── __init__.py +│ └── kafka_event_bus.py # KafkaEventBus (ProtocolEventBus impl) +├── runtime/ # Runtime host process +│ ├── __init__.py +│ ├── runtime_host_process.py # Main process wrapper +│ └── wiring.py # Single source of truth for handlers +├── contracts/ # Example contracts +│ ├── runtime/ +│ │ └── infra_runtime_host.yaml # Example runtime contract +│ └── nodes/ +│ ├── vault_adapter.yaml +│ ├── consul_adapter.yaml +│ └── postgres_adapter.yaml +└── ... (existing structure) +``` + +### 3.2 HTTP Handler (Strongly Typed) + +**File**: `src/omnibase_infra/handlers/http_handler.py` + +```python +"""HTTP REST protocol handler using httpx.""" +from __future__ import annotations + +import logging +from typing import Any + +import httpx + +from omnibase_core.enums.enum_handler_type import EnumHandlerType +from omnibase_core.models.runtime.model_onex_envelope import ModelOnexEnvelope +from omnibase_core.protocols.protocol_handler import ProtocolHandler + + +class HttpHandler(ProtocolHandler): + """HTTP REST protocol handler. + + Executes HTTP requests defined in OnexEnvelopes. + """ + + def __init__(self) -> None: + self._logger = logging.getLogger("handler.http") + self._client: httpx.AsyncClient | None = None + self._config: dict[str, Any] = {} + + @property + def handler_type(self) -> EnumHandlerType: + """Return handler type as EnumHandlerType.""" + return EnumHandlerType.HTTP + + async def initialize(self, config: dict) -> None: + """Initialize HTTP client.""" + self._config = config + timeout = config.get("timeout_seconds", 30) + + self._client = httpx.AsyncClient( + timeout=httpx.Timeout(timeout), + follow_redirects=True, + ) + self._logger.info("HTTP handler initialized") + + async def shutdown(self) -> None: + """Close HTTP client.""" + if self._client: + await self._client.aclose() + self._logger.info("HTTP handler shutdown") + + async def execute(self, envelope: ModelOnexEnvelope) -> ModelOnexEnvelope: + """Execute HTTP request from envelope.""" + if not self._client: + return self._error_response(envelope, "HTTP client not initialized") + + payload = envelope.payload + + try: + method = payload.get("method", "GET").upper() + url = payload.get("url") + headers = payload.get("headers", {}) + body = payload.get("body") + params = payload.get("params", {}) + + if not url: + return self._error_response(envelope, "URL is required") + + response = await self._client.request( + method=method, + url=url, + headers=headers, + json=body if body else None, + params=params, + ) + + return self._success_response(envelope, { + "status_code": response.status_code, + "headers": dict(response.headers), + "body": response.json() if response.headers.get("content-type", "").startswith("application/json") else response.text, + }) + + except httpx.TimeoutException: + return self._error_response(envelope, "Request timed out") + except Exception as e: + return self._error_response(envelope, str(e)) + + async def health_check(self) -> dict: + """Check HTTP handler health.""" + return { + "healthy": self._client is not None, + "handler_type": self.handler_type.value, + } + + def _success_response(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: + return ModelOnexEnvelope( + correlation_id=envelope.correlation_id, + causation_id=envelope.envelope_id, + source_node="http_handler", + target_node=envelope.source_node, + handler_type=envelope.handler_type, + operation=envelope.operation, + payload=payload, + is_response=True, + success=True, + ) + + def _error_response(self, envelope: ModelOnexEnvelope, error: str) -> ModelOnexEnvelope: + return ModelOnexEnvelope( + correlation_id=envelope.correlation_id, + causation_id=envelope.envelope_id, + source_node="http_handler", + target_node=envelope.source_node, + handler_type=envelope.handler_type, + operation=envelope.operation, + is_response=True, + success=False, + error=error, + ) +``` + +### 3.3 Database Handler (Strongly Typed) + +**File**: `src/omnibase_infra/handlers/db_handler.py` + +```python +"""PostgreSQL database protocol handler.""" +from __future__ import annotations + +import logging +from typing import Any + +import asyncpg + +from omnibase_core.enums.enum_handler_type import EnumHandlerType +from omnibase_core.models.runtime.model_onex_envelope import ModelOnexEnvelope +from omnibase_core.protocols.protocol_handler import ProtocolHandler + + +class DbHandler(ProtocolHandler): + """PostgreSQL database protocol handler. + + Executes database operations defined in OnexEnvelopes. + """ + + def __init__(self) -> None: + self._logger = logging.getLogger("handler.db") + self._pool: asyncpg.Pool | None = None + self._config: dict[str, Any] = {} + + @property + def handler_type(self) -> EnumHandlerType: + """Return handler type as EnumHandlerType.""" + return EnumHandlerType.DB + + async def initialize(self, config: dict) -> None: + """Initialize database connection pool.""" + self._config = config + + self._pool = await asyncpg.create_pool( + host=config.get("host", "localhost"), + port=config.get("port", 5432), + database=config.get("database", "omnibase"), + user=config.get("user", "postgres"), + password=config.get("password", ""), + min_size=config.get("pool_min_size", 2), + max_size=config.get("pool_max_size", 10), + ) + self._logger.info("Database handler initialized") + + async def shutdown(self) -> None: + """Close database connection pool.""" + if self._pool: + await self._pool.close() + self._logger.info("Database handler shutdown") + + async def execute(self, envelope: ModelOnexEnvelope) -> ModelOnexEnvelope: + """Execute database operation from envelope.""" + if not self._pool: + return self._error_response(envelope, "Database pool not initialized") + + payload = envelope.payload + operation = envelope.operation + + try: + if operation == "query": + return await self._execute_query(envelope, payload) + elif operation == "execute": + return await self._execute_command(envelope, payload) + elif operation == "transaction": + return await self._execute_transaction(envelope, payload) + else: + return self._error_response(envelope, f"Unknown database operation: {operation}") + + except asyncpg.PostgresError as e: + return self._error_response(envelope, f"Database error: {e}") + except Exception as e: + return self._error_response(envelope, str(e)) + + async def _execute_query(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: + """Execute a query and return results.""" + sql = payload.get("sql") + params = payload.get("params", []) + + if not sql: + return self._error_response(envelope, "SQL query is required") + + async with self._pool.acquire() as conn: + rows = await conn.fetch(sql, *params) + return self._success_response(envelope, { + "rows": [dict(row) for row in rows], + "row_count": len(rows), + }) + + async def _execute_command(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: + """Execute a command (INSERT, UPDATE, DELETE).""" + sql = payload.get("sql") + params = payload.get("params", []) + + if not sql: + return self._error_response(envelope, "SQL command is required") + + async with self._pool.acquire() as conn: + result = await conn.execute(sql, *params) + return self._success_response(envelope, { + "result": result, + }) + + async def _execute_transaction(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: + """Execute multiple statements in a transaction.""" + statements = payload.get("statements", []) + + if not statements: + return self._error_response(envelope, "Transaction statements required") + + async with self._pool.acquire() as conn: + async with conn.transaction(): + results = [] + for stmt in statements: + sql = stmt.get("sql") + params = stmt.get("params", []) + result = await conn.execute(sql, *params) + results.append(result) + + return self._success_response(envelope, { + "results": results, + "statement_count": len(results), + }) + + async def health_check(self) -> dict: + """Check database handler health.""" + if not self._pool: + return {"healthy": False, "handler_type": self.handler_type.value, "error": "Pool not initialized"} + + try: + async with self._pool.acquire() as conn: + await conn.fetchval("SELECT 1") + return {"healthy": True, "handler_type": self.handler_type.value} + except Exception as e: + return {"healthy": False, "handler_type": self.handler_type.value, "error": str(e)} + + def _success_response(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: + return ModelOnexEnvelope( + correlation_id=envelope.correlation_id, + causation_id=envelope.envelope_id, + source_node="db_handler", + target_node=envelope.source_node, + handler_type=envelope.handler_type, + operation=envelope.operation, + payload=payload, + is_response=True, + success=True, + ) + + def _error_response(self, envelope: ModelOnexEnvelope, error: str) -> ModelOnexEnvelope: + return ModelOnexEnvelope( + correlation_id=envelope.correlation_id, + causation_id=envelope.envelope_id, + source_node="db_handler", + target_node=envelope.source_node, + handler_type=envelope.handler_type, + operation=envelope.operation, + is_response=True, + success=False, + error=error, + ) +``` + +### 3.4 Kafka Event Bus (ProtocolEventBus, NOT ProtocolHandler) + +**File**: `src/omnibase_infra/event_bus/kafka_event_bus.py` + +```python +"""Kafka event bus implementation.""" +from __future__ import annotations + +import json +import logging +from typing import Any, Callable, Awaitable + +from aiokafka import AIOKafkaConsumer, AIOKafkaProducer + +from omnibase_core.models.runtime.model_onex_envelope import ModelOnexEnvelope +from omnibase_spi.protocols.protocol_event_bus import ProtocolEventBus + + +class KafkaEventBus(ProtocolEventBus): + """Kafka implementation of ProtocolEventBus. + + This is the transport layer that feeds envelopes into NodeRuntime. + It implements ProtocolEventBus, NOT ProtocolHandler. + + RuntimeHostProcess owns this and calls runtime.route_envelope() + for each message received. + """ + + def __init__(self) -> None: + self._logger = logging.getLogger("event_bus.kafka") + self._producer: AIOKafkaProducer | None = None + self._consumer: AIOKafkaConsumer | None = None + self._config: dict[str, Any] = {} + self._running = False + self._handler: Callable[[ModelOnexEnvelope], Awaitable[ModelOnexEnvelope]] | None = None + self._subscribed_topics: list[str] = [] + + async def initialize(self, config: dict) -> None: + """Initialize Kafka connections.""" + self._config = config + bootstrap_servers = config.get("bootstrap_servers", "localhost:9092") + consumer_group = config.get("consumer_group", "runtime-host") + + # Producer for publishing responses + self._producer = AIOKafkaProducer( + bootstrap_servers=bootstrap_servers, + value_serializer=lambda v: json.dumps(v).encode("utf-8"), + ) + await self._producer.start() + + # Consumer will be started when subscribe is called + self._bootstrap_servers = bootstrap_servers + self._consumer_group = consumer_group + + self._logger.info("Kafka event bus initialized") + + async def shutdown(self) -> None: + """Close Kafka connections.""" + self._running = False + + if self._producer: + await self._producer.stop() + + if self._consumer: + await self._consumer.stop() + + self._logger.info("Kafka event bus shutdown") + + async def publish_envelope(self, envelope: ModelOnexEnvelope, topic: str) -> None: + """Publish an envelope to a topic.""" + if not self._producer: + raise RuntimeError("Kafka producer not initialized") + + key = str(envelope.correlation_id).encode("utf-8") + value = envelope.model_dump(mode="json") + + await self._producer.send_and_wait(topic, value=value, key=key) + self._logger.debug(f"Published envelope to {topic}: {envelope.envelope_id}") + + async def subscribe( + self, + topic: str, + handler: Callable[[ModelOnexEnvelope], Awaitable[ModelOnexEnvelope]], + ) -> None: + """Subscribe to a topic with a handler.""" + self._handler = handler + self._subscribed_topics.append(topic) + self._logger.info(f"Subscribed to topic: {topic}") + + async def start_consuming(self) -> None: + """Start the consumer loop.""" + if not self._subscribed_topics: + self._logger.warning("No topics subscribed, not starting consumer") + return + + self._consumer = AIOKafkaConsumer( + *self._subscribed_topics, + bootstrap_servers=self._bootstrap_servers, + group_id=self._consumer_group, + value_deserializer=lambda v: json.loads(v.decode("utf-8")), + ) + await self._consumer.start() + + self._running = True + self._logger.info(f"Started consuming from: {self._subscribed_topics}") + + try: + async for msg in self._consumer: + if not self._running: + break + + try: + # Parse envelope from message + envelope = ModelOnexEnvelope.model_validate(msg.value) + + # Call the handler (which calls runtime.route_envelope) + if self._handler: + response = await self._handler(envelope) + + # Optionally publish response to response topic + if response.is_response and response.target_node: + response_topic = self._config.get("response_topic") + if response_topic: + await self.publish_envelope(response, response_topic) + + except Exception as e: + self._logger.exception(f"Error processing message: {e}") + + except Exception as e: + self._logger.exception(f"Consumer loop error: {e}") + finally: + self._running = False + + async def health_check(self) -> dict: + """Check event bus health.""" + return { + "healthy": self._producer is not None, + "running": self._running, + "subscribed_topics": self._subscribed_topics, + } +``` + +### 3.5 Vault Handler (Strongly Typed) + +**File**: `src/omnibase_infra/handlers/vault_handler.py` + +```python +"""Vault secrets management protocol handler.""" +from __future__ import annotations + +import logging +from typing import Any + +import hvac + +from omnibase_core.enums.enum_handler_type import EnumHandlerType +from omnibase_core.models.runtime.model_onex_envelope import ModelOnexEnvelope +from omnibase_core.protocols.protocol_handler import ProtocolHandler + + +class VaultHandler(ProtocolHandler): + """Vault secrets management protocol handler. + + Handles secret read/write operations via HashiCorp Vault. + """ + + def __init__(self) -> None: + self._logger = logging.getLogger("handler.vault") + self._client: hvac.Client | None = None + self._config: dict[str, Any] = {} + + @property + def handler_type(self) -> EnumHandlerType: + """Return handler type as EnumHandlerType.""" + return EnumHandlerType.VAULT + + async def initialize(self, config: dict) -> None: + """Initialize Vault client.""" + self._config = config + + self._client = hvac.Client( + url=config.get("url", "http://localhost:8200"), + token=config.get("token"), + namespace=config.get("namespace"), + ) + self._logger.info("Vault handler initialized") + + async def shutdown(self) -> None: + """Close Vault client.""" + self._client = None + self._logger.info("Vault handler shutdown") + + async def execute(self, envelope: ModelOnexEnvelope) -> ModelOnexEnvelope: + """Execute Vault operation from envelope.""" + if not self._client: + return self._error_response(envelope, "Vault client not initialized") + + operation = envelope.operation + payload = envelope.payload + + try: + if operation == "get_secret": + return await self._get_secret(envelope, payload) + elif operation == "set_secret": + return await self._set_secret(envelope, payload) + elif operation == "delete_secret": + return await self._delete_secret(envelope, payload) + elif operation == "list_secrets": + return await self._list_secrets(envelope, payload) + else: + return self._error_response(envelope, f"Unknown Vault operation: {operation}") + + except hvac.exceptions.VaultError as e: + return self._error_response(envelope, f"Vault error: {e}") + except Exception as e: + return self._error_response(envelope, str(e)) + + async def _get_secret(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: + """Get a secret from Vault.""" + path = payload.get("path") + mount_point = payload.get("mount_point", "secret") + + if not path: + return self._error_response(envelope, "Secret path is required") + + response = self._client.secrets.kv.v2.read_secret_version( + path=path, + mount_point=mount_point, + ) + + return self._success_response(envelope, { + "data": response["data"]["data"], + "metadata": response["data"]["metadata"], + }) + + async def _set_secret(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: + """Set a secret in Vault.""" + path = payload.get("path") + data = payload.get("data") + mount_point = payload.get("mount_point", "secret") + + if not path or not data: + return self._error_response(envelope, "Secret path and data are required") + + response = self._client.secrets.kv.v2.create_or_update_secret( + path=path, + secret=data, + mount_point=mount_point, + ) + + return self._success_response(envelope, { + "version": response["data"]["version"], + "created_time": response["data"]["created_time"], + }) + + async def _delete_secret(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: + """Delete a secret from Vault.""" + path = payload.get("path") + mount_point = payload.get("mount_point", "secret") + + if not path: + return self._error_response(envelope, "Secret path is required") + + self._client.secrets.kv.v2.delete_metadata_and_all_versions( + path=path, + mount_point=mount_point, + ) + + return self._success_response(envelope, {"deleted": True}) + + async def _list_secrets(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: + """List secrets at a path.""" + path = payload.get("path", "") + mount_point = payload.get("mount_point", "secret") + + response = self._client.secrets.kv.v2.list_secrets( + path=path, + mount_point=mount_point, + ) + + return self._success_response(envelope, { + "keys": response["data"]["keys"], + }) + + async def health_check(self) -> dict: + """Check Vault handler health.""" + if not self._client: + return {"healthy": False, "handler_type": self.handler_type.value, "error": "Client not initialized"} + + try: + health = self._client.sys.read_health_status(method="GET") + return { + "healthy": health.get("initialized", False) and not health.get("sealed", True), + "handler_type": self.handler_type.value, + "vault_version": health.get("version"), + } + except Exception as e: + return {"healthy": False, "handler_type": self.handler_type.value, "error": str(e)} + + def _success_response(self, envelope: ModelOnexEnvelope, payload: dict) -> ModelOnexEnvelope: + return ModelOnexEnvelope( + correlation_id=envelope.correlation_id, + causation_id=envelope.envelope_id, + source_node="vault_handler", + target_node=envelope.source_node, + handler_type=envelope.handler_type, + operation=envelope.operation, + payload=payload, + is_response=True, + success=True, + ) + + def _error_response(self, envelope: ModelOnexEnvelope, error: str) -> ModelOnexEnvelope: + return ModelOnexEnvelope( + correlation_id=envelope.correlation_id, + causation_id=envelope.envelope_id, + source_node="vault_handler", + target_node=envelope.source_node, + handler_type=envelope.handler_type, + operation=envelope.operation, + is_response=True, + success=False, + error=error, + ) +``` + +### 3.6 Handler Wiring (Single Source of Truth) + +**File**: `src/omnibase_infra/runtime/wiring.py` + +```python +"""Handler registration and wiring utilities. + +This is the SINGLE SOURCE OF TRUTH for handler registration. +RuntimeHostProcess uses this - no duplicate logic. +""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +from omnibase_core.enums.enum_handler_type import EnumHandlerType +from omnibase_core.runtime.handlers.local_handler import LocalHandler + +from omnibase_infra.handlers.http_handler import HttpHandler +from omnibase_infra.handlers.db_handler import DbHandler +from omnibase_infra.handlers.vault_handler import VaultHandler + +if TYPE_CHECKING: + from omnibase_core.protocols.protocol_handler import ProtocolHandler + from omnibase_core.runtime.node_runtime import NodeRuntime + + +# Single source of truth for handler classes +HANDLER_REGISTRY: dict[EnumHandlerType, type["ProtocolHandler"]] = { + EnumHandlerType.LOCAL: LocalHandler, # Dev/test only + EnumHandlerType.HTTP: HttpHandler, + EnumHandlerType.DB: DbHandler, + EnumHandlerType.VAULT: VaultHandler, + # EnumHandlerType.CONSUL: ConsulHandler, # TODO: implement + # EnumHandlerType.LLM: LlmHandler, # TODO: implement +} + + +def register_handlers_from_config( + runtime: "NodeRuntime", + handler_configs: list[dict], +) -> None: + """Register handlers with the runtime based on config. + + This is the ONLY place where handlers are registered. + + Args: + runtime: NodeRuntime instance + handler_configs: List of handler config dicts from contract + """ + for config in handler_configs: + handler_type = EnumHandlerType(config["handler_type"]) + enabled = config.get("enabled", True) + + if not enabled: + continue + + handler_class = HANDLER_REGISTRY.get(handler_type) + if handler_class: + handler = handler_class() + runtime.register_handler(handler) + else: + raise ValueError(f"Unknown handler type: {handler_type}") + + +def get_handler_class(handler_type: EnumHandlerType) -> type["ProtocolHandler"] | None: + """Get handler class by type.""" + return HANDLER_REGISTRY.get(handler_type) + + +def list_available_handlers() -> list[EnumHandlerType]: + """List all available handler types.""" + return list(HANDLER_REGISTRY.keys()) +``` + +### 3.7 RuntimeHostProcess (Uses Wiring, Owns Event Bus) + +**File**: `src/omnibase_infra/runtime/runtime_host_process.py` + +```python +"""Runtime Host Process - Infrastructure-level wrapper for NodeRuntime.""" +from __future__ import annotations + +import asyncio +import logging +import signal +from pathlib import Path +from typing import TYPE_CHECKING + +from omnibase_core.models.contracts.model_runtime_host_contract import ModelRuntimeHostContract +from omnibase_core.runtime.node_runtime import NodeRuntime + +from omnibase_infra.event_bus.kafka_event_bus import KafkaEventBus +from omnibase_infra.runtime.wiring import register_handlers_from_config + +if TYPE_CHECKING: + from omnibase_spi.protocols.protocol_event_bus import ProtocolEventBus + + +class RuntimeHostProcess: + """Infrastructure-level process wrapper for NodeRuntime. + + RuntimeHostProcess is responsible for: + - Loading and validating the runtime contract + - Registering infrastructure handlers (via wiring.py) + - Owning and managing the event bus (ProtocolEventBus) + - Driving NodeRuntime with envelopes from the event bus + - Managing process lifecycle and signals + - Providing health/metrics endpoints + + IMPORTANT: + This is where event bus consumption happens. + NodeRuntime is transport-agnostic - it only knows about envelopes. + This class bridges the transport (Kafka) to the runtime. + """ + + def __init__(self, contract_path: Path) -> None: + self._contract_path = contract_path + self._logger = logging.getLogger("runtime_host_process") + self._runtime: NodeRuntime | None = None + self._event_bus: "ProtocolEventBus | None" = None + self._shutdown_requested = False + + async def start(self) -> None: + """Start the runtime host process.""" + self._logger.info(f"Starting Runtime Host Process: {self._contract_path}") + + # Load contract + contract = ModelRuntimeHostContract.from_yaml(self._contract_path) + + # Create runtime + self._runtime = NodeRuntime(contract) + + # Register handlers (single source of truth: wiring.py) + handler_configs = [ + {"handler_type": h.handler_type.value, "enabled": h.enabled, **h.config} + for h in contract.handlers + ] + register_handlers_from_config(self._runtime, handler_configs) + + # Load nodes + contracts_dir = self._contract_path.parent / contract.contracts_directory + if contracts_dir.exists(): + await self._runtime.load_nodes_from_directory(contracts_dir) + + # Initialize runtime (handlers + nodes) + await self._runtime.initialize() + + # Initialize event bus if enabled + if contract.event_bus.enabled: + await self._setup_event_bus(contract) + + # Setup signal handlers + self._setup_signals() + + self._logger.info("Runtime Host Process started") + + # Start event bus consumer loop (this is where envelopes flow in) + if self._event_bus: + await self._event_bus.start_consuming() + else: + # No event bus - just wait for shutdown + while not self._shutdown_requested: + await asyncio.sleep(1) + + async def _setup_event_bus(self, contract: ModelRuntimeHostContract) -> None: + """Setup the event bus.""" + self._event_bus = KafkaEventBus() + await self._event_bus.initialize(contract.event_bus.config) + + # Subscribe to command topic + command_topic = contract.event_bus.config.get( + "command_topic", + f"onex.cmd.runtime.{contract.name}.v1" + ) + + # The handler bridges event bus to NodeRuntime + await self._event_bus.subscribe( + command_topic, + self._runtime.route_envelope, # This is where the bridge happens + ) + + self._logger.info(f"Event bus subscribed to: {command_topic}") + + async def stop(self) -> None: + """Stop the runtime host process.""" + self._shutdown_requested = True + + if self._event_bus: + await self._event_bus.shutdown() + + if self._runtime: + await self._runtime.shutdown() + + self._logger.info("Runtime Host Process stopped") + + def _setup_signals(self) -> None: + """Setup signal handlers for graceful shutdown.""" + loop = asyncio.get_running_loop() + + for sig in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(sig, self._signal_handler) + + def _signal_handler(self) -> None: + """Handle shutdown signals.""" + if not self._shutdown_requested: + self._shutdown_requested = True + self._logger.info("Shutdown signal received") + asyncio.create_task(self.stop()) +``` + +### 3.8 Example Runtime Contract (Clean Separation) + +**File**: `src/omnibase_infra/contracts/runtime/infra_runtime_host.yaml` + +```yaml +# Infrastructure Runtime Host Contract +name: "infra_runtime_host" +version: "1.0.0" +description: "ONEX Infrastructure Runtime Host - hosts all infrastructure nodes" + +# Node Configuration +contracts_directory: "../nodes" +nodes: + - slug: "vault_adapter" + contract_path: "vault_adapter.yaml" + enabled: true + - slug: "consul_adapter" + contract_path: "consul_adapter.yaml" + enabled: true + - slug: "postgres_adapter" + contract_path: "postgres_adapter.yaml" + enabled: true + +# Handler Configuration (per-request operations) +# Note: Kafka is NOT a handler - it's the event bus (see below) +handlers: + # LocalHandler is dev/test only - DO NOT enable in production + # - handler_type: "local" + # enabled: false + + - handler_type: "http" + enabled: true + config: + timeout_seconds: 30 + + - handler_type: "db" + enabled: true + config: + host: "${POSTGRES_HOST}" + port: "${POSTGRES_PORT}" + database: "${POSTGRES_DB}" + user: "${POSTGRES_USER}" + password: "${POSTGRES_PASSWORD}" + pool_min_size: 2 + pool_max_size: 10 + + - handler_type: "vault" + enabled: true + config: + url: "${VAULT_ADDR}" + token: "${VAULT_TOKEN}" + namespace: "${VAULT_NAMESPACE}" + +# Event Bus Configuration (transport layer - feeds envelopes to runtime) +# This is SEPARATE from handlers. Kafka is a transport, not a per-request handler. +event_bus: + enabled: true + config: + bootstrap_servers: "${KAFKA_BOOTSTRAP_SERVERS}" + consumer_group: "infra-runtime-host" + command_topic: "onex.cmd.runtime.infra.v1" + response_topic: "onex.evt.runtime.infra.v1" + +# Health & Metrics +health_endpoint: + enabled: true + port: 8080 + path: "/health" + +metrics_endpoint: + enabled: true + port: 9090 + path: "/metrics" + +# Runtime Settings +max_concurrent_operations: 100 +shutdown_timeout_seconds: 30 +``` + +### 3.9 Phase 3 Success Criteria + +- [ ] HttpHandler with `EnumHandlerType.HTTP` return type +- [ ] DbHandler with `EnumHandlerType.DB` return type +- [ ] VaultHandler with `EnumHandlerType.VAULT` return type +- [ ] KafkaEventBus implements `ProtocolEventBus` (NOT ProtocolHandler) +- [ ] `wiring.py` is single source of truth for handler registration +- [ ] RuntimeHostProcess uses `wiring.py` (no duplicate handler map) +- [ ] RuntimeHostProcess owns event bus and calls `runtime.route_envelope()` +- [ ] Example contract shows clean handler vs event_bus separation +- [ ] Integration tests for each handler +- [ ] End-to-end test: event bus → runtime → handler → response + +--- + +## Phase 4-5: Integration, Testing & Deployment + +*(Phases 4 and 5 remain largely the same as before, with tests updated to use the new architecture)* + +### Key Test Scenarios + +1. **LocalHandler echo** (dev/test validation) +2. **HttpHandler external call** (real HTTP request) +3. **DbHandler query** (real database query) +4. **VaultHandler secret** (real Vault operation) +5. **Event bus → NodeRuntime → handler flow** (full integration) +6. **Multi-node routing** (envelope routing to correct node) + +--- + +## Architectural Sanity Checklist + +Use this checklist to verify the implementation maintains architectural invariants: + +### omnibase_core +- [ ] Has: `EnumHandlerType`, `ModelOnexEnvelope`, `ModelRuntimeHostContract`, `NodeInstance`, `NodeRuntime`, `ProtocolHandler`, `LocalHandler` (dev-only), CLI test runtime +- [ ] Has **NO** Kafka/HTTP/DB/Vault imports anywhere +- [ ] `NodeRuntime` has NO `_event_bus_loop` method +- [ ] `ProtocolHandler.handler_type` returns `EnumHandlerType` (not `str`) +- [ ] `LocalHandler` has dev/test warnings in docstring + +### omnibase_spi +- [ ] Exposes: `ProtocolHandler`, `ProtocolEventBus` +- [ ] `ProtocolEventBus` has envelope methods +- [ ] Clear documentation of handler vs event bus distinction + +### omnibase_infra +- [ ] Handlers return `EnumHandlerType` (not `str`) +- [ ] `KafkaEventBus` implements `ProtocolEventBus` (NOT `ProtocolHandler`) +- [ ] `wiring.py` is single source of truth +- [ ] `RuntimeHostProcess` uses `wiring.py` for handler registration +- [ ] `RuntimeHostProcess` owns event bus, calls `runtime.route_envelope()` +- [ ] No duplicate handler registration logic + +### Contracts +- [ ] `handlers` section contains only per-request handlers (http, db, vault) +- [ ] `event_bus` section is separate from handlers +- [ ] `local` handler is NOT enabled in production contracts + +--- + +## Summary Timeline + +| Week | Phase | Deliverables | +|------|-------|--------------| +| 1 | Phase 0 + Phase 1 Start | Prerequisites validated, core types begun | +| 2 | Phase 1 Complete | All core classes (transport-agnostic), CLI, local handler | +| 3 | Phase 2 + Phase 3 Start | SPI updates, HTTP handler, KafkaEventBus | +| 4 | Phase 3 Continue | DB, Vault handlers, wiring | +| 5 | Phase 3 Complete + Phase 4 Start | RuntimeHostProcess, integration tests | +| 6 | Phase 4 Complete | All tests passing, benchmarks met | +| 7-8 | Phase 5 | Docker, deployment, migration | + +--- + +## Success Metrics + +| Metric | Target | Measurement | +|--------|--------|-------------| +| Memory per 10 nodes | <200MB | tracemalloc | +| Envelope throughput | >100/sec | Benchmark suite | +| Handler latency (local) | <1ms | p99 latency | +| Handler latency (http) | <100ms | p99 latency | +| Handler latency (db) | <50ms | p99 latency | +| Test coverage | >90% | pytest-cov | +| Migration downtime | 0 | Shadow deployment | +| Core Kafka imports | 0 | grep verification | + +--- + +*This implementation plan was created on December 3, 2025 and updated with architectural refinements.* diff --git a/src/omnibase_infra/cli/commands.py b/src/omnibase_infra/cli/commands.py index a63c419bd8..998f5bf1a6 100644 --- a/src/omnibase_infra/cli/commands.py +++ b/src/omnibase_infra/cli/commands.py @@ -4,7 +4,7 @@ Provides CLI interface for infrastructure management and validation. """ -from typing import Any +from typing import Optional import click from rich.console import Console @@ -30,7 +30,7 @@ def validate() -> None: default=None, help="Maximum allowed violations (default: INFRA_MAX_VIOLATIONS)", ) -def validate_architecture_cmd(directory: str, max_violations: int | None) -> None: +def validate_architecture_cmd(directory: str, max_violations: Optional[int]) -> None: """Validate architecture (one-model-per-file).""" from omnibase_infra.validation.infra_validators import ( INFRA_MAX_VIOLATIONS, @@ -66,7 +66,7 @@ def validate_contracts_cmd(directory: str) -> None: default=None, help="Enable strict mode (default: INFRA_PATTERNS_STRICT)", ) -def validate_patterns_cmd(directory: str, strict: bool | None) -> None: +def validate_patterns_cmd(directory: str, strict: Optional[bool]) -> None: """Validate code patterns and naming conventions.""" from omnibase_infra.validation.infra_validators import ( INFRA_PATTERNS_STRICT, @@ -94,7 +94,7 @@ def validate_patterns_cmd(directory: str, strict: bool | None) -> None: help="Enable strict mode (default: INFRA_UNIONS_STRICT)", ) def validate_unions_cmd( - directory: str, max_unions: int | None, strict: bool | None + directory: str, max_unions: Optional[int], strict: Optional[bool] ) -> None: """Validate Union type usage.""" from omnibase_infra.validation.infra_validators import ( @@ -187,7 +187,7 @@ def _is_result_valid(result: object) -> bool: return False -def _get_error_count(result: Any) -> int: +def _get_error_count(result: object) -> int: """Get the error count from a validation result.""" if hasattr(result, "has_circular_imports"): if hasattr(result, "cycles"): @@ -198,7 +198,7 @@ def _get_error_count(result: Any) -> int: return 0 -def _print_result(name: str, result: Any) -> None: +def _print_result(name: str, result: object) -> None: """Print validation result with rich formatting.""" if hasattr(result, "is_valid"): if result.is_valid: diff --git a/src/omnibase_infra/handlers/__init__.py b/src/omnibase_infra/handlers/__init__.py new file mode 100644 index 0000000000..2cf1d18d49 --- /dev/null +++ b/src/omnibase_infra/handlers/__init__.py @@ -0,0 +1,21 @@ +"""Handlers module for omnibase_infra. + +This module provides adapter implementations for various infrastructure +communication patterns including HTTP REST and database operations. + +Adapters are responsible for: +- Processing incoming requests and messages +- Routing to appropriate services +- Formatting and returning responses +- Error handling and logging + +Available Adapters: +- HttpRestAdapter: HTTP/REST protocol adapter (MVP: GET, POST only) +""" + +from omnibase_infra.handlers.handler_http import HttpRestAdapter + +__all__: list[str] = [ + "HttpRestAdapter", + # "DbAdapter", # Database operation adapter (future) +] diff --git a/src/omnibase_infra/handlers/db_handler.py b/src/omnibase_infra/handlers/db_handler.py new file mode 100644 index 0000000000..290063493f --- /dev/null +++ b/src/omnibase_infra/handlers/db_handler.py @@ -0,0 +1,32 @@ +"""Database Handler for omnibase_infra. + +This module will implement the database handler for processing +database operations and managing database connections. + +Planned Responsibilities: +- Execute database queries and transactions +- Manage connection pooling via postgres_connection_manager +- Handle query parameter sanitization +- Implement retry logic with exponential backoff +- Provide structured logging for database operations +- Collect metrics for query performance + +Implementation Notes: +- Will use asyncpg for async PostgreSQL operations +- Will integrate with postgres_connection_manager for connection pooling +- Will follow ONEX contract-driven patterns +- Will use Pydantic models for query parameters and results +- Will implement SQL injection protection + +Dependencies: +- asyncpg: Async PostgreSQL driver +- postgres_connection_manager: Connection pooling +- opentelemetry: Distributed tracing +- structlog: Structured logging +- sqlparse: SQL query sanitization +""" + +from __future__ import annotations + +# Placeholder for future implementation +# This file will contain the DBHandler class implementing database operations diff --git a/src/omnibase_infra/handlers/handler_http.py b/src/omnibase_infra/handlers/handler_http.py new file mode 100644 index 0000000000..2acfe8910d --- /dev/null +++ b/src/omnibase_infra/handlers/handler_http.py @@ -0,0 +1,290 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2025 OmniNode Team +"""HTTP REST Adapter - MVP implementation using httpx async client. + +Supports GET and POST operations with 30-second fixed timeout. +PUT, DELETE, PATCH deferred to Beta. Retry logic and rate limiting deferred to Beta. +""" + +from __future__ import annotations + +import json +import logging +from typing import Optional +from uuid import UUID, uuid4 + +import httpx +from omnibase_core.enums.enum_handler_type import EnumHandlerType + +from omnibase_infra.enums import EnumInfraTransportType +from omnibase_infra.errors import ( + InfraConnectionError, + InfraTimeoutError, + ModelInfraErrorContext, + RuntimeHostError, +) + +logger = logging.getLogger(__name__) + +_DEFAULT_TIMEOUT_SECONDS: float = 30.0 +_SUPPORTED_OPERATIONS: frozenset[str] = frozenset({"http.get", "http.post"}) + + +class HttpRestAdapter: + """HTTP REST protocol adapter using httpx async client (MVP: GET, POST only).""" + + def __init__(self) -> None: + """Initialize HttpRestAdapter in uninitialized state.""" + self._client: Optional[httpx.AsyncClient] = None + self._timeout: float = _DEFAULT_TIMEOUT_SECONDS + self._initialized: bool = False + + @property + def handler_type(self) -> EnumHandlerType: + """Return EnumHandlerType.HTTP.""" + return EnumHandlerType.HTTP + + async def initialize(self, config: dict[str, object]) -> None: + """Initialize HTTP client with 30s fixed timeout (config unused in MVP).""" + try: + self._timeout = _DEFAULT_TIMEOUT_SECONDS + self._client = httpx.AsyncClient( + timeout=httpx.Timeout(self._timeout), + follow_redirects=True, + ) + self._initialized = True + logger.info( + "HttpRestAdapter initialized", extra={"timeout_seconds": self._timeout} + ) + except Exception as e: + ctx = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.HTTP, + operation="initialize", + target_name="http_rest_adapter", + ) + raise RuntimeHostError( + "Failed to initialize HTTP adapter", context=ctx + ) from e + + async def shutdown(self) -> None: + """Close HTTP client and release resources.""" + if self._client is not None: + await self._client.aclose() + self._client = None + self._initialized = False + logger.info("HttpRestAdapter shutdown complete") + + async def execute(self, envelope: dict[str, object]) -> dict[str, object]: + """Execute HTTP operation (http.get or http.post) from envelope.""" + correlation_id = self._extract_correlation_id(envelope) + + if not self._initialized or self._client is None: + ctx = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.HTTP, + operation="execute", + target_name="http_rest_adapter", + correlation_id=correlation_id, + ) + raise RuntimeHostError( + "HttpRestAdapter not initialized. Call initialize() first.", context=ctx + ) + + operation = envelope.get("operation") + if not isinstance(operation, str): + ctx = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.HTTP, + operation="execute", + target_name="http_rest_adapter", + correlation_id=correlation_id, + ) + raise RuntimeHostError( + "Missing or invalid 'operation' in envelope", context=ctx + ) + + if operation not in _SUPPORTED_OPERATIONS: + ctx = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.HTTP, + operation=operation, + target_name="http_rest_adapter", + correlation_id=correlation_id, + ) + raise RuntimeHostError( + f"Operation '{operation}' not supported in MVP. Available: {', '.join(sorted(_SUPPORTED_OPERATIONS))}", + context=ctx, + ) + + payload = envelope.get("payload") + if not isinstance(payload, dict): + ctx = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.HTTP, + operation=operation, + target_name="http_rest_adapter", + correlation_id=correlation_id, + ) + raise RuntimeHostError( + "Missing or invalid 'payload' in envelope", context=ctx + ) + + url = payload.get("url") + if not isinstance(url, str) or not url: + ctx = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.HTTP, + operation=operation, + target_name="http_rest_adapter", + correlation_id=correlation_id, + ) + raise RuntimeHostError("Missing or invalid 'url' in payload", context=ctx) + + headers = self._extract_headers(payload, operation, url, correlation_id) + + if operation == "http.get": + return await self._execute_request( + "GET", url, headers, None, correlation_id + ) + else: # http.post + return await self._execute_request( + "POST", url, headers, payload.get("body"), correlation_id + ) + + def _extract_correlation_id(self, envelope: dict[str, object]) -> UUID: + """Extract or generate correlation ID from envelope.""" + raw = envelope.get("correlation_id") + if isinstance(raw, UUID): + return raw + if isinstance(raw, str): + try: + return UUID(raw) + except ValueError: + pass + return uuid4() + + def _extract_headers( + self, payload: dict[str, object], operation: str, url: str, correlation_id: UUID + ) -> dict[str, str]: + """Extract and validate headers from payload.""" + headers_raw = payload.get("headers") + if headers_raw is None: + return {} + if isinstance(headers_raw, dict): + return {str(k): str(v) for k, v in headers_raw.items()} + ctx = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.HTTP, + operation=operation, + target_name=url, + correlation_id=correlation_id, + ) + raise RuntimeHostError( + "Invalid 'headers' in payload - must be a dict", context=ctx + ) + + async def _execute_request( + self, + method: str, + url: str, + headers: dict[str, str], + body: object, + correlation_id: UUID, + ) -> dict[str, object]: + """Execute HTTP request and handle errors.""" + if self._client is None: + ctx = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.HTTP, + operation=f"http.{method.lower()}", + target_name=url, + correlation_id=correlation_id, + ) + raise RuntimeHostError( + "HttpRestAdapter not initialized - call initialize() first", context=ctx + ) + + ctx = ModelInfraErrorContext( + transport_type=EnumInfraTransportType.HTTP, + operation=f"http.{method.lower()}", + target_name=url, + correlation_id=correlation_id, + ) + + try: + if method == "GET": + response = await self._client.get(url, headers=headers) + elif body is None: + response = await self._client.post(url, headers=headers) + elif isinstance(body, dict): + response = await self._client.post(url, headers=headers, json=body) + elif isinstance(body, str): + response = await self._client.post(url, headers=headers, content=body) + else: + try: + serialized_body = json.dumps(body) + except TypeError as e: + raise RuntimeHostError( + f"Body is not JSON-serializable: {type(body).__name__}", + context=ctx, + ) from e + response = await self._client.post( + url, headers=headers, content=serialized_body + ) + + return self._build_response(response, correlation_id) + + except httpx.TimeoutException as e: + raise InfraTimeoutError( + f"HTTP {method} request timed out after {self._timeout}s", + context=ctx, + timeout_seconds=self._timeout, + ) from e + except httpx.ConnectError as e: + raise InfraConnectionError( + f"Failed to connect to {url}", context=ctx + ) from e + except httpx.HTTPStatusError as e: + return self._build_response(e.response, correlation_id) + except httpx.HTTPError as e: + raise InfraConnectionError( + f"HTTP error during {method} request: {type(e).__name__}", context=ctx + ) from e + + def _build_response( + self, response: httpx.Response, correlation_id: UUID + ) -> dict[str, object]: + """Build response envelope from httpx Response.""" + content_type = response.headers.get("content-type", "") + if "application/json" in content_type: + try: + body: object = response.json() + except json.JSONDecodeError: + body = response.text + else: + body = response.text + + return { + "status": "success", + "payload": { + "status_code": response.status_code, + "headers": dict(response.headers), + "body": body, + }, + "correlation_id": str(correlation_id), + } + + async def health_check(self) -> dict[str, object]: + """Return adapter health status.""" + return { + "healthy": self._initialized and self._client is not None, + "initialized": self._initialized, + "adapter_type": self.handler_type.value, + "timeout_seconds": self._timeout, + } + + def describe(self) -> dict[str, object]: + """Return adapter metadata and capabilities.""" + return { + "adapter_type": self.handler_type.value, + "supported_operations": sorted(_SUPPORTED_OPERATIONS), + "timeout_seconds": self._timeout, + "initialized": self._initialized, + "version": "0.1.0-mvp", + } + + +__all__: list[str] = ["HttpRestAdapter"] diff --git a/src/omnibase_infra/runtime/runtime_host_process.py b/src/omnibase_infra/runtime/runtime_host_process.py new file mode 100644 index 0000000000..2501be2005 --- /dev/null +++ b/src/omnibase_infra/runtime/runtime_host_process.py @@ -0,0 +1,36 @@ +"""Infrastructure runtime host process implementation. + +This module will implement the BaseRuntimeHostProcess for the omnibase_infra layer. + +The RuntimeHostProcess is responsible for: +- Initializing the infrastructure runtime environment +- Managing the lifecycle of infrastructure services +- Coordinating between handlers, nodes, and external services +- Providing health checks and observability endpoints +- Managing graceful shutdown and resource cleanup + +Implementation Notes: +- Will extend omnibase_core.runtime.BaseRuntimeHostProcess +- Will integrate with the wiring module for handler registration +- Will support infrastructure-specific configuration via contracts +- Must follow ONEX 4-node architecture patterns + +Dependencies: +- omnibase_core.runtime.BaseRuntimeHostProcess (base class) +- omnibase_infra.runtime.wiring (handler registration) +- omnibase_core.container.ONEXContainer (dependency injection) + +Example Usage (future): + ```python + from omnibase_infra.runtime import RuntimeHostProcess + + async def main() -> None: + process = RuntimeHostProcess() + await process.start() + ``` +""" + +from __future__ import annotations + +# Placeholder: Implementation will be added in subsequent tickets +# This file serves as the designated location for the RuntimeHostProcess class diff --git a/tests/unit/handlers/__init__.py b/tests/unit/handlers/__init__.py new file mode 100644 index 0000000000..c91e9f00d1 --- /dev/null +++ b/tests/unit/handlers/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2025 OmniNode Team +"""Unit tests for handlers module.""" diff --git a/tests/unit/handlers/test_handler_http.py b/tests/unit/handlers/test_handler_http.py new file mode 100644 index 0000000000..01135e512f --- /dev/null +++ b/tests/unit/handlers/test_handler_http.py @@ -0,0 +1,1104 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2025 OmniNode Team +# mypy: disable-error-code="index, operator, arg-type" +"""Unit tests for HttpRestAdapter. + +Comprehensive test suite covering initialization, GET/POST operations, +error handling, health checks, describe, and lifecycle management. +""" + +from __future__ import annotations + +from typing import cast +from unittest.mock import AsyncMock, MagicMock, patch +from uuid import UUID, uuid4 + +import httpx +import pytest +from omnibase_core.enums.enum_handler_type import EnumHandlerType + +from omnibase_infra.errors import ( + InfraConnectionError, + InfraTimeoutError, + RuntimeHostError, +) +from omnibase_infra.handlers.handler_http import HttpRestAdapter + +# Type alias for response dict with nested structure +ResponseDict = dict[str, object] + + +class TestHttpRestAdapterInitialization: + """Test suite for HttpRestAdapter initialization.""" + + @pytest.fixture + def handler(self) -> HttpRestAdapter: + """Create HttpRestAdapter fixture.""" + return HttpRestAdapter() + + def test_handler_init_default_state(self, handler: HttpRestAdapter) -> None: + """Test handler initializes in uninitialized state.""" + assert handler._initialized is False + assert handler._client is None + assert handler._timeout == 30.0 + + def test_handler_type_returns_http(self, handler: HttpRestAdapter) -> None: + """Test handler_type property returns EnumHandlerType.HTTP.""" + assert handler.handler_type == EnumHandlerType.HTTP + + @pytest.mark.asyncio + async def test_initialize_with_empty_config(self, handler: HttpRestAdapter) -> None: + """Test handler initializes with empty config (uses defaults).""" + await handler.initialize({}) + + assert handler._initialized is True + assert handler._client is not None + assert handler._timeout == 30.0 + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_initialize_with_config_dict(self, handler: HttpRestAdapter) -> None: + """Test handler initializes with config dict (config ignored in MVP).""" + config: dict[str, object] = {"timeout": 60.0, "custom_option": "value"} + await handler.initialize(config) + + # MVP ignores config, uses fixed 30s timeout + assert handler._initialized is True + assert handler._timeout == 30.0 + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_initialize_creates_async_client( + self, handler: HttpRestAdapter + ) -> None: + """Test initialize creates httpx.AsyncClient with correct timeout.""" + await handler.initialize({}) + + assert isinstance(handler._client, httpx.AsyncClient) + # Verify timeout is set (30s default) + assert handler._client.timeout.connect == 30.0 + + await handler.shutdown() + + +class TestHttpRestAdapterGetOperations: + """Test suite for HTTP GET operations.""" + + @pytest.fixture + def handler(self) -> HttpRestAdapter: + """Create HttpRestAdapter fixture.""" + return HttpRestAdapter() + + @pytest.mark.asyncio + async def test_get_successful_response(self, handler: HttpRestAdapter) -> None: + """Test successful GET request returns correct response structure.""" + await handler.initialize({}) + + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = { + "content-type": "application/json", + "x-custom": "value", + } + mock_response.json.return_value = {"data": "test_value"} + + with patch.object(handler._client, "get", new_callable=AsyncMock) as mock_get: + mock_get.return_value = mock_response + + envelope: dict[str, object] = { + "operation": "http.get", + "payload": {"url": "https://api.example.com/resource"}, + "correlation_id": str(uuid4()), + } + + result = cast(ResponseDict, await handler.execute(envelope)) + + assert result["status"] == "success" + payload = cast(ResponseDict, result["payload"]) + assert payload["status_code"] == 200 + assert payload["body"] == {"data": "test_value"} + assert "correlation_id" in result + + mock_get.assert_called_once_with( + "https://api.example.com/resource", headers={} + ) + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_get_with_custom_headers(self, handler: HttpRestAdapter) -> None: + """Test GET request passes custom headers correctly.""" + await handler.initialize({}) + + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = {"content-type": "text/plain"} + mock_response.text = "OK" + mock_response.json.side_effect = Exception("Not JSON") + + with patch.object(handler._client, "get", new_callable=AsyncMock) as mock_get: + mock_get.return_value = mock_response + + envelope: dict[str, object] = { + "operation": "http.get", + "payload": { + "url": "https://api.example.com/status", + "headers": { + "Authorization": "Bearer token123", + "X-Request-ID": "req-456", + }, + }, + } + + result = await handler.execute(envelope) + + mock_get.assert_called_once_with( + "https://api.example.com/status", + headers={"Authorization": "Bearer token123", "X-Request-ID": "req-456"}, + ) + + assert result["payload"]["body"] == "OK" + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_get_with_query_params_in_url(self, handler: HttpRestAdapter) -> None: + """Test GET request with query parameters in URL.""" + await handler.initialize({}) + + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/json"} + mock_response.json.return_value = {"items": [1, 2, 3]} + + with patch.object(handler._client, "get", new_callable=AsyncMock) as mock_get: + mock_get.return_value = mock_response + + envelope: dict[str, object] = { + "operation": "http.get", + "payload": { + "url": "https://api.example.com/items?page=1&limit=10&filter=active", + }, + } + + result = await handler.execute(envelope) + + mock_get.assert_called_once_with( + "https://api.example.com/items?page=1&limit=10&filter=active", + headers={}, + ) + + assert result["payload"]["body"] == {"items": [1, 2, 3]} + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_get_text_response(self, handler: HttpRestAdapter) -> None: + """Test GET request with text/plain response.""" + await handler.initialize({}) + + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = {"content-type": "text/plain; charset=utf-8"} + mock_response.text = "Hello, World!" + + with patch.object(handler._client, "get", new_callable=AsyncMock) as mock_get: + mock_get.return_value = mock_response + + envelope: dict[str, object] = { + "operation": "http.get", + "payload": {"url": "https://example.com/hello"}, + } + + result = await handler.execute(envelope) + + assert result["payload"]["body"] == "Hello, World!" + assert result["payload"]["status_code"] == 200 + + await handler.shutdown() + + +class TestHttpRestAdapterPostOperations: + """Test suite for HTTP POST operations.""" + + @pytest.fixture + def handler(self) -> HttpRestAdapter: + """Create HttpRestAdapter fixture.""" + return HttpRestAdapter() + + @pytest.mark.asyncio + async def test_post_with_json_body(self, handler: HttpRestAdapter) -> None: + """Test POST request with JSON body.""" + await handler.initialize({}) + + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 201 + mock_response.headers = {"content-type": "application/json"} + mock_response.json.return_value = {"id": 123, "created": True} + + with patch.object(handler._client, "post", new_callable=AsyncMock) as mock_post: + mock_post.return_value = mock_response + + envelope: dict[str, object] = { + "operation": "http.post", + "payload": { + "url": "https://api.example.com/users", + "body": {"name": "John", "email": "john@example.com"}, + }, + } + + result = await handler.execute(envelope) + + mock_post.assert_called_once_with( + "https://api.example.com/users", + headers={}, + json={"name": "John", "email": "john@example.com"}, + ) + + assert result["status"] == "success" + assert result["payload"]["status_code"] == 201 + assert result["payload"]["body"] == {"id": 123, "created": True} + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_post_with_string_body(self, handler: HttpRestAdapter) -> None: + """Test POST request with string body.""" + await handler.initialize({}) + + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = {"content-type": "text/plain"} + mock_response.text = "Received" + + with patch.object(handler._client, "post", new_callable=AsyncMock) as mock_post: + mock_post.return_value = mock_response + + envelope: dict[str, object] = { + "operation": "http.post", + "payload": { + "url": "https://api.example.com/message", + "body": "Hello from client", + }, + } + + result = await handler.execute(envelope) + + mock_post.assert_called_once_with( + "https://api.example.com/message", + headers={}, + content="Hello from client", + ) + + assert result["payload"]["body"] == "Received" + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_post_with_no_body(self, handler: HttpRestAdapter) -> None: + """Test POST request with no body.""" + await handler.initialize({}) + + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 204 + mock_response.headers = {"content-type": "text/plain"} + mock_response.text = "" + + with patch.object(handler._client, "post", new_callable=AsyncMock) as mock_post: + mock_post.return_value = mock_response + + envelope: dict[str, object] = { + "operation": "http.post", + "payload": {"url": "https://api.example.com/trigger"}, + } + + result = await handler.execute(envelope) + + mock_post.assert_called_once_with( + "https://api.example.com/trigger", + headers={}, + ) + + assert result["payload"]["status_code"] == 204 + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_post_with_custom_headers(self, handler: HttpRestAdapter) -> None: + """Test POST request with custom headers.""" + await handler.initialize({}) + + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/json"} + mock_response.json.return_value = {"success": True} + + with patch.object(handler._client, "post", new_callable=AsyncMock) as mock_post: + mock_post.return_value = mock_response + + envelope: dict[str, object] = { + "operation": "http.post", + "payload": { + "url": "https://api.example.com/data", + "headers": { + "Content-Type": "application/json", + "X-API-Key": "secret-key-123", + }, + "body": {"value": 42}, + }, + } + + result = await handler.execute(envelope) + + mock_post.assert_called_once_with( + "https://api.example.com/data", + headers={ + "Content-Type": "application/json", + "X-API-Key": "secret-key-123", + }, + json={"value": 42}, + ) + + assert result["payload"]["body"] == {"success": True} + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_post_with_list_body_serialized_to_json( + self, handler: HttpRestAdapter + ) -> None: + """Test POST with list body gets JSON serialized.""" + await handler.initialize({}) + + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/json"} + mock_response.json.return_value = {"processed": 3} + + with patch.object(handler._client, "post", new_callable=AsyncMock) as mock_post: + mock_post.return_value = mock_response + + envelope: dict[str, object] = { + "operation": "http.post", + "payload": { + "url": "https://api.example.com/batch", + "body": [1, 2, 3], # List body, not dict or string + }, + } + + result = await handler.execute(envelope) + + # List body uses content= with json.dumps() + mock_post.assert_called_once() + call_args = mock_post.call_args + assert call_args.kwargs["content"] == "[1, 2, 3]" + + assert result["payload"]["body"] == {"processed": 3} + + await handler.shutdown() + + +class TestHttpRestAdapterErrorHandling: + """Test suite for error handling.""" + + @pytest.fixture + def handler(self) -> HttpRestAdapter: + """Create HttpRestAdapter fixture.""" + return HttpRestAdapter() + + @pytest.mark.asyncio + async def test_timeout_error_raises_infra_timeout( + self, handler: HttpRestAdapter + ) -> None: + """Test timeout error is converted to InfraTimeoutError.""" + await handler.initialize({}) + + with patch.object(handler._client, "get", new_callable=AsyncMock) as mock_get: + mock_get.side_effect = httpx.TimeoutException("Connection timed out") + + envelope: dict[str, object] = { + "operation": "http.get", + "payload": {"url": "https://slow.example.com/api"}, + } + + with pytest.raises(InfraTimeoutError) as exc_info: + await handler.execute(envelope) + + assert "timed out" in str(exc_info.value) + assert "30" in str(exc_info.value) + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_connection_error_raises_infra_connection( + self, handler: HttpRestAdapter + ) -> None: + """Test connection error is converted to InfraConnectionError.""" + await handler.initialize({}) + + with patch.object(handler._client, "get", new_callable=AsyncMock) as mock_get: + mock_get.side_effect = httpx.ConnectError("Connection refused") + + envelope: dict[str, object] = { + "operation": "http.get", + "payload": {"url": "https://unreachable.example.com/api"}, + } + + with pytest.raises(InfraConnectionError) as exc_info: + await handler.execute(envelope) + + assert "Failed to connect" in str(exc_info.value) + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_unsupported_operation_put_raises_error( + self, handler: HttpRestAdapter + ) -> None: + """Test http.put operation raises RuntimeHostError (not supported in MVP).""" + await handler.initialize({}) + + envelope: dict[str, object] = { + "operation": "http.put", + "payload": {"url": "https://api.example.com/resource/123"}, + } + + with pytest.raises(RuntimeHostError) as exc_info: + await handler.execute(envelope) + + assert "http.put" in str(exc_info.value) + assert "not supported" in str(exc_info.value).lower() + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_unsupported_operation_delete_raises_error( + self, handler: HttpRestAdapter + ) -> None: + """Test http.delete operation raises RuntimeHostError (not supported in MVP).""" + await handler.initialize({}) + + envelope: dict[str, object] = { + "operation": "http.delete", + "payload": {"url": "https://api.example.com/resource/123"}, + } + + with pytest.raises(RuntimeHostError) as exc_info: + await handler.execute(envelope) + + assert "http.delete" in str(exc_info.value) + assert "not supported" in str(exc_info.value).lower() + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_unsupported_operation_patch_raises_error( + self, handler: HttpRestAdapter + ) -> None: + """Test http.patch operation raises RuntimeHostError (not supported in MVP).""" + await handler.initialize({}) + + envelope: dict[str, object] = { + "operation": "http.patch", + "payload": {"url": "https://api.example.com/resource/123"}, + } + + with pytest.raises(RuntimeHostError) as exc_info: + await handler.execute(envelope) + + assert "http.patch" in str(exc_info.value) + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_missing_url_field_raises_error( + self, handler: HttpRestAdapter + ) -> None: + """Test missing URL field raises RuntimeHostError.""" + await handler.initialize({}) + + envelope: dict[str, object] = { + "operation": "http.get", + "payload": {"headers": {"X-Test": "value"}}, # No URL + } + + with pytest.raises(RuntimeHostError) as exc_info: + await handler.execute(envelope) + + assert "url" in str(exc_info.value).lower() + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_empty_url_field_raises_error(self, handler: HttpRestAdapter) -> None: + """Test empty URL field raises RuntimeHostError.""" + await handler.initialize({}) + + envelope: dict[str, object] = { + "operation": "http.get", + "payload": {"url": ""}, + } + + with pytest.raises(RuntimeHostError) as exc_info: + await handler.execute(envelope) + + assert "url" in str(exc_info.value).lower() + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_missing_operation_raises_error( + self, handler: HttpRestAdapter + ) -> None: + """Test missing operation field raises RuntimeHostError.""" + await handler.initialize({}) + + envelope: dict[str, object] = { + "payload": {"url": "https://example.com"}, + } + + with pytest.raises(RuntimeHostError) as exc_info: + await handler.execute(envelope) + + assert "operation" in str(exc_info.value).lower() + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_missing_payload_raises_error(self, handler: HttpRestAdapter) -> None: + """Test missing payload field raises RuntimeHostError.""" + await handler.initialize({}) + + envelope: dict[str, object] = { + "operation": "http.get", + } + + with pytest.raises(RuntimeHostError) as exc_info: + await handler.execute(envelope) + + assert "payload" in str(exc_info.value).lower() + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_invalid_headers_type_raises_error( + self, handler: HttpRestAdapter + ) -> None: + """Test invalid headers type raises RuntimeHostError.""" + await handler.initialize({}) + + envelope: dict[str, object] = { + "operation": "http.get", + "payload": { + "url": "https://example.com", + "headers": "not-a-dict", # Invalid type + }, + } + + with pytest.raises(RuntimeHostError) as exc_info: + await handler.execute(envelope) + + assert "headers" in str(exc_info.value).lower() + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_http_status_error_returns_response( + self, handler: HttpRestAdapter + ) -> None: + """Test HTTPStatusError still returns the response (not an exception).""" + await handler.initialize({}) + + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 404 + mock_response.headers = {"content-type": "application/json"} + mock_response.json.return_value = {"error": "Not found"} + + with patch.object(handler._client, "get", new_callable=AsyncMock) as mock_get: + http_error = httpx.HTTPStatusError( + "404 Not Found", request=MagicMock(), response=mock_response + ) + mock_get.side_effect = http_error + + envelope: dict[str, object] = { + "operation": "http.get", + "payload": {"url": "https://api.example.com/missing"}, + } + + result = await handler.execute(envelope) + + # Should return the error response, not raise + assert result["status"] == "success" + assert result["payload"]["status_code"] == 404 + assert result["payload"]["body"] == {"error": "Not found"} + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_generic_http_error_raises_connection_error( + self, handler: HttpRestAdapter + ) -> None: + """Test generic HTTPError raises InfraConnectionError.""" + await handler.initialize({}) + + with patch.object(handler._client, "get", new_callable=AsyncMock) as mock_get: + mock_get.side_effect = httpx.HTTPError("Unknown HTTP error") + + envelope: dict[str, object] = { + "operation": "http.get", + "payload": {"url": "https://api.example.com/broken"}, + } + + with pytest.raises(InfraConnectionError) as exc_info: + await handler.execute(envelope) + + assert "HTTP error" in str(exc_info.value) + + await handler.shutdown() + + +class TestHttpRestAdapterHealthCheck: + """Test suite for health check operations.""" + + @pytest.fixture + def handler(self) -> HttpRestAdapter: + """Create HttpRestAdapter fixture.""" + return HttpRestAdapter() + + @pytest.mark.asyncio + async def test_health_check_structure(self, handler: HttpRestAdapter) -> None: + """Test health_check returns correct structure.""" + await handler.initialize({}) + + health = await handler.health_check() + + assert "healthy" in health + assert "initialized" in health + assert "adapter_type" in health + assert "timeout_seconds" in health + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_health_check_healthy_when_initialized( + self, handler: HttpRestAdapter + ) -> None: + """Test health_check shows healthy=True when initialized.""" + await handler.initialize({}) + + health = await handler.health_check() + + assert health["healthy"] is True + assert health["initialized"] is True + assert health["adapter_type"] == "http" + assert health["timeout_seconds"] == 30.0 + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_health_check_unhealthy_when_not_initialized( + self, handler: HttpRestAdapter + ) -> None: + """Test health_check shows healthy=False when not initialized.""" + health = await handler.health_check() + + assert health["healthy"] is False + assert health["initialized"] is False + + @pytest.mark.asyncio + async def test_health_check_unhealthy_after_shutdown( + self, handler: HttpRestAdapter + ) -> None: + """Test health_check shows healthy=False after shutdown.""" + await handler.initialize({}) + await handler.shutdown() + + health = await handler.health_check() + + assert health["healthy"] is False + assert health["initialized"] is False + + +class TestHttpRestAdapterDescribe: + """Test suite for describe operations.""" + + @pytest.fixture + def handler(self) -> HttpRestAdapter: + """Create HttpRestAdapter fixture.""" + return HttpRestAdapter() + + def test_describe_returns_handler_metadata(self, handler: HttpRestAdapter) -> None: + """Test describe returns correct handler metadata.""" + description = handler.describe() + + assert description["adapter_type"] == "http" + assert description["timeout_seconds"] == 30.0 + assert description["version"] == "0.1.0-mvp" + assert description["initialized"] is False + + def test_describe_lists_supported_operations( + self, handler: HttpRestAdapter + ) -> None: + """Test describe lists supported operations.""" + description = handler.describe() + + assert "supported_operations" in description + operations = description["supported_operations"] + + assert "http.get" in operations + assert "http.post" in operations + assert len(operations) == 2 + + @pytest.mark.asyncio + async def test_describe_reflects_initialized_state( + self, handler: HttpRestAdapter + ) -> None: + """Test describe shows correct initialized state.""" + assert handler.describe()["initialized"] is False + + await handler.initialize({}) + assert handler.describe()["initialized"] is True + + await handler.shutdown() + assert handler.describe()["initialized"] is False + + +class TestHttpRestAdapterLifecycle: + """Test suite for lifecycle management.""" + + @pytest.fixture + def handler(self) -> HttpRestAdapter: + """Create HttpRestAdapter fixture.""" + return HttpRestAdapter() + + @pytest.mark.asyncio + async def test_shutdown_closes_client(self, handler: HttpRestAdapter) -> None: + """Test shutdown closes the HTTP client properly.""" + await handler.initialize({}) + + client = handler._client + assert client is not None + + with patch.object(client, "aclose", new_callable=AsyncMock) as mock_close: + await handler.shutdown() + + mock_close.assert_called_once() + assert handler._client is None + assert handler._initialized is False + + @pytest.mark.asyncio + async def test_execute_after_shutdown_raises_error( + self, handler: HttpRestAdapter + ) -> None: + """Test execute after shutdown raises RuntimeHostError.""" + await handler.initialize({}) + await handler.shutdown() + + envelope: dict[str, object] = { + "operation": "http.get", + "payload": {"url": "https://example.com"}, + } + + with pytest.raises(RuntimeHostError) as exc_info: + await handler.execute(envelope) + + assert "not initialized" in str(exc_info.value).lower() + + @pytest.mark.asyncio + async def test_execute_before_initialize_raises_error( + self, handler: HttpRestAdapter + ) -> None: + """Test execute before initialize raises RuntimeHostError.""" + envelope: dict[str, object] = { + "operation": "http.get", + "payload": {"url": "https://example.com"}, + } + + with pytest.raises(RuntimeHostError) as exc_info: + await handler.execute(envelope) + + assert "not initialized" in str(exc_info.value).lower() + + @pytest.mark.asyncio + async def test_multiple_shutdown_calls_safe(self, handler: HttpRestAdapter) -> None: + """Test multiple shutdown calls are safe (idempotent).""" + await handler.initialize({}) + await handler.shutdown() + await handler.shutdown() # Second call should not raise + + assert handler._initialized is False + assert handler._client is None + + @pytest.mark.asyncio + async def test_reinitialize_after_shutdown(self, handler: HttpRestAdapter) -> None: + """Test handler can be reinitialized after shutdown.""" + await handler.initialize({}) + await handler.shutdown() + + assert handler._initialized is False + + await handler.initialize({}) + + assert handler._initialized is True + assert handler._client is not None + + await handler.shutdown() + + +class TestHttpRestAdapterCorrelationId: + """Test suite for correlation ID handling.""" + + @pytest.fixture + def handler(self) -> HttpRestAdapter: + """Create HttpRestAdapter fixture.""" + return HttpRestAdapter() + + @pytest.mark.asyncio + async def test_correlation_id_from_envelope_uuid( + self, handler: HttpRestAdapter + ) -> None: + """Test correlation ID extracted from envelope as UUID.""" + await handler.initialize({}) + + correlation_id = uuid4() + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/json"} + mock_response.json.return_value = {} + + with patch.object(handler._client, "get", new_callable=AsyncMock) as mock_get: + mock_get.return_value = mock_response + + envelope: dict[str, object] = { + "operation": "http.get", + "payload": {"url": "https://example.com"}, + "correlation_id": correlation_id, + } + + result = await handler.execute(envelope) + + assert result["correlation_id"] == str(correlation_id) + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_correlation_id_from_envelope_string( + self, handler: HttpRestAdapter + ) -> None: + """Test correlation ID extracted from envelope as string.""" + await handler.initialize({}) + + correlation_id = str(uuid4()) + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/json"} + mock_response.json.return_value = {} + + with patch.object(handler._client, "get", new_callable=AsyncMock) as mock_get: + mock_get.return_value = mock_response + + envelope: dict[str, object] = { + "operation": "http.get", + "payload": {"url": "https://example.com"}, + "correlation_id": correlation_id, + } + + result = await handler.execute(envelope) + + assert result["correlation_id"] == correlation_id + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_correlation_id_generated_when_missing( + self, handler: HttpRestAdapter + ) -> None: + """Test correlation ID generated when not in envelope.""" + await handler.initialize({}) + + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/json"} + mock_response.json.return_value = {} + + with patch.object(handler._client, "get", new_callable=AsyncMock) as mock_get: + mock_get.return_value = mock_response + + envelope: dict[str, object] = { + "operation": "http.get", + "payload": {"url": "https://example.com"}, + } + + result = await handler.execute(envelope) + + # Should have a generated UUID + assert "correlation_id" in result + # Verify it's a valid UUID string + UUID(result["correlation_id"]) + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_correlation_id_invalid_string_generates_new( + self, handler: HttpRestAdapter + ) -> None: + """Test invalid correlation ID string generates new UUID.""" + await handler.initialize({}) + + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/json"} + mock_response.json.return_value = {} + + with patch.object(handler._client, "get", new_callable=AsyncMock) as mock_get: + mock_get.return_value = mock_response + + envelope: dict[str, object] = { + "operation": "http.get", + "payload": {"url": "https://example.com"}, + "correlation_id": "not-a-valid-uuid", + } + + result = await handler.execute(envelope) + + # Should have a generated UUID (not the invalid string) + assert "correlation_id" in result + generated_id = result["correlation_id"] + assert generated_id != "not-a-valid-uuid" + # Verify it's a valid UUID string + UUID(generated_id) + + await handler.shutdown() + + +class TestHttpRestAdapterResponseParsing: + """Test suite for response parsing.""" + + @pytest.fixture + def handler(self) -> HttpRestAdapter: + """Create HttpRestAdapter fixture.""" + return HttpRestAdapter() + + @pytest.mark.asyncio + async def test_json_response_parsed(self, handler: HttpRestAdapter) -> None: + """Test JSON response is parsed correctly.""" + await handler.initialize({}) + + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/json; charset=utf-8"} + mock_response.json.return_value = {"key": "value", "nested": {"a": 1}} + + with patch.object(handler._client, "get", new_callable=AsyncMock) as mock_get: + mock_get.return_value = mock_response + + envelope: dict[str, object] = { + "operation": "http.get", + "payload": {"url": "https://example.com"}, + } + + result = await handler.execute(envelope) + + assert result["payload"]["body"] == {"key": "value", "nested": {"a": 1}} + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_invalid_json_returns_text(self, handler: HttpRestAdapter) -> None: + """Test invalid JSON response falls back to text.""" + import json as json_module + + await handler.initialize({}) + + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/json"} + mock_response.json.side_effect = json_module.JSONDecodeError( + "Invalid JSON", "doc", 0 + ) + mock_response.text = "Not valid JSON {" + + with patch.object(handler._client, "get", new_callable=AsyncMock) as mock_get: + mock_get.return_value = mock_response + + envelope: dict[str, object] = { + "operation": "http.get", + "payload": {"url": "https://example.com"}, + } + + result = await handler.execute(envelope) + + assert result["payload"]["body"] == "Not valid JSON {" + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_non_json_content_type_returns_text( + self, handler: HttpRestAdapter + ) -> None: + """Test non-JSON content type returns text body.""" + await handler.initialize({}) + + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = {"content-type": "text/html; charset=utf-8"} + mock_response.text = "Hello" + + with patch.object(handler._client, "get", new_callable=AsyncMock) as mock_get: + mock_get.return_value = mock_response + + envelope: dict[str, object] = { + "operation": "http.get", + "payload": {"url": "https://example.com"}, + } + + result = await handler.execute(envelope) + + assert result["payload"]["body"] == "Hello" + + await handler.shutdown() + + @pytest.mark.asyncio + async def test_response_headers_included(self, handler: HttpRestAdapter) -> None: + """Test response headers are included in result.""" + await handler.initialize({}) + + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = { + "content-type": "application/json", + "x-request-id": "req-123", + "x-rate-limit-remaining": "99", + } + mock_response.json.return_value = {} + + with patch.object(handler._client, "get", new_callable=AsyncMock) as mock_get: + mock_get.return_value = mock_response + + envelope: dict[str, object] = { + "operation": "http.get", + "payload": {"url": "https://example.com"}, + } + + result = await handler.execute(envelope) + + assert result["payload"]["headers"]["content-type"] == "application/json" + assert result["payload"]["headers"]["x-request-id"] == "req-123" + assert result["payload"]["headers"]["x-rate-limit-remaining"] == "99" + + await handler.shutdown() + + +__all__: list[str] = [ + "TestHttpRestAdapterInitialization", + "TestHttpRestAdapterGetOperations", + "TestHttpRestAdapterPostOperations", + "TestHttpRestAdapterErrorHandling", + "TestHttpRestAdapterHealthCheck", + "TestHttpRestAdapterDescribe", + "TestHttpRestAdapterLifecycle", + "TestHttpRestAdapterCorrelationId", + "TestHttpRestAdapterResponseParsing", +]