diff --git a/config/validation/handler_dispatch_entrypoint_baseline.yaml b/config/validation/handler_dispatch_entrypoint_baseline.yaml index a3414fbb08..6458cbcb17 100644 --- a/config/validation/handler_dispatch_entrypoint_baseline.yaml +++ b/config/validation/handler_dispatch_entrypoint_baseline.yaml @@ -67,9 +67,6 @@ known_entrypointless: - contract: node_model_health_effect handler: HandlerProbeHealth module: omnibase_infra.nodes.node_model_health_effect.handlers.handler_probe_health - - contract: node_model_router_compute - handler: HandlerScoreModels - module: omnibase_infra.nodes.node_model_router_compute.handlers.handler_score_models - contract: node_registration_storage_effect handler: HandlerRegistrationStorageMock module: omnibase_infra.handlers.registration_storage.handler_registration_storage_mock diff --git a/scripts/ci/adequacy_receipts/omnibase_infra.nodes.node_model_router_compute.handflip.json b/scripts/ci/adequacy_receipts/omnibase_infra.nodes.node_model_router_compute.handflip.json new file mode 100644 index 0000000000..17fb69b30c --- /dev/null +++ b/scripts/ci/adequacy_receipts/omnibase_infra.nodes.node_model_router_compute.handflip.json @@ -0,0 +1,33 @@ +{ + "base_ref": "a87e5057f1b930c31bbad5e1e5fa215f962629d4", + "canonical_handler_module": "omnibase_infra.nodes.node_model_router_compute.handlers.handler_score_models", + "legacy_handler_module": "omnibase_infra.nodes.node_model_router_compute.handlers.handler_score_models", + "node_id": "omnibase_infra.nodes.node_model_router_compute", + "parity": { + "selected_input_hashes": [ + "sha256:1fafe196504d954c7a2ceb4189e4c9ff9e943fd9508229435d403194b03d7d4b", + "sha256:95a94c4db487c77e3b6f8dac94d2237f6ba8e28257140e02b834eaad064e7f5b", + "sha256:69d42fd7417f1162362a63da06aa892156715e4fa6e2a2f38d3e61d747d5c043", + "sha256:eda6bf7ca95e765816dc81059d90033b86049cfb89652b2f1f2f5bde5b7467a8", + "sha256:b1b7f8d27abb387363e8d0410ab19e0f3ac795bff2bf58882a0cb43f99aa29ff", + "sha256:3307bfa257f8eb127f2b1e808778c660166132767b8631ebcaf6c39638220cdd", + "sha256:80038766312f579f667536cc061c1e3817257ba52fad6fa8dfc61e5c478c5776", + "sha256:45e9ecd1e409c4929e6d0f4c843d379c12419ff4d759fdb9ea391d677209a087", + "sha256:fded21763b6bee5d0dad438ff1db32f6591743fe5edd29a57f111ffec57dccf0" + ], + "status": "pass", + "test_ids": [ + "tests/unit/nodes/test_model_router/test_score_models_defb_omn14825.py::TestModelRouterDefBFlip::test_handle_entrypoint_exists", + "tests/unit/nodes/test_model_router/test_score_models_defb_omn14825.py::TestModelRouterDefBFlip::test_defb_handle_reproduces_legacy_goldens", + "tests/unit/nodes/test_model_router/test_score_models_defb_omn14825.py::TestModelRouterDefBFlip::test_golden_equivalence_is_non_vacuous" + ] + }, + "preserved_symbols": [ + "_capability_match", + "_effective_cost_per_1k", + "_passes_hard_constraints", + "_compute_score", + "ScoringContext.__init__" + ], + "receipt_schema": "handflip_proof.v1" +} diff --git a/scripts/ci/adequacy_receipts/omnibase_infra.nodes.node_model_router_compute.json b/scripts/ci/adequacy_receipts/omnibase_infra.nodes.node_model_router_compute.json new file mode 100644 index 0000000000..41bde9a0c8 --- /dev/null +++ b/scripts/ci/adequacy_receipts/omnibase_infra.nodes.node_model_router_compute.json @@ -0,0 +1,29 @@ +{ + "branch_coverage_pct": 74.13, + "candidate_count": 11, + "coverage_target": 80.0, + "handler_module": "omnibase_infra.nodes.node_model_router_compute.handlers.handler_score_models", + "handler_module_sha256": "sha256:01fb59552d3a6854d3600b8f3175f7deb2ff8ce518a9b2524034935ee1c2d7af", + "meets_target": false, + "node_id": "omnibase_infra.nodes.node_model_router_compute", + "receipt_schema": "adequacy_receipt.v1", + "recorded_at": "2026-07-20T05:22:00.905592+00:00", + "recorder_schema": "adequacy_receipt.v1", + "selected_count": 9, + "selected_input_hashes": [ + "sha256:1fafe196504d954c7a2ceb4189e4c9ff9e943fd9508229435d403194b03d7d4b", + "sha256:95a94c4db487c77e3b6f8dac94d2237f6ba8e28257140e02b834eaad064e7f5b", + "sha256:69d42fd7417f1162362a63da06aa892156715e4fa6e2a2f38d3e61d747d5c043", + "sha256:eda6bf7ca95e765816dc81059d90033b86049cfb89652b2f1f2f5bde5b7467a8", + "sha256:b1b7f8d27abb387363e8d0410ab19e0f3ac795bff2bf58882a0cb43f99aa29ff", + "sha256:3307bfa257f8eb127f2b1e808778c660166132767b8631ebcaf6c39638220cdd", + "sha256:80038766312f579f667536cc061c1e3817257ba52fad6fa8dfc61e5c478c5776", + "sha256:45e9ecd1e409c4929e6d0f4c843d379c12419ff4d759fdb9ea391d677209a087", + "sha256:fded21763b6bee5d0dad438ff1db32f6591743fe5edd29a57f111ffec57dccf0" + ], + "uncovered_waiver": { + "reason": "OMN-14825 hand-flips node_model_router_compute to canonical def-B via a PURE RENAME score_candidates->handle (OMN-14781 path). The scoring body and every module-level scoring helper (_capability_match/_effective_cost_per_1k/_passes_hard_constraints/_compute_score/ScoringContext) are byte-identical base_ref<->HEAD, re-derived from git by the ratchet; equivalence is proven by that git-verbatim preservation plus the green legacy-vs-def-B golden parity over the selected input corpus, not by a broad branch-coverage claim.", + "uncovered_branch_count": 0 + }, + "volatile_mask": [] +} diff --git a/scripts/ci/canonical_handler_shape_baseline.py b/scripts/ci/canonical_handler_shape_baseline.py index c9a1513bd6..75bf6b9d9d 100644 --- a/scripts/ci/canonical_handler_shape_baseline.py +++ b/scripts/ci/canonical_handler_shape_baseline.py @@ -51,7 +51,6 @@ "omnibase_infra.nodes.node_merge_sweep_workflow_orchestrator", "omnibase_infra.nodes.node_merge_sweep_workflow_reducer", "omnibase_infra.nodes.node_model_health_effect", - "omnibase_infra.nodes.node_model_router_compute", "omnibase_infra.nodes.node_node_graph_reducer", "omnibase_infra.nodes.node_onboarding_orchestrator", "omnibase_infra.nodes.node_pattern_lifecycle_effect", diff --git a/src/omnibase_infra/nodes/node_model_router_compute/contract.yaml b/src/omnibase_infra/nodes/node_model_router_compute/contract.yaml index 6237f7f6b3..c53fa0b1b3 100644 --- a/src/omnibase_infra/nodes/node_model_router_compute/contract.yaml +++ b/src/omnibase_infra/nodes/node_model_router_compute/contract.yaml @@ -7,14 +7,18 @@ # # Tickets: OMN-7267, OMN-7841 # +# Related Tickets: +# - OMN-14825: HandlerScoreModels exposes the canonical def-B handle() +# dispatch entrypoint while preserving deterministic model scoring. +# name: "node_model_router_compute" node_type: "COMPUTE_GENERIC" contract_version: major: 1 minor: 0 - patch: 1 -node_version: "1.0.1" -description: "Pure scoring of candidate models by quality, measured cost tier, live speed metrics, and availability." + patch: 2 +node_version: "1.0.2" +description: "Pure scoring of candidate models by quality, measured cost tier, live speed metrics, and availability. Contract patch 2 records the OMN-14825 canonical def-B handler entrypoint flip while preserving deterministic model scoring." input_model: name: "ModelScoringInput" module: "omnibase_infra.nodes.node_model_router_compute.models.model_scoring_input" diff --git a/src/omnibase_infra/nodes/node_model_router_compute/handlers/handler_score_models.py b/src/omnibase_infra/nodes/node_model_router_compute/handlers/handler_score_models.py index bd969bee48..7f132fc62c 100644 --- a/src/omnibase_infra/nodes/node_model_router_compute/handlers/handler_score_models.py +++ b/src/omnibase_infra/nodes/node_model_router_compute/handlers/handler_score_models.py @@ -179,9 +179,7 @@ def handler_type(self) -> EnumHandlerType: def handler_category(self) -> EnumHandlerTypeCategory: return EnumHandlerTypeCategory.COMPUTE - def score_candidates( - self, scoring_input: ModelScoringInput - ) -> ModelRoutingDecision: + def handle(self, scoring_input: ModelScoringInput) -> ModelRoutingDecision: health_map: dict[str, ModelEndpointHealth] = { e.model_key: e for e in scoring_input.health } diff --git a/src/omnibase_infra/services/routing_api/routes.py b/src/omnibase_infra/services/routing_api/routes.py index e4e6d6cd2c..571f3f837f 100644 --- a/src/omnibase_infra/services/routing_api/routes.py +++ b/src/omnibase_infra/services/routing_api/routes.py @@ -160,7 +160,7 @@ async def route_request(body: ModelRouteRequestBody) -> ModelRoutingDecision: ) handler = HandlerScoreModels() - decision = handler.score_candidates(scoring_input) + decision = handler.handle(scoring_input) if not decision.success: raise HTTPException( diff --git a/src/omnibase_infra/validation/validation_exemptions.yaml b/src/omnibase_infra/validation/validation_exemptions.yaml index 91529d52fb..2c5ad9b139 100644 --- a/src/omnibase_infra/validation/validation_exemptions.yaml +++ b/src/omnibase_infra/validation/validation_exemptions.yaml @@ -281,6 +281,23 @@ pattern_exemptions: - CLAUDE.md (ONEX Architecture - canonical def-B handler entrypoint, OMN-14355) ticket: OMN-14805 # ========================================================================== + # Canonical def-B handler dispatch entrypoint (OMN-14355 / OMN-14825) + # ========================================================================== + # `handle` is the ProtocolMessageHandler dispatch entrypoint that auto-wiring's + # _make_dispatch_callback binds (definition B, operator-locked OMN-14355). It is a + # protocol-defined name, not a candidate for domain-specific renaming. The generic- + # pattern checker only misses the equivalent `handle` on other canonical handlers + # because it visits ast.FunctionDef but not ast.AsyncFunctionDef; a SYNCHRONOUS def-B + # handle is equally canonical and equally mandated. + - file_pattern: 'node_model_router_compute/handlers/handler_score_models\.py' + violation_pattern: "Function name 'handle' is too generic" + reason: > + OMN-14825 hand-flip to canonical def-B: handle(scoring_input: ModelScoringInput) -> ModelRoutingDecision is the architecturally-mandated dispatch entrypoint that auto-wiring binds (definition B, OMN-14355). The name is protocol-defined; renaming it would break the canonical shape this flip establishes. + + documentation: + - CLAUDE.md (ONEX Architecture - canonical def-B handler entrypoint, OMN-14355) + ticket: OMN-14825 + # ========================================================================== # RuntimeHostProcess Exemptions (OMN-756) # ========================================================================== # Central coordinator class that legitimately requires multiple methods for: diff --git a/tests/fixtures/golden/node_model_router_compute/legacy_equivalence_goldens.json b/tests/fixtures/golden/node_model_router_compute/legacy_equivalence_goldens.json new file mode 100644 index 0000000000..eba0e8f955 --- /dev/null +++ b/tests/fixtures/golden/node_model_router_compute/legacy_equivalence_goldens.json @@ -0,0 +1,1062 @@ +{ + "cases": [ + { + "expected": { + "correlation_id": "00000000-0000-0000-0000-000000000001", + "error_message": "", + "estimated_cost": 0.0, + "estimated_latency_ms": 4096000, + "fallback_model_key": "deepseek-r1-32b", + "rationale": "Selected qwen3-coder-30b (score=0.500); task_type=code_generation; tier=local", + "scores": { + "claude-sonnet": 0.275, + "deepseek-r1-32b": 0.42, + "qwen3-coder-30b": 0.5 + }, + "selected_endpoint_env": "LLM_CODER_URL", + "selected_model_key": "qwen3-coder-30b", + "success": true + }, + "input": { + "chain_hit": false, + "chain_hit_model_key": null, + "constraints": { + "max_cost_per_1k": 0.1, + "max_latency_ms": 30000, + "min_context_window": 4096, + "min_quality_score": 0.0, + "needs_computer_use": false, + "needs_tool_use": false, + "needs_vision": false, + "prefer_local": true + }, + "context_length_estimate": 4096, + "correlation_id": "00000000-0000-0000-0000-000000000001", + "health": [], + "live_metrics": [], + "registry": [ + { + "api_key_env": "", + "base_url_env": "LLM_CODER_URL", + "capabilities": [ + "code_generation", + "refactoring" + ], + "concurrency_limit": null, + "context_window": 65536, + "model_key": "qwen3-coder-30b", + "provider": "local", + "tier": "local", + "transport": "http" + }, + { + "api_key_env": "", + "base_url_env": "", + "capabilities": [ + "code_generation", + "reasoning", + "vision" + ], + "concurrency_limit": null, + "context_window": 200000, + "model_key": "claude-sonnet", + "provider": "anthropic", + "tier": "frontier_api", + "transport": "oauth" + }, + { + "api_key_env": "", + "base_url_env": "LLM_DEEPSEEK_R1_URL", + "capabilities": [ + "deep_reasoning", + "code_review" + ], + "concurrency_limit": null, + "context_window": 32768, + "model_key": "deepseek-r1-32b", + "provider": "local", + "tier": "local", + "transport": "http" + } + ], + "scoring_weights": { + "chain_bonus": 0.1, + "cost": 0.3, + "quality": 0.4, + "speed": 0.2 + }, + "task_description": "", + "task_type": "code_generation" + }, + "scenario": "prefer_local_code_gen" + }, + { + "expected": { + "correlation_id": "00000000-0000-0000-0000-000000000002", + "error_message": "", + "estimated_cost": 0.06144, + "estimated_latency_ms": 4096000, + "fallback_model_key": null, + "rationale": "Selected claude-sonnet (score=0.275); task_type=vision; tier=frontier_api", + "scores": { + "claude-sonnet": 0.275 + }, + "selected_endpoint_env": "", + "selected_model_key": "claude-sonnet", + "success": true + }, + "input": { + "chain_hit": false, + "chain_hit_model_key": null, + "constraints": { + "max_cost_per_1k": 0.1, + "max_latency_ms": 30000, + "min_context_window": 4096, + "min_quality_score": 0.0, + "needs_computer_use": false, + "needs_tool_use": false, + "needs_vision": true, + "prefer_local": true + }, + "context_length_estimate": 4096, + "correlation_id": "00000000-0000-0000-0000-000000000002", + "health": [], + "live_metrics": [], + "registry": [ + { + "api_key_env": "", + "base_url_env": "LLM_CODER_URL", + "capabilities": [ + "code_generation", + "refactoring" + ], + "concurrency_limit": null, + "context_window": 65536, + "model_key": "qwen3-coder-30b", + "provider": "local", + "tier": "local", + "transport": "http" + }, + { + "api_key_env": "", + "base_url_env": "", + "capabilities": [ + "code_generation", + "reasoning", + "vision" + ], + "concurrency_limit": null, + "context_window": 200000, + "model_key": "claude-sonnet", + "provider": "anthropic", + "tier": "frontier_api", + "transport": "oauth" + }, + { + "api_key_env": "", + "base_url_env": "LLM_DEEPSEEK_R1_URL", + "capabilities": [ + "deep_reasoning", + "code_review" + ], + "concurrency_limit": null, + "context_window": 32768, + "model_key": "deepseek-r1-32b", + "provider": "local", + "tier": "local", + "transport": "http" + } + ], + "scoring_weights": { + "chain_bonus": 0.1, + "cost": 0.3, + "quality": 0.4, + "speed": 0.2 + }, + "task_description": "", + "task_type": "vision" + }, + "scenario": "needs_vision" + }, + { + "expected": { + "correlation_id": "00000000-0000-0000-0000-000000000003", + "error_message": "No eligible models after constraint filtering.", + "estimated_cost": 0.0, + "estimated_latency_ms": 0, + "fallback_model_key": null, + "rationale": "No models pass hard constraints.", + "scores": {}, + "selected_endpoint_env": "", + "selected_model_key": "", + "success": false + }, + "input": { + "chain_hit": false, + "chain_hit_model_key": null, + "constraints": { + "max_cost_per_1k": 0.1, + "max_latency_ms": 30000, + "min_context_window": 4096, + "min_quality_score": 0.0, + "needs_computer_use": true, + "needs_tool_use": false, + "needs_vision": false, + "prefer_local": true + }, + "context_length_estimate": 4096, + "correlation_id": "00000000-0000-0000-0000-000000000003", + "health": [], + "live_metrics": [], + "registry": [ + { + "api_key_env": "", + "base_url_env": "LLM_CODER_URL", + "capabilities": [ + "code_generation", + "refactoring" + ], + "concurrency_limit": null, + "context_window": 65536, + "model_key": "qwen3-coder-30b", + "provider": "local", + "tier": "local", + "transport": "http" + }, + { + "api_key_env": "", + "base_url_env": "", + "capabilities": [ + "code_generation", + "reasoning", + "vision" + ], + "concurrency_limit": null, + "context_window": 200000, + "model_key": "claude-sonnet", + "provider": "anthropic", + "tier": "frontier_api", + "transport": "oauth" + }, + { + "api_key_env": "", + "base_url_env": "LLM_DEEPSEEK_R1_URL", + "capabilities": [ + "deep_reasoning", + "code_review" + ], + "concurrency_limit": null, + "context_window": 32768, + "model_key": "deepseek-r1-32b", + "provider": "local", + "tier": "local", + "transport": "http" + } + ], + "scoring_weights": { + "chain_bonus": 0.1, + "cost": 0.3, + "quality": 0.4, + "speed": 0.2 + }, + "task_description": "", + "task_type": "code_generation" + }, + "scenario": "no_candidates_computer_use" + }, + { + "expected": { + "correlation_id": "00000000-0000-0000-0000-000000000004", + "error_message": "", + "estimated_cost": 0.0, + "estimated_latency_ms": 4096000, + "fallback_model_key": "claude-sonnet", + "rationale": "Selected deepseek-r1-32b (score=0.420); task_type=code_generation; tier=local", + "scores": { + "claude-sonnet": 0.275, + "deepseek-r1-32b": 0.42 + }, + "selected_endpoint_env": "LLM_DEEPSEEK_R1_URL", + "selected_model_key": "deepseek-r1-32b", + "success": true + }, + "input": { + "chain_hit": false, + "chain_hit_model_key": null, + "constraints": { + "max_cost_per_1k": 0.1, + "max_latency_ms": 30000, + "min_context_window": 4096, + "min_quality_score": 0.0, + "needs_computer_use": false, + "needs_tool_use": false, + "needs_vision": false, + "prefer_local": true + }, + "context_length_estimate": 4096, + "correlation_id": "00000000-0000-0000-0000-000000000004", + "health": [ + { + "error_message": "timeout", + "healthy": false, + "latency_ms": 0, + "model_key": "qwen3-coder-30b", + "queue_depth": 0 + }, + { + "error_message": "", + "healthy": true, + "latency_ms": 0, + "model_key": "claude-sonnet", + "queue_depth": 0 + }, + { + "error_message": "", + "healthy": true, + "latency_ms": 0, + "model_key": "deepseek-r1-32b", + "queue_depth": 0 + } + ], + "live_metrics": [], + "registry": [ + { + "api_key_env": "", + "base_url_env": "LLM_CODER_URL", + "capabilities": [ + "code_generation", + "refactoring" + ], + "concurrency_limit": null, + "context_window": 65536, + "model_key": "qwen3-coder-30b", + "provider": "local", + "tier": "local", + "transport": "http" + }, + { + "api_key_env": "", + "base_url_env": "", + "capabilities": [ + "code_generation", + "reasoning", + "vision" + ], + "concurrency_limit": null, + "context_window": 200000, + "model_key": "claude-sonnet", + "provider": "anthropic", + "tier": "frontier_api", + "transport": "oauth" + }, + { + "api_key_env": "", + "base_url_env": "LLM_DEEPSEEK_R1_URL", + "capabilities": [ + "deep_reasoning", + "code_review" + ], + "concurrency_limit": null, + "context_window": 32768, + "model_key": "deepseek-r1-32b", + "provider": "local", + "tier": "local", + "transport": "http" + } + ], + "scoring_weights": { + "chain_bonus": 0.1, + "cost": 0.3, + "quality": 0.4, + "speed": 0.2 + }, + "task_description": "", + "task_type": "code_generation" + }, + "scenario": "unhealthy_excluded" + }, + { + "expected": { + "correlation_id": "00000000-0000-0000-0000-000000000005", + "error_message": "", + "estimated_cost": 0.0, + "estimated_latency_ms": 4096000, + "fallback_model_key": "qwen3-coder-30b", + "rationale": "Selected deepseek-r1-32b (score=0.520); task_type=code_generation; tier=local; chain_hit_bonus_applied", + "scores": { + "claude-sonnet": 0.275, + "deepseek-r1-32b": 0.52, + "qwen3-coder-30b": 0.5 + }, + "selected_endpoint_env": "LLM_DEEPSEEK_R1_URL", + "selected_model_key": "deepseek-r1-32b", + "success": true + }, + "input": { + "chain_hit": true, + "chain_hit_model_key": "deepseek-r1-32b", + "constraints": { + "max_cost_per_1k": 0.1, + "max_latency_ms": 30000, + "min_context_window": 4096, + "min_quality_score": 0.0, + "needs_computer_use": false, + "needs_tool_use": false, + "needs_vision": false, + "prefer_local": true + }, + "context_length_estimate": 4096, + "correlation_id": "00000000-0000-0000-0000-000000000005", + "health": [], + "live_metrics": [], + "registry": [ + { + "api_key_env": "", + "base_url_env": "LLM_CODER_URL", + "capabilities": [ + "code_generation", + "refactoring" + ], + "concurrency_limit": null, + "context_window": 65536, + "model_key": "qwen3-coder-30b", + "provider": "local", + "tier": "local", + "transport": "http" + }, + { + "api_key_env": "", + "base_url_env": "", + "capabilities": [ + "code_generation", + "reasoning", + "vision" + ], + "concurrency_limit": null, + "context_window": 200000, + "model_key": "claude-sonnet", + "provider": "anthropic", + "tier": "frontier_api", + "transport": "oauth" + }, + { + "api_key_env": "", + "base_url_env": "LLM_DEEPSEEK_R1_URL", + "capabilities": [ + "deep_reasoning", + "code_review" + ], + "concurrency_limit": null, + "context_window": 32768, + "model_key": "deepseek-r1-32b", + "provider": "local", + "tier": "local", + "transport": "http" + } + ], + "scoring_weights": { + "chain_bonus": 0.1, + "cost": 0.3, + "quality": 0.4, + "speed": 0.2 + }, + "task_description": "", + "task_type": "code_generation" + }, + "scenario": "chain_hit_bonus" + }, + { + "expected": { + "correlation_id": "00000000-0000-0000-0000-000000000006", + "error_message": "", + "estimated_cost": 0.0, + "estimated_latency_ms": 4096000, + "fallback_model_key": "deepseek-r1-32b", + "rationale": "Selected qwen3-coder-30b (score=0.500); task_type=code_generation; tier=local", + "scores": { + "deepseek-r1-32b": 0.42, + "qwen3-coder-30b": 0.5 + }, + "selected_endpoint_env": "LLM_CODER_URL", + "selected_model_key": "qwen3-coder-30b", + "success": true + }, + "input": { + "chain_hit": false, + "chain_hit_model_key": null, + "constraints": { + "max_cost_per_1k": 0.0, + "max_latency_ms": 30000, + "min_context_window": 4096, + "min_quality_score": 0.0, + "needs_computer_use": false, + "needs_tool_use": false, + "needs_vision": false, + "prefer_local": true + }, + "context_length_estimate": 4096, + "correlation_id": "00000000-0000-0000-0000-000000000006", + "health": [], + "live_metrics": [], + "registry": [ + { + "api_key_env": "", + "base_url_env": "LLM_CODER_URL", + "capabilities": [ + "code_generation", + "refactoring" + ], + "concurrency_limit": null, + "context_window": 65536, + "model_key": "qwen3-coder-30b", + "provider": "local", + "tier": "local", + "transport": "http" + }, + { + "api_key_env": "", + "base_url_env": "", + "capabilities": [ + "code_generation", + "reasoning", + "vision" + ], + "concurrency_limit": null, + "context_window": 200000, + "model_key": "claude-sonnet", + "provider": "anthropic", + "tier": "frontier_api", + "transport": "oauth" + }, + { + "api_key_env": "", + "base_url_env": "LLM_DEEPSEEK_R1_URL", + "capabilities": [ + "deep_reasoning", + "code_review" + ], + "concurrency_limit": null, + "context_window": 32768, + "model_key": "deepseek-r1-32b", + "provider": "local", + "tier": "local", + "transport": "http" + } + ], + "scoring_weights": { + "chain_bonus": 0.1, + "cost": 0.3, + "quality": 0.4, + "speed": 0.2 + }, + "task_description": "", + "task_type": "code_generation" + }, + "scenario": "cost_cap_filters_frontier" + }, + { + "expected": { + "correlation_id": "00000000-0000-0000-0000-000000000007", + "error_message": "", + "estimated_cost": 0.0, + "estimated_latency_ms": 409600, + "fallback_model_key": "qwen3-coder-30b", + "rationale": "Selected deepseek-r1-32b (score=0.708); task_type=code_generation; tier=local", + "scores": { + "claude-sonnet": 0.275, + "deepseek-r1-32b": 0.708, + "qwen3-coder-30b": 0.5 + }, + "selected_endpoint_env": "LLM_DEEPSEEK_R1_URL", + "selected_model_key": "deepseek-r1-32b", + "success": true + }, + "input": { + "chain_hit": false, + "chain_hit_model_key": null, + "constraints": { + "max_cost_per_1k": 0.1, + "max_latency_ms": 30000, + "min_context_window": 4096, + "min_quality_score": 0.0, + "needs_computer_use": false, + "needs_tool_use": false, + "needs_vision": false, + "prefer_local": true + }, + "context_length_estimate": 4096, + "correlation_id": "00000000-0000-0000-0000-000000000007", + "health": [], + "live_metrics": [ + { + "avg_latency_ms": 500, + "avg_tokens_per_sec": 10.0, + "graduated": true, + "model_key": "deepseek-r1-32b", + "sample_count": 30, + "success_rate": 0.95, + "task_type": "code_generation" + } + ], + "registry": [ + { + "api_key_env": "", + "base_url_env": "LLM_CODER_URL", + "capabilities": [ + "code_generation", + "refactoring" + ], + "concurrency_limit": null, + "context_window": 65536, + "model_key": "qwen3-coder-30b", + "provider": "local", + "tier": "local", + "transport": "http" + }, + { + "api_key_env": "", + "base_url_env": "", + "capabilities": [ + "code_generation", + "reasoning", + "vision" + ], + "concurrency_limit": null, + "context_window": 200000, + "model_key": "claude-sonnet", + "provider": "anthropic", + "tier": "frontier_api", + "transport": "oauth" + }, + { + "api_key_env": "", + "base_url_env": "LLM_DEEPSEEK_R1_URL", + "capabilities": [ + "deep_reasoning", + "code_review" + ], + "concurrency_limit": null, + "context_window": 32768, + "model_key": "deepseek-r1-32b", + "provider": "local", + "tier": "local", + "transport": "http" + } + ], + "scoring_weights": { + "chain_bonus": 0.1, + "cost": 0.3, + "quality": 0.4, + "speed": 0.2 + }, + "task_description": "", + "task_type": "code_generation" + }, + "scenario": "live_metrics_graduated" + }, + { + "expected": { + "correlation_id": "00000000-0000-0000-0000-000000000008", + "error_message": "", + "estimated_cost": 0.0, + "estimated_latency_ms": 27306, + "fallback_model_key": "deepseek-r1-32b", + "rationale": "Selected qwen3-coder-30b (score=0.780); task_type=code_generation; tier=local", + "scores": { + "claude-sonnet": 0.275, + "deepseek-r1-32b": 0.42, + "qwen3-coder-30b": 0.78 + }, + "selected_endpoint_env": "LLM_CODER_URL", + "selected_model_key": "qwen3-coder-30b", + "success": true + }, + "input": { + "chain_hit": false, + "chain_hit_model_key": null, + "constraints": { + "max_cost_per_1k": 0.1, + "max_latency_ms": 30000, + "min_context_window": 4096, + "min_quality_score": 0.0, + "needs_computer_use": false, + "needs_tool_use": false, + "needs_vision": false, + "prefer_local": true + }, + "context_length_estimate": 4096, + "correlation_id": "00000000-0000-0000-0000-000000000008", + "health": [], + "live_metrics": [ + { + "avg_latency_ms": 200, + "avg_tokens_per_sec": 150.0, + "graduated": false, + "model_key": "qwen3-coder-30b", + "sample_count": 25, + "success_rate": 0.9, + "task_type": "code_generation" + } + ], + "registry": [ + { + "api_key_env": "", + "base_url_env": "LLM_CODER_URL", + "capabilities": [ + "code_generation", + "refactoring" + ], + "concurrency_limit": null, + "context_window": 65536, + "model_key": "qwen3-coder-30b", + "provider": "local", + "tier": "local", + "transport": "http" + }, + { + "api_key_env": "", + "base_url_env": "", + "capabilities": [ + "code_generation", + "reasoning", + "vision" + ], + "concurrency_limit": null, + "context_window": 200000, + "model_key": "claude-sonnet", + "provider": "anthropic", + "tier": "frontier_api", + "transport": "oauth" + }, + { + "api_key_env": "", + "base_url_env": "LLM_DEEPSEEK_R1_URL", + "capabilities": [ + "deep_reasoning", + "code_review" + ], + "concurrency_limit": null, + "context_window": 32768, + "model_key": "deepseek-r1-32b", + "provider": "local", + "tier": "local", + "transport": "http" + } + ], + "scoring_weights": { + "chain_bonus": 0.1, + "cost": 0.3, + "quality": 0.4, + "speed": 0.2 + }, + "task_description": "", + "task_type": "code_generation" + }, + "scenario": "live_metrics_measured_speed" + }, + { + "expected": { + "correlation_id": "00000000-0000-0000-0000-000000000009", + "error_message": "", + "estimated_cost": 1.2, + "estimated_latency_ms": 80000000, + "fallback_model_key": null, + "rationale": "Selected claude-sonnet (score=0.275); task_type=code_generation; tier=frontier_api", + "scores": { + "claude-sonnet": 0.275 + }, + "selected_endpoint_env": "", + "selected_model_key": "claude-sonnet", + "success": true + }, + "input": { + "chain_hit": false, + "chain_hit_model_key": null, + "constraints": { + "max_cost_per_1k": 0.1, + "max_latency_ms": 30000, + "min_context_window": 100000, + "min_quality_score": 0.0, + "needs_computer_use": false, + "needs_tool_use": false, + "needs_vision": false, + "prefer_local": true + }, + "context_length_estimate": 80000, + "correlation_id": "00000000-0000-0000-0000-000000000009", + "health": [], + "live_metrics": [], + "registry": [ + { + "api_key_env": "", + "base_url_env": "LLM_CODER_URL", + "capabilities": [ + "code_generation", + "refactoring" + ], + "concurrency_limit": null, + "context_window": 65536, + "model_key": "qwen3-coder-30b", + "provider": "local", + "tier": "local", + "transport": "http" + }, + { + "api_key_env": "", + "base_url_env": "", + "capabilities": [ + "code_generation", + "reasoning", + "vision" + ], + "concurrency_limit": null, + "context_window": 200000, + "model_key": "claude-sonnet", + "provider": "anthropic", + "tier": "frontier_api", + "transport": "oauth" + }, + { + "api_key_env": "", + "base_url_env": "LLM_DEEPSEEK_R1_URL", + "capabilities": [ + "deep_reasoning", + "code_review" + ], + "concurrency_limit": null, + "context_window": 32768, + "model_key": "deepseek-r1-32b", + "provider": "local", + "tier": "local", + "transport": "http" + } + ], + "scoring_weights": { + "chain_bonus": 0.1, + "cost": 0.3, + "quality": 0.4, + "speed": 0.2 + }, + "task_description": "", + "task_type": "code_generation" + }, + "scenario": "min_context_window" + }, + { + "expected": { + "correlation_id": "00000000-0000-0000-0000-00000000000a", + "error_message": "", + "estimated_cost": 0.0, + "estimated_latency_ms": 102400, + "fallback_model_key": "qwen3-coder-30b", + "rationale": "Selected deepseek-r1-32b (score=0.502); task_type=code_generation; tier=local", + "scores": { + "claude-sonnet": 0.275, + "deepseek-r1-32b": 0.502, + "qwen3-coder-30b": 0.5 + }, + "selected_endpoint_env": "LLM_DEEPSEEK_R1_URL", + "selected_model_key": "deepseek-r1-32b", + "success": true + }, + "input": { + "chain_hit": false, + "chain_hit_model_key": null, + "constraints": { + "max_cost_per_1k": 0.1, + "max_latency_ms": 30000, + "min_context_window": 4096, + "min_quality_score": 0.0, + "needs_computer_use": false, + "needs_tool_use": false, + "needs_vision": false, + "prefer_local": true + }, + "context_length_estimate": 4096, + "correlation_id": "00000000-0000-0000-0000-00000000000a", + "health": [], + "live_metrics": [ + { + "avg_latency_ms": 300, + "avg_tokens_per_sec": 40.0, + "graduated": false, + "model_key": "deepseek-r1-32b", + "sample_count": 5, + "success_rate": 0.8, + "task_type": "code_generation" + } + ], + "registry": [ + { + "api_key_env": "", + "base_url_env": "LLM_CODER_URL", + "capabilities": [ + "code_generation", + "refactoring" + ], + "concurrency_limit": null, + "context_window": 65536, + "model_key": "qwen3-coder-30b", + "provider": "local", + "tier": "local", + "transport": "http" + }, + { + "api_key_env": "", + "base_url_env": "", + "capabilities": [ + "code_generation", + "reasoning", + "vision" + ], + "concurrency_limit": null, + "context_window": 200000, + "model_key": "claude-sonnet", + "provider": "anthropic", + "tier": "frontier_api", + "transport": "oauth" + }, + { + "api_key_env": "", + "base_url_env": "LLM_DEEPSEEK_R1_URL", + "capabilities": [ + "deep_reasoning", + "code_review" + ], + "concurrency_limit": null, + "context_window": 32768, + "model_key": "deepseek-r1-32b", + "provider": "local", + "tier": "local", + "transport": "http" + } + ], + "scoring_weights": { + "chain_bonus": 0.1, + "cost": 0.3, + "quality": 0.4, + "speed": 0.2 + }, + "task_description": "", + "task_type": "code_generation" + }, + "scenario": "live_metrics_partial" + }, + { + "expected": { + "correlation_id": "00000000-0000-0000-0000-00000000000b", + "error_message": "", + "estimated_cost": 0.0, + "estimated_latency_ms": 4096000, + "fallback_model_key": "deepseek-r1-32b", + "rationale": "Selected qwen3-coder-30b (score=0.000); task_type=code_generation; tier=local", + "scores": { + "claude-sonnet": 0.0, + "deepseek-r1-32b": 0.0, + "qwen3-coder-30b": 0.0 + }, + "selected_endpoint_env": "LLM_CODER_URL", + "selected_model_key": "qwen3-coder-30b", + "success": true + }, + "input": { + "chain_hit": false, + "chain_hit_model_key": null, + "constraints": { + "max_cost_per_1k": 0.1, + "max_latency_ms": 30000, + "min_context_window": 4096, + "min_quality_score": 0.0, + "needs_computer_use": false, + "needs_tool_use": false, + "needs_vision": false, + "prefer_local": true + }, + "context_length_estimate": 4096, + "correlation_id": "00000000-0000-0000-0000-00000000000b", + "health": [ + { + "error_message": "", + "healthy": true, + "latency_ms": 0, + "model_key": "qwen3-coder-30b", + "queue_depth": 60 + }, + { + "error_message": "", + "healthy": true, + "latency_ms": 0, + "model_key": "claude-sonnet", + "queue_depth": 60 + }, + { + "error_message": "", + "healthy": true, + "latency_ms": 0, + "model_key": "deepseek-r1-32b", + "queue_depth": 60 + } + ], + "live_metrics": [], + "registry": [ + { + "api_key_env": "", + "base_url_env": "LLM_CODER_URL", + "capabilities": [ + "code_generation", + "refactoring" + ], + "concurrency_limit": null, + "context_window": 65536, + "model_key": "qwen3-coder-30b", + "provider": "local", + "tier": "local", + "transport": "http" + }, + { + "api_key_env": "", + "base_url_env": "", + "capabilities": [ + "code_generation", + "reasoning", + "vision" + ], + "concurrency_limit": null, + "context_window": 200000, + "model_key": "claude-sonnet", + "provider": "anthropic", + "tier": "frontier_api", + "transport": "oauth" + }, + { + "api_key_env": "", + "base_url_env": "LLM_DEEPSEEK_R1_URL", + "capabilities": [ + "deep_reasoning", + "code_review" + ], + "concurrency_limit": null, + "context_window": 32768, + "model_key": "deepseek-r1-32b", + "provider": "local", + "tier": "local", + "transport": "http" + } + ], + "scoring_weights": { + "chain_bonus": 0.1, + "cost": 0.3, + "quality": 0.4, + "speed": 0.2 + }, + "task_description": "", + "task_type": "code_generation" + }, + "scenario": "queue_depth_unavailable" + } + ], + "golden_version": "compute_golden.v1", + "handler_module": "omnibase_infra.nodes.node_model_router_compute.handlers.handler_score_models", + "node_id": "omnibase_infra.nodes.node_model_router_compute", + "recorded_from_base_ref": "a87e5057f1b930c31bbad5e1e5fa215f962629d4" +} diff --git a/tests/integration/test_model_router_registry_scoring_integration.py b/tests/integration/test_model_router_registry_scoring_integration.py index b3312e0a0b..877f94441f 100644 --- a/tests/integration/test_model_router_registry_scoring_integration.py +++ b/tests/integration/test_model_router_registry_scoring_integration.py @@ -75,7 +75,7 @@ def test_registry_loads_without_seed_fields_and_routes_by_live_metrics( ), ) - decision = HandlerScoreModels().score_candidates(scoring_input) + decision = HandlerScoreModels().handle(scoring_input) assert decision.success assert decision.selected_model_key == measured_model diff --git a/tests/unit/nodes/test_model_router/test_score_models.py b/tests/unit/nodes/test_model_router/test_score_models.py index cf623a93a0..51f4aa6efc 100644 --- a/tests/unit/nodes/test_model_router/test_score_models.py +++ b/tests/unit/nodes/test_model_router/test_score_models.py @@ -78,7 +78,7 @@ def test_selects_local_model_for_code_generation(self) -> None: registry=registry, ) - decision = handler.score_candidates(scoring_input) + decision = handler.handle(scoring_input) assert decision.success assert decision.selected_model_key == "qwen3-coder-30b" @@ -97,7 +97,7 @@ def test_selects_vision_model_when_required(self) -> None: registry=registry, ) - decision = handler.score_candidates(scoring_input) + decision = handler.handle(scoring_input) assert decision.success assert decision.selected_model_key == "claude-sonnet" @@ -116,7 +116,7 @@ def test_no_candidates_returns_failure(self) -> None: registry=registry, ) - decision = handler.score_candidates(scoring_input) + decision = handler.handle(scoring_input) assert not decision.success assert decision.selected_model_key == "" @@ -143,7 +143,7 @@ def test_unhealthy_model_excluded(self) -> None: health=health, ) - decision = handler.score_candidates(scoring_input) + decision = handler.handle(scoring_input) assert decision.success assert decision.selected_model_key != "qwen3-coder-30b" @@ -159,7 +159,7 @@ def test_chain_hit_boosts_target_model(self) -> None: context_length_estimate=4096, registry=registry, ) - decision_no_chain = handler.score_candidates(input_no_chain) + decision_no_chain = handler.handle(input_no_chain) score_no_chain = decision_no_chain.scores.get("deepseek-r1-32b", 0) input_chain = ModelScoringInput( @@ -171,7 +171,7 @@ def test_chain_hit_boosts_target_model(self) -> None: chain_hit=True, chain_hit_model_key="deepseek-r1-32b", ) - decision_chain = handler.score_candidates(input_chain) + decision_chain = handler.handle(input_chain) score_chain = decision_chain.scores.get("deepseek-r1-32b", 0) assert score_chain > score_no_chain @@ -188,7 +188,7 @@ def test_cost_cap_filters_frontier_models(self) -> None: registry=registry, ) - decision = handler.score_candidates(scoring_input) + decision = handler.handle(scoring_input) assert decision.success assert decision.selected_model_key in ("qwen3-coder-30b", "deepseek-r1-32b") @@ -218,7 +218,7 @@ def test_live_metrics_influence_scoring(self) -> None: live_metrics=live_metrics, ) - decision = handler.score_candidates(scoring_input) + decision = handler.handle(scoring_input) assert decision.scores["deepseek-r1-32b"] > 0 @@ -246,7 +246,7 @@ def test_unmeasured_models_get_lowest_speed(self) -> None: live_metrics=live_metrics, ) - decision = handler.score_candidates(scoring_input) + decision = handler.handle(scoring_input) qwen_score = decision.scores["qwen3-coder-30b"] deepseek_score = decision.scores["deepseek-r1-32b"] @@ -264,7 +264,7 @@ def test_context_window_constraint(self) -> None: registry=registry, ) - decision = handler.score_candidates(scoring_input) + decision = handler.handle(scoring_input) assert decision.success assert decision.selected_model_key == "claude-sonnet" @@ -282,8 +282,8 @@ def test_deterministic_for_same_inputs(self) -> None: registry=registry, ) - d1 = handler.score_candidates(scoring_input) - d2 = handler.score_candidates(scoring_input) + d1 = handler.handle(scoring_input) + d2 = handler.handle(scoring_input) assert d1.selected_model_key == d2.selected_model_key assert d1.scores == d2.scores diff --git a/tests/unit/nodes/test_model_router/test_score_models_defb_omn14825.py b/tests/unit/nodes/test_model_router/test_score_models_defb_omn14825.py new file mode 100644 index 0000000000..72eec44751 --- /dev/null +++ b/tests/unit/nodes/test_model_router/test_score_models_defb_omn14825.py @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +"""Canonical def-B flip proof for node_model_router_compute (OMN-14825). + +RED->GREEN: on the pre-flip tree ``HandlerScoreModels`` exposed only the def-A +operation method ``score_candidates`` and NO ``handle`` entrypoint, so every +assertion here AttributeErrors / fails (RED). The canonical def-B flip renames +that method to ``handle(request) -> response`` (byte-identical body); these tests +then pass (GREEN). + +``test_defb_handle_reproduces_legacy_goldens`` replays routing decisions recorded +against the git-BASE legacy handler through the live def-B ``handle`` and asserts +byte-equality (behaviour equivalence). ``test_golden_equivalence_is_non_vacuous`` +proves the comparator would CATCH a divergence (RED-vs-exists-but-wrong). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from omnibase_infra.nodes.node_model_router_compute.handlers.handler_score_models import ( + HandlerScoreModels, +) +from omnibase_infra.nodes.node_model_router_compute.models.model_scoring_input import ( + ModelScoringInput, +) + +_GOLDEN = ( + Path(__file__).resolve().parents[3] + / "fixtures/golden/node_model_router_compute/legacy_equivalence_goldens.json" +) + + +def _load_cases() -> list[dict[str, object]]: + data = json.loads(_GOLDEN.read_text(encoding="utf-8")) + cases = data["cases"] + assert isinstance(cases, list) + return cases + + +@pytest.mark.unit +class TestModelRouterDefBFlip: + """Proof that HandlerScoreModels is canonical definition B and behaviour-equivalent.""" + + def test_handle_entrypoint_exists(self) -> None: + handler = HandlerScoreModels() + # Canonical def-B: the runtime adapts handle(request) -> response. + assert hasattr(handler, "handle") + assert callable(handler.handle) + # The legacy def-A operation method must be gone (no retained shim). + assert not hasattr(handler, "score_candidates") + + def test_defb_handle_reproduces_legacy_goldens(self) -> None: + handler = HandlerScoreModels() + cases = _load_cases() + assert cases, "golden corpus must be non-empty" + for case in cases: + inp = ModelScoringInput.model_validate(case["input"]) + decision = handler.handle(inp) + assert decision.model_dump(mode="json") == case["expected"], case[ + "scenario" + ] + + def test_golden_equivalence_is_non_vacuous(self) -> None: + handler = HandlerScoreModels() + case = _load_cases()[0] + inp = ModelScoringInput.model_validate(case["input"]) + decision = handler.handle(inp) + perturbed = dict(case["expected"]) + perturbed["selected_model_key"] = ( + str(perturbed["selected_model_key"]) + "__WRONG" + ) + assert decision.model_dump(mode="json") != perturbed