diff --git a/contextual_orchestrator/admin.py b/contextual_orchestrator/admin.py index b025b5105..d5c8914e5 100644 --- a/contextual_orchestrator/admin.py +++ b/contextual_orchestrator/admin.py @@ -21,6 +21,14 @@ "active_status": "Active", "agent_pool_title": "Agent Pool", "register_agent": "Register Agent", + "model_groups_title": "Model groups", + "group_name_label": "Group name", + "group_members_label": "Provider model members", + "save_group": "Save group", + "delete_group": "Delete group", + "no_model_groups": "No model groups yet. Create one to route a logical model across providers.", + "group_saved": "Group saved. Send requests with this group name to use measured routing.", + "group_deleted": "Group deleted. Its provider models remain available.", "search_agents": "Search agents", "all_statuses": "All statuses", "no_agents_match": "No agents match the current filters.", @@ -255,6 +263,14 @@ "active_status": "활성", "agent_pool_title": "에이전트 풀", "register_agent": "에이전트 등록", + "model_groups_title": "모델 그룹", + "group_name_label": "그룹 이름", + "group_members_label": "공급자 모델 멤버", + "save_group": "그룹 저장", + "delete_group": "그룹 삭제", + "no_model_groups": "모델 그룹이 없습니다. 논리 모델을 여러 공급자로 라우팅하려면 그룹을 만드세요.", + "group_saved": "그룹을 저장했습니다. 측정 기반 라우팅에는 이 그룹 이름으로 요청하세요.", + "group_deleted": "그룹을 삭제했습니다. 공급자 모델은 그대로 사용할 수 있습니다.", "search_agents": "에이전트 검색", "all_statuses": "전체 상태", "no_agents_match": "현재 필터와 일치하는 에이전트가 없습니다.", @@ -869,6 +885,18 @@ +
+

Model groups

+ + +
+

+ +
@@ -1032,10 +1060,15 @@ viewAudit: document.querySelector("#viewAudit"), agentSettings: document.querySelector("#agentSettings"), registerAgent: document.querySelector("#registerAgent"), + modelGroupForm: document.querySelector("#modelGroupForm"), + modelGroupName: document.querySelector("#modelGroupName"), + modelGroupMembers: document.querySelector("#modelGroupMembers"), + modelGroupFeedback: document.querySelector("#modelGroupFeedback"), + modelGroups: document.querySelector("#modelGroups"), mobileView: document.querySelector("#mobileView"), language: document.querySelector("#language") }; - let state = {agents: [], last: null, analytics: null, readiness: null, buyerHandoffBundle: null, saleabilityDecision: null, commercialEvidenceExport: null, commercialAcceptanceCheck: null, commercialReleaseCandidate: null, commercialGapRegister: null, commercialProcurementReadiness: null, commercialContractReadiness: null, commercialOnboardingReadiness: null, commercialOperationsReadiness: null, commercialSecurityAttestation: null, commercialValueReadiness: null, commercialCloseReadiness: null, commercialGoToMarketReadiness: null, commercialLaunchReadiness: null, commercialCompletionScorecard: null, commercialBuyerAcceptanceWorkflow: null, commercialDemoScenarios: null, commercialProposalPacket: null, commercialPurchaseApprovalPacket: null, commercialDueDiligenceRoom: null, commercialInvestmentCommitteeMemo: null}; + let state = {agents: [], modelGroups: [], last: null, analytics: null, readiness: null, buyerHandoffBundle: null, saleabilityDecision: null, commercialEvidenceExport: null, commercialAcceptanceCheck: null, commercialReleaseCandidate: null, commercialGapRegister: null, commercialProcurementReadiness: null, commercialContractReadiness: null, commercialOnboardingReadiness: null, commercialOperationsReadiness: null, commercialSecurityAttestation: null, commercialValueReadiness: null, commercialCloseReadiness: null, commercialGoToMarketReadiness: null, commercialLaunchReadiness: null, commercialCompletionScorecard: null, commercialBuyerAcceptanceWorkflow: null, commercialDemoScenarios: null, commercialProposalPacket: null, commercialPurchaseApprovalPacket: null, commercialDueDiligenceRoom: null, commercialInvestmentCommitteeMemo: null}; let currentLang = "en"; let activeTraceTab = "timeline"; const datasets = [ @@ -1077,6 +1110,41 @@ ${agent.group_routing ? `${(agent.group_routing.success_posterior_mean * 100).toFixed(1)}%` : "—"} `).join("") || `${t("no_agents_match")}`; } + function renderModelGroups() { + const selected = new Set(Array.from(els.modelGroupMembers.selectedOptions).map(option => option.value)); + els.modelGroupMembers.innerHTML = state.agents.map(agent => + `` + ).join(""); + els.modelGroups.innerHTML = state.modelGroups.map(group => ` +
  • ${escapeHtml(group.group_name)} — ${escapeHtml(group.member_agent_ids.join(", "))}
    + ${escapeHtml(Object.entries(group.capability_coverage || {}).map(([name, count]) => `${name}: ${count}`).join(" · "))} + +
    • `).join("") || `
  • ${t("no_model_groups")}
    • `; + } + + async function refreshModelGroups() { + const response = await fetch("/api/v1/model_groups"); + const payload = await response.json(); + if (!response.ok) throw new Error(payload.error?.message || "Could not load model groups"); + state.modelGroups = payload.items || []; + renderModelGroups(); + } + + async function saveModelGroup(event) { + event.preventDefault(); + const groupName = els.modelGroupName.value.trim(); + const memberIds = Array.from(els.modelGroupMembers.selectedOptions).map(option => option.value); + const exists = state.modelGroups.some(group => group.group_name === groupName.replaceAll("-", "_").toLowerCase()); + const response = await fetch(exists ? `/api/v1/model_groups/${encodeURIComponent(groupName)}` : "/api/v1/model_groups", { + method: exists ? "PATCH" : "POST", + headers: {"content-type": "application/json"}, + body: JSON.stringify(exists ? {member_agent_ids: memberIds} : {group_name: groupName, member_agent_ids: memberIds}) + }); + const payload = await response.json(); + if (!response.ok) throw new Error(payload.error?.message || "Could not save model group"); + els.modelGroupFeedback.textContent = t("group_saved"); + await refreshModelGroups(); + } function renderTrace(result) { els.traceMode.textContent = result.mode; const trace = result.trace || []; @@ -1471,11 +1539,13 @@ }); localStorage.setItem("admin_lang", lang); if (state.agents.length) renderAgents(); + if (state.agents.length) renderModelGroups(); if (state.policy) renderSecondaryViews(); } async function load() { const res = await fetch("/admin/state"); state = await res.json(); + await refreshModelGroups(); await refreshAnalytics(); await refreshReadiness(); els.hintCount.textContent = state.policy.complex_hints.length; @@ -1572,6 +1642,20 @@ els.viewAudit.addEventListener("click", () => showView("audit")); els.agentSettings.addEventListener("click", () => showView("settings")); els.registerAgent.addEventListener("click", () => showView("integrations")); + els.modelGroupForm.addEventListener("submit", event => saveModelGroup(event).catch(error => { + els.modelGroupFeedback.textContent = error.message; + })); + els.modelGroups.addEventListener("click", event => { + const name = event.target.dataset?.deleteGroup; + if (!name) return; + fetch(`/api/v1/model_groups/${encodeURIComponent(name)}`, {method: "DELETE"}) + .then(response => response.ok ? response.json() : Promise.reject(new Error("Could not delete model group"))) + .then(() => { + els.modelGroupFeedback.textContent = t("group_deleted"); + return refreshModelGroups(); + }) + .catch(error => { els.modelGroupFeedback.textContent = error.message; }); + }); els.language.addEventListener("change", () => applyI18n(els.language.value)); els.mobileView.addEventListener("change", () => showView(els.mobileView.value)); document.querySelector("#copyJson").addEventListener("click", () => { diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index b9be7d606..0b94c4260 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -51,6 +51,9 @@ ProviderDestination = tuple[int, tuple[Any, ...]] MAX_LOCAL_CONCURRENCY = 64 DEFAULT_PROVIDER_PROBE_TIMEOUT = 5.0 +MODEL_CAPABILITIES = frozenset( + {"text", "image", "video", "speech", "transcription", "embedding", "rerank", "audio"} +) MAX_PROVIDER_PROBE_TIMEOUT = 30.0 _SAFE_PROVIDER_PROBE_ERROR_TYPES = frozenset({ "ConnectionError", @@ -1585,29 +1588,67 @@ def load_agents(path: str) -> list[ModelAgent]: # pragma: no cover class _AgentPoolStore: - """Durable agent-pool (model group) storage: operator changes survive restarts. - - ponytail: one sqlite table keyed by agent id, JSON payloads, thread-safe. The - seed agents file stays the bootstrap; stored rows overlay it at startup. - """ + """Durable, normalized agent and model-group storage.""" def __init__(self, path: str) -> None: self._lock = threading.Lock() self._path = path conn = sqlite3.connect(self._path) try: + conn.execute("PRAGMA foreign_keys = ON") conn.execute("CREATE TABLE IF NOT EXISTS agent_pool (agent_id TEXT PRIMARY KEY, payload TEXT NOT NULL)") + conn.execute("CREATE TABLE IF NOT EXISTS model_group (group_name TEXT PRIMARY KEY)") + conn.execute( + "CREATE TABLE IF NOT EXISTS model_group_member (" + "agent_id TEXT PRIMARY KEY REFERENCES agent_pool(agent_id) ON DELETE CASCADE, " + "group_name TEXT NOT NULL REFERENCES model_group(group_name) ON DELETE CASCADE)" + ) + self._migrate_legacy_groups(conn) conn.commit() finally: conn.close() + @staticmethod + def _migrate_legacy_groups(conn: sqlite3.Connection) -> None: + """Move legacy payload group names into the normalized membership relation.""" + for agent_id, raw_payload in conn.execute("SELECT agent_id, payload FROM agent_pool"): + payload = json.loads(raw_payload) + group_name = payload.pop("group_name", "") + if not group_name: + continue + group_name = canonical_group_name(group_name) + conn.execute("INSERT OR IGNORE INTO model_group (group_name) VALUES (?)", (group_name,)) + conn.execute( + "INSERT OR REPLACE INTO model_group_member (agent_id, group_name) VALUES (?, ?)", + (agent_id, group_name), + ) + conn.execute( + "UPDATE agent_pool SET payload = ? WHERE agent_id = ?", + (json.dumps(payload, ensure_ascii=False), agent_id), + ) + def save(self, agent: "ModelAgent") -> None: with self._lock: conn = sqlite3.connect(self._path) try: + conn.execute("PRAGMA foreign_keys = ON") + payload = agent.to_config() + payload.pop("group_name", None) conn.execute( - "INSERT OR REPLACE INTO agent_pool (agent_id, payload) VALUES (?, ?)", - (agent.id, json.dumps(agent.to_config(), ensure_ascii=False)), + "INSERT INTO agent_pool (agent_id, payload) VALUES (?, ?) " + "ON CONFLICT(agent_id) DO UPDATE SET payload = excluded.payload", + (agent.id, json.dumps(payload, ensure_ascii=False)), + ) + conn.execute("DELETE FROM model_group_member WHERE agent_id = ?", (agent.id,)) + if agent.group_name: + conn.execute("INSERT OR IGNORE INTO model_group (group_name) VALUES (?)", (agent.group_name,)) + conn.execute( + "INSERT INTO model_group_member (agent_id, group_name) VALUES (?, ?)", + (agent.id, agent.group_name), + ) + conn.execute( + "DELETE FROM model_group WHERE NOT EXISTS (" + "SELECT 1 FROM model_group_member WHERE model_group_member.group_name = model_group.group_name)" ) conn.commit() finally: @@ -1617,10 +1658,19 @@ def load_all(self) -> list["ModelAgent"]: with self._lock: conn = sqlite3.connect(self._path) try: - rows = conn.execute("SELECT payload FROM agent_pool ORDER BY agent_id").fetchall() + rows = conn.execute( + "SELECT agent_pool.payload, model_group_member.group_name " + "FROM agent_pool LEFT JOIN model_group_member USING (agent_id) " + "ORDER BY agent_pool.agent_id" + ).fetchall() finally: conn.close() - return [ModelAgent.from_dict(json.loads(row[0])) for row in rows] + values = [] + for raw_payload, group_name in rows: + payload = json.loads(raw_payload) + payload["group_name"] = group_name or "" + values.append(ModelAgent.from_dict(payload)) + return values def close(self) -> None: """Compatibility no-op: agent-pool operations use short-lived sqlite handles.""" @@ -2432,6 +2482,11 @@ def get_model_group(self, group_name: str) -> dict[str, Any]: "group_name": name, "member_agent_ids": ranked_ids, "enabled_member_count": sum(1 for agent in members if not agent.disabled), + "capability_coverage": { + capability: sum(capability in agent.tags for agent in members) + for capability in sorted(MODEL_CAPABILITIES) + if any(capability in agent.tags for agent in members) + }, "members": [self._agent_to_admin_payload(self._agent(agent_id)) for agent_id in ranked_ids], } diff --git a/docs/planning/adrs/0026-model-group-cost-aware-discovery.md b/docs/planning/adrs/0026-model-group-cost-aware-discovery.md index 657d19c23..d8d48fd8f 100644 --- a/docs/planning/adrs/0026-model-group-cost-aware-discovery.md +++ b/docs/planning/adrs/0026-model-group-cost-aware-discovery.md @@ -35,7 +35,24 @@ sequenceDiagram ## Data, web, and operational boundaries -Group membership survives restart in the existing `agent_pool` database payload. Authenticated REST resources provide `GET/POST /api/v1/model_groups` and `GET/PATCH/DELETE /api/v1/model_groups/{group_name}`; deleting a group retains its provider agents. The existing worker-agent create/PATCH API also accepts `group_name`. Admin shows the group, posterior success probability, and EWMA latency instead of fabricated capacity/success figures. The observation ledger intentionally resets on process restart: persisting it without a measurement horizon would let stale provider incidents dominate current routing. Add a normalized, time-windowed observation table when multi-instance aggregation and an explicit retention/decay policy are specified. +Group membership survives restart in normalized `model_group` and `model_group_member` relations; the agent JSON payload no longer duplicates `group_name`. Startup migrates legacy payload membership transactionally without dropping agent configuration. Authenticated REST resources provide `GET/POST /api/v1/model_groups` and `GET/PATCH/DELETE /api/v1/model_groups/{group_name}`; deleting a group retains its provider agents. The existing worker-agent create/PATCH API also accepts `group_name`. Admin provides a keyboard/native-form editor backed by those same resources and shows capability coverage, posterior success probability, and EWMA latency instead of fabricated capacity/success figures. The observation ledger intentionally resets on process restart: persisting it without a measurement horizon would let stale provider incidents dominate current routing. Add a normalized, time-windowed observation table when multi-instance aggregation and an explicit retention/decay policy are specified. + +```mermaid +erDiagram + agent_pool ||--o| model_group_member : "belongs through" + model_group ||--o{ model_group_member : contains + agent_pool { + text agent_id PK + text payload + } + model_group { + text group_name PK + } + model_group_member { + text agent_id PK,FK + text group_name FK + } +``` ## Verification and gaps diff --git a/tests/test_admin_contract.py b/tests/test_admin_contract.py index efb5aa6c7..f96f2242c 100644 --- a/tests/test_admin_contract.py +++ b/tests/test_admin_contract.py @@ -51,6 +51,24 @@ def test_admin_surface_exists_for_enterprise_operations() -> None: assert 'id="registerAgent"' in ADMIN_HTML assert "registerAgent: document.querySelector" in ADMIN_HTML assert 'els.registerAgent.addEventListener("click", () => showView("integrations"))' in ADMIN_HTML + assert 'id="modelGroupForm"' in ADMIN_HTML + assert 'id="modelGroupName" required pattern=' in ADMIN_HTML + assert 'id="modelGroupMembers" multiple required' in ADMIN_HTML + assert 'role="status" aria-live="polite"' in ADMIN_HTML + assert 'fetch("/api/v1/model_groups")' in ADMIN_HTML + assert 'method: exists ? "PATCH" : "POST"' in ADMIN_HTML + assert 'method: "DELETE"' in ADMIN_HTML + for key in ( + "model_groups_title", + "group_name_label", + "group_members_label", + "save_group", + "delete_group", + "no_model_groups", + "group_saved", + "group_deleted", + ): + assert key in ADMIN_TRANSLATIONS["en"] and key in ADMIN_TRANSLATIONS["ko"] assert ADMIN_TRANSLATIONS["en"]["no_agents_configured"] == "No agents are configured yet." assert '|| `${t("no_agents_configured")}`' in ADMIN_HTML assert ADMIN_TRANSLATIONS["en"]["no_audit_events"] == "No audit events yet." diff --git a/tests/test_agent_pool_db.py b/tests/test_agent_pool_db.py index b64acfd57..7d9b6b803 100644 --- a/tests/test_agent_pool_db.py +++ b/tests/test_agent_pool_db.py @@ -9,6 +9,7 @@ import json import os +import sqlite3 from pathlib import Path import sys import tempfile @@ -52,12 +53,37 @@ def test_add_patch_remove_survive_restart() -> None: assert by_id["general_agent"].priority == 9 # patch restored over the seed assert by_id["coding_agent"].model == "gpt-5.5" assert {a.group_name for a in by_id.values()} == {"example_logical_model"} + with sqlite3.connect(db) as conn: + payloads = [json.loads(row[0]) for row in conn.execute("SELECT payload FROM agent_pool")] + assert all("group_name" not in payload for payload in payloads) + assert conn.execute("SELECT group_name FROM model_group").fetchall() == [ + ("example_logical_model",) + ] + assert set(conn.execute("SELECT agent_id FROM model_group_member").fetchall()) == { + ("general_agent",), + ("coding_agent",), + } second.remove_agent("default", "coding_agent") third = TaskOrchestrator(_seed(), agents_db=db) assert {a.id for a in third.agents} == {"general_agent"} # removal survived restart +def test_legacy_payload_group_is_migrated_without_data_loss() -> None: + with tempfile.TemporaryDirectory() as directory: + db = os.path.join(directory, "pool.db") + legacy = ModelAgent("legacy_agent", "legacy-model", group_name="legacy-group").to_config() + with sqlite3.connect(db) as conn: + conn.execute("CREATE TABLE agent_pool (agent_id TEXT PRIMARY KEY, payload TEXT NOT NULL)") + conn.execute("INSERT INTO agent_pool VALUES (?, ?)", ("legacy_agent", json.dumps(legacy))) + + restored = TaskOrchestrator([], agents_db=db) + + assert restored.candidates[0].group_name == "legacy_group" + with sqlite3.connect(db) as conn: + assert "group_name" not in json.loads(conn.execute("SELECT payload FROM agent_pool").fetchone()[0]) + + def test_seed_agent_removal_tombstones_across_restart() -> None: with tempfile.TemporaryDirectory() as directory: db = os.path.join(directory, "pool.db") @@ -159,6 +185,7 @@ def test_http_model_group_crud_uses_arbitrary_member_names() -> None: status, created = _call(base, "POST", token, {"group_name": "vendor-neutral-example", "member_agent_ids": ["general_agent", "coding_agent"]}) assert status == 201 and created["group_name"] == "vendor_neutral_example" assert set(created["member_agent_ids"]) == {"general_agent", "coding_agent"} + assert created["capability_coverage"] == {} status, duplicate = _call(base, "POST", token, {"group_name": "vendor-neutral-example", "member_agent_ids": ["coding_agent"]}) assert status == 409 and duplicate["error"]["code"] == "model_group_exists"