Repository navigation
feat: normalize and edit model groups #838
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -21,6 +21,14 @@ | |
| "active_status": "Active", | ||
| "agent_pool_title": "Agent Pool", | ||
| "register_agent": "Register Agent", | ||
| "model_groups_title": "Model groups", | ||
| "group_name_label": "Group name", | ||
| "group_members_label": "Provider model members", | ||
| "save_group": "Save group", | ||
| "delete_group": "Delete group", | ||
| "no_model_groups": "No model groups yet. Create one to route a logical model across providers.", | ||
| "group_saved": "Group saved. Send requests with this group name to use measured routing.", | ||
| "group_deleted": "Group deleted. Its provider models remain available.", | ||
| "search_agents": "Search agents", | ||
| "all_statuses": "All statuses", | ||
| "no_agents_match": "No agents match the current filters.", | ||
|
|
@@ -255,6 +263,14 @@ | |
| "active_status": "활성", | ||
| "agent_pool_title": "에이전트 풀", | ||
| "register_agent": "에이전트 등록", | ||
| "model_groups_title": "모델 그룹", | ||
| "group_name_label": "그룹 이름", | ||
| "group_members_label": "공급자 모델 멤버", | ||
| "save_group": "그룹 저장", | ||
| "delete_group": "그룹 삭제", | ||
| "no_model_groups": "모델 그룹이 없습니다. 논리 모델을 여러 공급자로 라우팅하려면 그룹을 만드세요.", | ||
| "group_saved": "그룹을 저장했습니다. 측정 기반 라우팅에는 이 그룹 이름으로 요청하세요.", | ||
| "group_deleted": "그룹을 삭제했습니다. 공급자 모델은 그대로 사용할 수 있습니다.", | ||
| "search_agents": "에이전트 검색", | ||
| "all_statuses": "전체 상태", | ||
| "no_agents_match": "현재 필터와 일치하는 에이전트가 없습니다.", | ||
|
|
@@ -869,6 +885,18 @@ | |
| </thead> | ||
| <tbody id="agents"></tbody> | ||
| </table> | ||
| <form id="modelGroupForm" class="policy-list" aria-labelledby="modelGroupsTitle"> | ||
| <h2 id="modelGroupsTitle" data-i18n="model_groups_title">Model groups</h2> | ||
| <label><span data-i18n="group_name_label">Group name</span> | ||
| <input id="modelGroupName" required pattern="[A-Za-z0-9]+(?:[-_][A-Za-z0-9]+)+" autocomplete="off"> | ||
| </label> | ||
| <label><span data-i18n="group_members_label">Provider model members</span> | ||
| <select id="modelGroupMembers" multiple required size="5"></select> | ||
| </label> | ||
| <div class="actions"><button class="btn primary" type="submit" data-i18n="save_group">Save group</button></div> | ||
| <p id="modelGroupFeedback" role="status" aria-live="polite"></p> | ||
| <ul id="modelGroups" class="policy-list"></ul> | ||
| </form> | ||
| </section> | ||
| <section class="panel" id="orchestration-policy" tabindex="-1"> | ||
| <div class="panel-header"> | ||
|
|
@@ -1032,10 +1060,15 @@ | |
| viewAudit: document.querySelector("#viewAudit"), | ||
| agentSettings: document.querySelector("#agentSettings"), | ||
| registerAgent: document.querySelector("#registerAgent"), | ||
| modelGroupForm: document.querySelector("#modelGroupForm"), | ||
| modelGroupName: document.querySelector("#modelGroupName"), | ||
| modelGroupMembers: document.querySelector("#modelGroupMembers"), | ||
| modelGroupFeedback: document.querySelector("#modelGroupFeedback"), | ||
| modelGroups: document.querySelector("#modelGroups"), | ||
| mobileView: document.querySelector("#mobileView"), | ||
| language: document.querySelector("#language") | ||
| }; | ||
| let state = {agents: [], modelGroups: [], last: null, analytics: null, readiness: null, buyerHandoffBundle: null, saleabilityDecision: null, commercialEvidenceExport: null, commercialAcceptanceCheck: null, commercialReleaseCandidate: null, commercialGapRegister: null, commercialProcurementReadiness: null, commercialContractReadiness: null, commercialOnboardingReadiness: null, commercialOperationsReadiness: null, commercialSecurityAttestation: null, commercialValueReadiness: null, commercialCloseReadiness: null, commercialGoToMarketReadiness: null, commercialLaunchReadiness: null, commercialCompletionScorecard: null, commercialBuyerAcceptanceWorkflow: null, commercialDemoScenarios: null, commercialProposalPacket: null, commercialPurchaseApprovalPacket: null, commercialDueDiligenceRoom: null, commercialInvestmentCommitteeMemo: null}; | ||
| let currentLang = "en"; | ||
| let activeTraceTab = "timeline"; | ||
| const datasets = [ | ||
|
|
@@ -1077,6 +1110,41 @@ | |
| <td>${agent.group_routing ? `${(agent.group_routing.success_posterior_mean * 100).toFixed(1)}%` : "—"}</td> | ||
| </tr>`).join("") || `<tr><td colspan="6" class="empty" data-i18n="no_agents_match">${t("no_agents_match")}</td></tr>`; | ||
| } | ||
| function renderModelGroups() { | ||
| const selected = new Set(Array.from(els.modelGroupMembers.selectedOptions).map(option => option.value)); | ||
| els.modelGroupMembers.innerHTML = state.agents.map(agent => | ||
| `<option value="${escapeHtml(agent.id)}" ${selected.has(agent.id) ? "selected" : ""}>${escapeHtml(agent.id)} — ${escapeHtml(agent.model)} [${escapeHtml((agent.tags || []).join(", "))}]</option>` | ||
| ).join(""); | ||
| els.modelGroups.innerHTML = state.modelGroups.map(group => ` | ||
| <li><strong>${escapeHtml(group.group_name)}</strong> — ${escapeHtml(group.member_agent_ids.join(", "))}<br> | ||
| <small>${escapeHtml(Object.entries(group.capability_coverage || {}).map(([name, count]) => `${name}: ${count}`).join(" · "))}</small> | ||
| <button class="btn" type="button" data-delete-group="${escapeHtml(group.group_name)}" data-i18n="delete_group">${t("delete_group")}</button> | ||
| </li>`).join("") || `<li class="empty">${t("no_model_groups")}</li>`; | ||
| } | ||
|
|
||
| async function refreshModelGroups() { | ||
| const response = await fetch("/api/v1/model_groups"); | ||
| const payload = await response.json(); | ||
| if (!response.ok) throw new Error(payload.error?.message || "Could not load model groups"); | ||
| state.modelGroups = payload.items || []; | ||
| renderModelGroups(); | ||
| } | ||
|
|
||
| async function saveModelGroup(event) { | ||
| event.preventDefault(); | ||
| const groupName = els.modelGroupName.value.trim(); | ||
| const memberIds = Array.from(els.modelGroupMembers.selectedOptions).map(option => option.value); | ||
| const exists = state.modelGroups.some(group => group.group_name === groupName.replaceAll("-", "_").toLowerCase()); | ||
| const response = await fetch(exists ? `/api/v1/model_groups/${encodeURIComponent(groupName)}` : "/api/v1/model_groups", { | ||
| method: exists ? "PATCH" : "POST", | ||
| headers: {"content-type": "application/json"}, | ||
| body: JSON.stringify(exists ? {member_agent_ids: memberIds} : {group_name: groupName, member_agent_ids: memberIds}) | ||
| }); | ||
| const payload = await response.json(); | ||
| if (!response.ok) throw new Error(payload.error?.message || "Could not save model group"); | ||
| els.modelGroupFeedback.textContent = t("group_saved"); | ||
| await refreshModelGroups(); | ||
| } | ||
| function renderTrace(result) { | ||
| els.traceMode.textContent = result.mode; | ||
| const trace = result.trace || []; | ||
|
|
@@ -1471,11 +1539,13 @@ | |
| }); | ||
| localStorage.setItem("admin_lang", lang); | ||
| if (state.agents.length) renderAgents(); | ||
| if (state.agents.length) renderModelGroups(); | ||
| if (state.policy) renderSecondaryViews(); | ||
| } | ||
| async function load() { | ||
| const res = await fetch("/admin/state"); | ||
| state = await res.json(); | ||
| await refreshModelGroups(); | ||
|
Contributor
There was a problem hiding this comment. 📝 Info: load() aborts entirely if model_groups fetch fails
Was this helpful? React with 👍 or 👎 to provide feedback. |
||
| await refreshAnalytics(); | ||
| await refreshReadiness(); | ||
| els.hintCount.textContent = state.policy.complex_hints.length; | ||
|
|
@@ -1572,6 +1642,20 @@ | |
| els.viewAudit.addEventListener("click", () => showView("audit")); | ||
| els.agentSettings.addEventListener("click", () => showView("settings")); | ||
| els.registerAgent.addEventListener("click", () => showView("integrations")); | ||
| els.modelGroupForm.addEventListener("submit", event => saveModelGroup(event).catch(error => { | ||
| els.modelGroupFeedback.textContent = error.message; | ||
| })); | ||
| els.modelGroups.addEventListener("click", event => { | ||
| const name = event.target.dataset?.deleteGroup; | ||
| if (!name) return; | ||
| fetch(`/api/v1/model_groups/${encodeURIComponent(name)}`, {method: "DELETE"}) | ||
| .then(response => response.ok ? response.json() : Promise.reject(new Error("Could not delete model group"))) | ||
| .then(() => { | ||
| els.modelGroupFeedback.textContent = t("group_deleted"); | ||
| return refreshModelGroups(); | ||
| }) | ||
| .catch(error => { els.modelGroupFeedback.textContent = error.message; }); | ||
| }); | ||
| els.language.addEventListener("change", () => applyI18n(els.language.value)); | ||
| els.mobileView.addEventListener("change", () => showView(els.mobileView.value)); | ||
| document.querySelector("#copyJson").addEventListener("click", () => { | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -51,6 +51,9 @@ | |
| ProviderDestination = tuple[int, tuple[Any, ...]] | ||
| MAX_LOCAL_CONCURRENCY = 64 | ||
| DEFAULT_PROVIDER_PROBE_TIMEOUT = 5.0 | ||
| MODEL_CAPABILITIES = frozenset( | ||
| {"text", "image", "video", "speech", "transcription", "embedding", "rerank", "audio"} | ||
| ) | ||
| MAX_PROVIDER_PROBE_TIMEOUT = 30.0 | ||
| _SAFE_PROVIDER_PROBE_ERROR_TYPES = frozenset({ | ||
| "ConnectionError", | ||
|
|
@@ -1585,29 +1588,67 @@ def load_agents(path: str) -> list[ModelAgent]: # pragma: no cover | |
|
|
||
|
|
||
| class _AgentPoolStore: | ||
| """Durable, normalized agent and model-group storage.""" | ||
|
|
||
| def __init__(self, path: str) -> None: | ||
| self._lock = threading.Lock() | ||
| self._path = path | ||
| conn = sqlite3.connect(self._path) | ||
| try: | ||
| conn.execute("PRAGMA foreign_keys = ON") | ||
| conn.execute("CREATE TABLE IF NOT EXISTS agent_pool (agent_id TEXT PRIMARY KEY, payload TEXT NOT NULL)") | ||
| conn.execute("CREATE TABLE IF NOT EXISTS model_group (group_name TEXT PRIMARY KEY)") | ||
| conn.execute( | ||
| "CREATE TABLE IF NOT EXISTS model_group_member (" | ||
| "agent_id TEXT PRIMARY KEY REFERENCES agent_pool(agent_id) ON DELETE CASCADE, " | ||
| "group_name TEXT NOT NULL REFERENCES model_group(group_name) ON DELETE CASCADE)" | ||
| ) | ||
| self._migrate_legacy_groups(conn) | ||
| conn.commit() | ||
| finally: | ||
| conn.close() | ||
|
|
||
| @staticmethod | ||
| def _migrate_legacy_groups(conn: sqlite3.Connection) -> None: | ||
| """Move legacy payload group names into the normalized membership relation.""" | ||
| for agent_id, raw_payload in conn.execute("SELECT agent_id, payload FROM agent_pool"): | ||
| payload = json.loads(raw_payload) | ||
| group_name = payload.pop("group_name", "") | ||
| if not group_name: | ||
| continue | ||
| group_name = canonical_group_name(group_name) | ||
| conn.execute("INSERT OR IGNORE INTO model_group (group_name) VALUES (?)", (group_name,)) | ||
| conn.execute( | ||
| "INSERT OR REPLACE INTO model_group_member (agent_id, group_name) VALUES (?, ?)", | ||
| (agent_id, group_name), | ||
| ) | ||
| conn.execute( | ||
| "UPDATE agent_pool SET payload = ? WHERE agent_id = ?", | ||
| (json.dumps(payload, ensure_ascii=False), agent_id), | ||
| ) | ||
|
Comment on lines
+1614
to
+1628
Contributor
There was a problem hiding this comment. 📝 Info: Migration mutates agent_pool while scanning it
Was this helpful? React with 👍 or 👎 to provide feedback. |
||
|
|
||
| def save(self, agent: "ModelAgent") -> None: | ||
| with self._lock: | ||
| conn = sqlite3.connect(self._path) | ||
| try: | ||
| conn.execute("PRAGMA foreign_keys = ON") | ||
| payload = agent.to_config() | ||
| payload.pop("group_name", None) | ||
| conn.execute( | ||
| "INSERT INTO agent_pool (agent_id, payload) VALUES (?, ?) " | ||
| "ON CONFLICT(agent_id) DO UPDATE SET payload = excluded.payload", | ||
| (agent.id, json.dumps(payload, ensure_ascii=False)), | ||
| ) | ||
| conn.execute("DELETE FROM model_group_member WHERE agent_id = ?", (agent.id,)) | ||
| if agent.group_name: | ||
| conn.execute("INSERT OR IGNORE INTO model_group (group_name) VALUES (?)", (agent.group_name,)) | ||
| conn.execute( | ||
| "INSERT INTO model_group_member (agent_id, group_name) VALUES (?, ?)", | ||
| (agent.id, agent.group_name), | ||
| ) | ||
| conn.execute( | ||
| "DELETE FROM model_group WHERE NOT EXISTS (" | ||
| "SELECT 1 FROM model_group_member WHERE model_group_member.group_name = model_group.group_name)" | ||
| ) | ||
| conn.commit() | ||
| finally: | ||
|
|
@@ -1617,10 +1658,19 @@ def load_all(self) -> list["ModelAgent"]: | |
| with self._lock: | ||
| conn = sqlite3.connect(self._path) | ||
| try: | ||
| rows = conn.execute( | ||
| "SELECT agent_pool.payload, model_group_member.group_name " | ||
| "FROM agent_pool LEFT JOIN model_group_member USING (agent_id) " | ||
| "ORDER BY agent_pool.agent_id" | ||
| ).fetchall() | ||
| finally: | ||
| conn.close() | ||
| values = [] | ||
| for raw_payload, group_name in rows: | ||
| payload = json.loads(raw_payload) | ||
| payload["group_name"] = group_name or "" | ||
| values.append(ModelAgent.from_dict(payload)) | ||
| return values | ||
|
|
||
| def close(self) -> None: | ||
| """Compatibility no-op: agent-pool operations use short-lived sqlite handles.""" | ||
|
|
@@ -2432,6 +2482,11 @@ def get_model_group(self, group_name: str) -> dict[str, Any]: | |
| "group_name": name, | ||
| "member_agent_ids": ranked_ids, | ||
| "enabled_member_count": sum(1 for agent in members if not agent.disabled), | ||
| "capability_coverage": { | ||
| capability: sum(capability in agent.tags for agent in members) | ||
| for capability in sorted(MODEL_CAPABILITIES) | ||
| if any(capability in agent.tags for agent in members) | ||
| }, | ||
|
Comment on lines
+2485
to
+2489
Contributor
There was a problem hiding this comment. 📝 Info: capability_coverage relies on discovery tags
Was this helpful? React with 👍 or 👎 to provide feedback. |
||
| "members": [self._agent_to_admin_payload(self._agent(agent_id)) for agent_id in ranked_ids], | ||
| } | ||
|
|
||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
📝 Info: Client group-name canonicalization matches server
saveModelGrouppicks POST vs PATCH by comparinggroupName.replaceAll("-","_").toLowerCase()to stored canonical names (admin.py). Because the inputpatternforbids whitespace and consecutive separators, this matches the server'scanonical_group_name(contextual_orchestrator/model_group.py:53-66) for every pattern-valid input, so the exists check stays reliable.Was this helpful? React with 👍 or 👎 to provide feedback.