From 63f460bfcef0bc73bc82dbf20ac104fa945b8142 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Wed, 30 Sep 2026 05:13:08 -0700 Subject: [PATCH] fix(devin): score family axes sparsely with own-key targets --- src/adapters/devin/live-models.ts | 36 +++++++-- structure/providers-and-adapters.md | 2 + .../providers/devin-family-resolution.test.ts | 81 ++++++++++++++++++- 3 files changed, 110 insertions(+), 9 deletions(-) diff --git a/src/adapters/devin/live-models.ts b/src/adapters/devin/live-models.ts index 5b72bd56ce2..9d0eead0037 100644 --- a/src/adapters/devin/live-models.ts +++ b/src/adapters/devin/live-models.ts @@ -207,6 +207,22 @@ function effortIndex(rung: string | undefined): number { return rung === undefined ? -1 : FAMILY_EFFORT_LADDER.indexOf(rung); } +function axisMismatchCount( + axes: ModelCatalogEntry["familyAxes"], + targets: Record, + baseline: number, +): number { + let mismatches = baseline; + for (const [axis, value] of Object.entries(axes ?? {})) { + if (isEffortAxis(axis)) continue; + const target = Object.hasOwn(targets, axis) ? targets[axis]! : 0; + if (target !== 0) { + if (value.order === target) mismatches--; + } else if (value.order !== 0) mismatches++; + } + return mismatches; +} + /** * The member closest to `targets` on the non-effort axes, then the lowest rung * at or above `effort`, falling back to the highest rung below it. A missing @@ -227,11 +243,15 @@ function closestMember( effort: string | undefined, ): ModelCatalogEntry | undefined { const want = effortIndex(effort); + // The baseline counts only `targets`, so scan it once: recomputing it per member + // made a wide anchor (many exposed axes) quadratic in members x axes. + let mismatchBaseline = 0; + for (const order of Object.values(targets)) if (order !== 0) mismatchBaseline++; let best: ModelCatalogEntry | undefined; let bestScore: number[] | undefined; for (const member of members) { const axes = member.familyAxes ?? {}; - const mismatches = Object.entries(targets).filter(([axis, order]) => (axes[axis]?.order ?? 0) !== order).length; + const mismatches = axisMismatchCount(axes, targets, mismatchBaseline); const have = effortIndex(devinFamilyEffortOf(member)); const lowered = want > 0 && have < want ? 1 : 0; const distance = want < 0 ? 0 @@ -263,18 +283,18 @@ export function selectDevinFamilyMember( anchor?: ModelCatalogEntry, options: { includeDisabled?: boolean } = {}, ): ModelCatalogEntry | undefined { - const axisNames = new Set(members.flatMap((m) => Object.keys(m.familyAxes ?? {})).filter((a) => !isEffortAxis(a))); const base = anchor ?? members.find((m) => m.isFamilyDefault) - ?? closestMember(members, Object.fromEntries([...axisNames].map((a) => [a, 0])), "medium"); + ?? closestMember(members, {}, "medium"); if (!base) return undefined; - const targets: Record = {}; - for (const axis of axisNames) targets[axis] = base.familyAxes?.[axis]?.order ?? 0; - if (request.fast && axisNames.has(FAST_AXIS)) targets[FAST_AXIS] = 1; - if (request.longContext && axisNames.has(LONG_CONTEXT_AXIS)) targets[LONG_CONTEXT_AXIS] = 1; + const targets = Object.fromEntries(Object.entries(base.familyAxes ?? {}) + .filter(([axis]) => !isEffortAxis(axis)) + .map(([axis, value]) => [axis, value.order])); + if (request.fast) targets[FAST_AXIS] = 1; + if (request.longContext) targets[LONG_CONTEXT_AXIS] = 1; // `none` means no reasoning. Families like Claude Sonnet 4.6 express that as // Thinking off rather than as an effort rung. - if (request.effort === "none" && axisNames.has(THINKING_AXIS)) targets[THINKING_AXIS] = 0; + if (request.effort === "none") targets[THINKING_AXIS] = 0; const enabled = options.includeDisabled ? members : members.filter((m) => !m.disabled); return closestMember(enabled.length > 0 ? enabled : members, targets, request.effort ?? devinFamilyEffortOf(base)); } diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 8f26f84df8d..1c4d5eab96f 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -192,6 +192,8 @@ rewrite rules and the routed-id settlement. Devin pairs a late signature only with immediately preceding unsigned thinking in one assistant message; a call between them breaks the pair. While a signed attempt is held for an optional unsigned retry, a timer emits plain heartbeats even when the upstream stalls. The held queue is capped at 1,024 events or approximately 1 MiB of UTF-16 reasoning/signature payload; crossing either cap releases the events and disables that retry. Held usage frames merge per cumulative field, and the refused attempt's usage is added to the retry. A signed `invalid_argument` refusal is offered the unsigned retry before the history-overflow classifier sees any final refusal. If the send budget withholds that retry, the original refusal reaches the classifier and the recovery is recorded as withheld. +Devin family selection in `src/adapters/devin/live-models.ts` counts the anchor's nonzero targets once and visits each candidate's own axes. Missing axes have order zero; extra nonzero axes, including prototype names such as `toString`, count as mismatches. Effort precedence and family confinement remain unchanged; `tests/providers/devin-family-resolution.test.ts` covers the sparse-scoring contract. + Inline document admission shares one encoding predicate between its scanner and parser in `src/responses/inline-document.ts`: malformed base64 quantum/padding lengths are refused, and valid padded or unpadded payloads pass unchanged without a decoding allocation. diff --git a/tests/providers/devin-family-resolution.test.ts b/tests/providers/devin-family-resolution.test.ts index 53d3c3f1331..b78d1cb7f4d 100644 --- a/tests/providers/devin-family-resolution.test.ts +++ b/tests/providers/devin-family-resolution.test.ts @@ -10,7 +10,7 @@ */ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; import { resolveWireModelUidForTests } from "../../src/adapters/devin"; -import { fetchDevinUsableModels } from "../../src/adapters/devin/live-models"; +import { fetchDevinUsableModels, selectDevinFamilyMember } from "../../src/adapters/devin/live-models"; import { parseCatalogBuffer, setCachedCatalogForTests, type CacheEntry } from "../../src/adapters/devin/cloud-direct/catalog"; import { encodeMessage, encodeString, encodeVarintField } from "../../src/adapters/devin/cloud-direct/wire"; @@ -259,6 +259,85 @@ describe("family-based wire model resolution", () => { expect(await resolve(catalogOf(rows), "devin-test-think", "none")).toBe("think-off"); expect(await resolve(catalogOf(rows), "devin-test-think", "high")).toBe("think-high"); }); + + test("scores distinct family axes without a members-by-axes cross-product", () => { + // The anchor row carries the wide axis set: the mismatch baseline scans `targets`, + // which the anchor fills. With per-member baseline recomputation this case is + // members x axes (~144M iterations); hoisted, it is anchor axes + member axes. + const wideAxes = Object.fromEntries( + Array.from({ length: 12_000 }, (_, index) => [`Anchor ${index}`, { order: 1 }]), + ); + const members = [ + { + modelUid: "wide-anchor", + displayName: "wide-anchor", + familyUid: "wide", + familyAxes: wideAxes, + isFamilyDefault: true, + }, + ...Array.from({ length: 12_000 }, (_, index) => ({ + modelUid: `wide-${index}`, + displayName: `wide-${index}`, + familyUid: "wide", + familyAxes: { [`Axis ${index}`]: { order: 1 } }, + })), + ]; + const started = performance.now(); + expect(selectDevinFamilyMember(members, {})?.modelUid).toBe("wide-anchor"); + // Wall-clock bound, generous on purpose: the hoist makes this ~36k axis touches + // (ms even on a contended runner), while the per-member rescan needs ~144M — the + // bound only has to distinguish those two orders, not measure fast hardware. + expect(performance.now() - started).toBeLessThan(5_000); + }); + + test.each(["toString", "constructor", "__proto__"])("counts an extra own %s axis instead of inheriting a target", axis => { + const member = (modelUid: string, familyAxes: Record) => ({ + modelUid, displayName: modelUid, familyUid: "own-axes", familyAxes, + }); + const anchor = member("anchor", { A: { order: 1 } }); + const exact = member("exact", { A: { order: 1 } }); + const extra = member("extra", Object.fromEntries([["A", { order: 1 }], [axis, { order: 1 }]])); + expect(selectDevinFamilyMember([extra, exact], {}, anchor)?.modelUid).toBe("exact"); + const namedAnchor = member("named-anchor", Object.fromEntries([[axis, { order: 2 }]])); + const namedExact = member("named-exact", Object.fromEntries([[axis, { order: 2 }]])); + expect(selectDevinFamilyMember([member("missing", {}), namedExact], {}, namedAnchor)?.modelUid).toBe("named-exact"); + }); + + test("ranks an exact axis match ahead of missing and extra axes", () => { + const anchor = { + modelUid: "axis-anchor", + displayName: "axis-anchor", + familyUid: "wide", + familyAxes: { A: { order: 1 }, B: { order: 2 } }, + isFamilyDefault: true, + }; + const exact = { + modelUid: "axis-exact", + displayName: "axis-exact", + familyUid: "wide", + familyAxes: { A: { order: 1 }, B: { order: 2 } }, + }; + const missing = { + modelUid: "axis-missing", + displayName: "axis-missing", + familyUid: "wide", + familyAxes: { A: { order: 1 } }, + }; + const extra = { + modelUid: "axis-extra", + displayName: "axis-extra", + familyUid: "wide", + familyAxes: { A: { order: 1 }, B: { order: 2 }, C: { order: 1 }, D: { order: 3 } }, + }; + // The anchor itself always wins while listed: the ranking claim is about the + // members it cannot see, so exclude it and pass it explicitly. + const members = [missing, extra, exact]; + expect(selectDevinFamilyMember(members, {}, anchor)?.modelUid).toBe("axis-exact"); + // Without the exact row, a member missing one target axis scores baseline - 1, + // while a member matching every target but exposing two extra axes scores + // 0 + 2 — unequal mismatch counts, so the ordering is proven rather than a tie. + expect(selectDevinFamilyMember([missing, extra], {}, anchor)?.modelUid).toBe("axis-missing"); + }); }); describe("suffix fallback without family metadata never crosses families", () => {