From bcedc5b642f254cac21be570b7e64db1182d3a14 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 5 Aug 2026 12:18:43 -0700 Subject: [PATCH 1/2] fix(inference): evaluate serving preset readiness comparisons Signed-off-by: Prekshi Vyas --- src/lib/inference/serving/resolver.test.ts | 96 ++++++++++++++++++++++ src/lib/inference/serving/resolver.ts | 48 ++++++++++- src/lib/inference/vllm-models.test.ts | 16 ++++ src/lib/inference/vllm-models.ts | 21 ++--- 4 files changed, 169 insertions(+), 12 deletions(-) diff --git a/src/lib/inference/serving/resolver.test.ts b/src/lib/inference/serving/resolver.test.ts index e72308836f8..5ba76ca95bb 100644 --- a/src/lib/inference/serving/resolver.test.ts +++ b/src/lib/inference/serving/resolver.test.ts @@ -499,6 +499,102 @@ describe("managed inference resolver", () => { ).toMatchObject({ outcome: "no-match", code: "requirements-not-met" }); }); + it("matches typed readiness observation comparisons (#8246)", () => { + const catalog = hostLocalFixtureCatalog(); + const preset = catalog.presets[0]!; + const comparedPreset = { + ...preset, + spec: { + ...preset.spec, + requirements: { + all: [ + { + readiness: { + scope: "everyNode", + kind: "observation", + id: "host.os.platform", + comparison: { operator: "equals", value: "linux" }, + }, + }, + { + readiness: { + scope: "everyNode", + kind: "observation", + id: "host.os.architecture", + comparison: { operator: "one-of", values: ["arm64", "amd64"] }, + }, + }, + { + readiness: { + scope: "everyNode", + kind: "observation", + id: "host.gpu.count", + comparison: { operator: "at-least", value: 1 }, + }, + }, + { + readiness: { + scope: "everyNode", + kind: "observation", + id: "host.gpu.driver_version", + comparison: { operator: "version-at-least", value: "580.65.6" }, + }, + }, + ], + }, + }, + } as ManagedInferenceServingPreset; + const comparedCatalog: CompiledManagedInferenceCatalog = { + ...catalog, + presets: [comparedPreset], + }; + const reports = readinessSources().map(({ nodeId, report }) => ({ + nodeId, + report: readinessReport({ + ...report, + observations: [ + { id: "host.os.platform", state: "present", value: "linux" }, + { id: "host.os.architecture", state: "present", value: "arm64" }, + { id: "host.gpu.count", state: "present", value: 1 }, + { id: "host.gpu.driver_version", state: "present", value: "580.65.06" }, + ], + }), + })); + + expect( + resolveManagedInferenceServing( + resolverInput({ + readinessReports: reports, + topologyQualifications: [], + intent: { preset: preset.metadata.id }, + }), + comparedCatalog, + ), + ).toMatchObject({ outcome: "selected" }); + + reports[1] = { + nodeId: reports[1]!.nodeId, + report: readinessReport({ + ...reports[1]!.report, + observations: reports[1]!.report.observations.map((observation) => + observation.id === "host.gpu.driver_version" + ? { ...observation, value: "579.99.0" } + : observation, + ), + }), + }; + expect( + resolveManagedInferenceServing( + resolverInput({ + readinessReports: reports, + topologyQualifications: [], + intent: { preset: preset.metadata.id }, + }), + comparedCatalog, + ), + ).toMatchObject({ outcome: "rejected", code: "requirements-not-met" }); + }); + it("applies any-node readiness requirements as an existential match", () => { const catalog = shippedCatalog(); const preset = shippedPreset(catalog); diff --git a/src/lib/inference/serving/resolver.ts b/src/lib/inference/serving/resolver.ts index 46b09a8647d..9e7ac9e3e38 100644 --- a/src/lib/inference/serving/resolver.ts +++ b/src/lib/inference/serving/resolver.ts @@ -19,8 +19,8 @@ import type { ManagedInferenceReadinessRequirement, ManagedInferenceReadinessSource, ManagedInferenceResolution, - ManagedInferenceRuntimeServingRecipe, ManagedInferenceResolverInput, + ManagedInferenceRuntimeServingRecipe, ManagedInferenceSelectionIntent, ManagedInferenceServingPreset, ManagedInferenceServingRecipe, @@ -28,6 +28,7 @@ import type { ManagedInferenceTopologyRequirement, ResolvedHostLocalInferenceSelection, ResolvedManagedInferenceSelection, + ServingReadinessComparison, } from "./types.js"; export const MANAGED_INFERENCE_READINESS_MAX_AGE_MS = 30_000; @@ -162,6 +163,42 @@ function readinessScopeMatches( return false; } +function compareNumericDottedVersions(left: string, right: string): number | undefined { + const parse = (value: string): number[] | undefined => { + if (!/^\d+(?:\.\d+)+$/u.test(value)) return undefined; + const parts = value.split(".").map(Number); + return parts.every(Number.isSafeInteger) ? parts : undefined; + }; + const leftParts = parse(left); + const rightParts = parse(right); + if (!leftParts || !rightParts) return undefined; + const length = Math.max(leftParts.length, rightParts.length); + for (let index = 0; index < length; index += 1) { + const difference = (leftParts[index] ?? 0) - (rightParts[index] ?? 0); + if (difference !== 0) return difference < 0 ? -1 : 1; + } + return 0; +} + +function readinessComparisonMatches( + actual: unknown, + comparison: ServingReadinessComparison, +): boolean { + switch (comparison.operator) { + case "equals": + return scalarEquals(actual, comparison.value); + case "one-of": + return comparison.values.some((candidate) => scalarEquals(actual, candidate)); + case "at-least": + return typeof actual === "number" && actual >= comparison.value; + case "version-at-least": { + if (typeof actual !== "string") return false; + const order = compareNumericDottedVersions(actual, comparison.value); + return order !== undefined && order >= 0; + } + } +} + function readinessRequirementMatches( requirement: ManagedInferenceReadinessRequirement["readiness"], reports: readonly ManagedInferenceReadinessSource[], @@ -171,7 +208,14 @@ function readinessRequirementMatches( const matches = report.qualifications.filter(({ id }) => id === requirement.id); return matches.length === 1 && matches[0]!.status === requirement.status; } - if ("comparison" in requirement) return false; + if ("comparison" in requirement) { + const matches = report.observations.filter(({ id }) => id === requirement.id); + return ( + matches.length === 1 && + matches[0]!.state === "present" && + readinessComparisonMatches(matches[0]!.value, requirement.comparison) + ); + } const collection = requirement.kind === "observation" ? report.observations : report.capabilities; const matches = collection.filter(({ id }) => id === requirement.id); diff --git a/src/lib/inference/vllm-models.test.ts b/src/lib/inference/vllm-models.test.ts index 9a9869f6997..c557aa574d4 100644 --- a/src/lib/inference/vllm-models.test.ts +++ b/src/lib/inference/vllm-models.test.ts @@ -17,6 +17,22 @@ import { } from "./vllm-models"; describe("vllm model registry", () => { + it("starts directly with setup when the serving environment is empty (#8246)", () => { + const command = buildVllmServeCommand({ + id: "test/model", + label: "Test model", + envValue: "test-model", + downloadSizeBytes: 1, + maxModelLen: 4096, + modelArgs: [], + gated: false, + platforms: ["spark"], + serveEnv: {}, + }); + + expect(command).toMatch(/^pip install vllm\[fastsafetensors\] && vllm serve/u); + }); + it("records a finite positive Hugging Face file size for every model", () => { for (const model of VLLM_MODELS) { expect(Number.isFinite(model.downloadSizeBytes)).toBe(true); diff --git a/src/lib/inference/vllm-models.ts b/src/lib/inference/vllm-models.ts index 459329a4e66..b63605a62f1 100644 --- a/src/lib/inference/vllm-models.ts +++ b/src/lib/inference/vllm-models.ts @@ -675,16 +675,17 @@ export function buildVllmServeCommand( model: VllmModelDef, env: NodeJS.ProcessEnv = process.env, ): string { - const envPrefix = model.serveEnv - ? `${Object.entries(model.serveEnv) - .map(([key, value]) => { - if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key)) { - throw new Error(`Invalid vLLM serving environment variable name: ${key}`); - } - return `export ${key}=${shellQuote(value)}`; - }) - .join(" && ")} && ` - : ""; + const envPrefix = + model.serveEnv && Object.keys(model.serveEnv).length > 0 + ? `${Object.entries(model.serveEnv) + .map(([key, value]) => { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key)) { + throw new Error(`Invalid vLLM serving environment variable name: ${key}`); + } + return `export ${key}=${shellQuote(value)}`; + }) + .join(" && ")} && ` + : ""; const args = [ ...SHARED_VLLM_ARGS, "--max-model-len", From d6491799cd9815eab4f6f6ca02e47bf6300ce9b8 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 5 Aug 2026 13:35:05 -0700 Subject: [PATCH 2/2] test(inference): reject unmatched readiness comparisons Signed-off-by: Prekshi Vyas --- src/lib/inference/serving/resolver.test.ts | 50 +++++++++++++--------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/src/lib/inference/serving/resolver.test.ts b/src/lib/inference/serving/resolver.test.ts index 5ba76ca95bb..1e566e285b7 100644 --- a/src/lib/inference/serving/resolver.test.ts +++ b/src/lib/inference/serving/resolver.test.ts @@ -499,7 +499,7 @@ describe("managed inference resolver", () => { ).toMatchObject({ outcome: "no-match", code: "requirements-not-met" }); }); - it("matches typed readiness observation comparisons (#8246)", () => { + it("selects a preset only when readiness observation comparisons match (#8246)", () => { const catalog = hostLocalFixtureCatalog(); const preset = catalog.presets[0]!; const comparedPreset = { @@ -572,27 +572,35 @@ describe("managed inference resolver", () => { ), ).toMatchObject({ outcome: "selected" }); - reports[1] = { - nodeId: reports[1]!.nodeId, - report: readinessReport({ - ...reports[1]!.report, - observations: reports[1]!.report.observations.map((observation) => - observation.id === "host.gpu.driver_version" - ? { ...observation, value: "579.99.0" } - : observation, - ), - }), - }; - expect( - resolveManagedInferenceServing( - resolverInput({ - readinessReports: reports, - topologyQualifications: [], - intent: { preset: preset.metadata.id }, + const nonmatchingObservations = [ + ["equals", "host.os.platform", "windows"], + ["one-of", "host.os.architecture", "riscv64"], + ["at-least", "host.gpu.count", 0], + ["version-at-least", "host.gpu.driver_version", "579.99.0"], + ["malformed version-at-least", "host.gpu.driver_version", "580.65.x"], + ] as const; + for (const [caseName, id, value] of nonmatchingObservations) { + const rejectedReports = reports.map(({ nodeId, report }, index) => ({ + nodeId, + report: readinessReport({ + ...report, + observations: report.observations.map((observation) => + index === 1 && observation.id === id ? { ...observation, value } : observation, + ), }), - comparedCatalog, - ), - ).toMatchObject({ outcome: "rejected", code: "requirements-not-met" }); + })); + expect( + resolveManagedInferenceServing( + resolverInput({ + readinessReports: rejectedReports, + topologyQualifications: [], + intent: { preset: preset.metadata.id }, + }), + comparedCatalog, + ), + `${caseName} must reject a nonmatching observation`, + ).toMatchObject({ outcome: "rejected", code: "requirements-not-met" }); + } }); it("applies any-node readiness requirements as an existential match", () => {