Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 104 additions & 0 deletions src/lib/inference/serving/resolver.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -499,6 +499,110 @@ describe("managed inference resolver", () => {
).toMatchObject({ outcome: "no-match", code: "requirements-not-met" });
});

it("selects a preset only when readiness observation comparisons match (#8246)", () => {
const catalog = hostLocalFixtureCatalog();
const preset = catalog.presets[0]!;
const comparedPreset = {
...preset,
spec: {
...preset.spec,
requirements: {
all: [
{
readiness: {
scope: "everyNode",
kind: "observation",
id: "host.os.platform",
comparison: { operator: "equals", value: "linux" },
},
},
{
readiness: {
scope: "everyNode",
kind: "observation",
id: "host.os.architecture",
comparison: { operator: "one-of", values: ["arm64", "amd64"] },
},
},
{
readiness: {
scope: "everyNode",
kind: "observation",
id: "host.gpu.count",
comparison: { operator: "at-least", value: 1 },
},
},
{
readiness: {
scope: "everyNode",
kind: "observation",
id: "host.gpu.driver_version",
comparison: { operator: "version-at-least", value: "580.65.6" },
},
},
],
},
},
} as ManagedInferenceServingPreset;
const comparedCatalog: CompiledManagedInferenceCatalog = {
...catalog,
presets: [comparedPreset],
};
const reports = readinessSources().map(({ nodeId, report }) => ({
nodeId,
report: readinessReport({
...report,
observations: [
{ id: "host.os.platform", state: "present", value: "linux" },
{ id: "host.os.architecture", state: "present", value: "arm64" },
{ id: "host.gpu.count", state: "present", value: 1 },
{ id: "host.gpu.driver_version", state: "present", value: "580.65.06" },
],
}),
}));

expect(
resolveManagedInferenceServing(
resolverInput({
readinessReports: reports,
topologyQualifications: [],
intent: { preset: preset.metadata.id },
}),
comparedCatalog,
),
).toMatchObject({ outcome: "selected" });

const nonmatchingObservations = [
["equals", "host.os.platform", "windows"],
["one-of", "host.os.architecture", "riscv64"],
["at-least", "host.gpu.count", 0],
["version-at-least", "host.gpu.driver_version", "579.99.0"],
["malformed version-at-least", "host.gpu.driver_version", "580.65.x"],
] as const;
for (const [caseName, id, value] of nonmatchingObservations) {
const rejectedReports = reports.map(({ nodeId, report }, index) => ({
nodeId,
report: readinessReport({
...report,
observations: report.observations.map((observation) =>
index === 1 && observation.id === id ? { ...observation, value } : observation,
),
}),
}));
expect(
resolveManagedInferenceServing(
resolverInput({
readinessReports: rejectedReports,
topologyQualifications: [],
intent: { preset: preset.metadata.id },
}),
comparedCatalog,
),
`${caseName} must reject a nonmatching observation`,
).toMatchObject({ outcome: "rejected", code: "requirements-not-met" });
}
});

it("applies any-node readiness requirements as an existential match", () => {
const catalog = shippedCatalog();
const preset = shippedPreset(catalog);
Expand Down
48 changes: 46 additions & 2 deletions src/lib/inference/serving/resolver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,16 @@ import type {
ManagedInferenceReadinessRequirement,
ManagedInferenceReadinessSource,
ManagedInferenceResolution,
ManagedInferenceRuntimeServingRecipe,
ManagedInferenceResolverInput,
ManagedInferenceRuntimeServingRecipe,
ManagedInferenceSelectionIntent,
ManagedInferenceServingPreset,
ManagedInferenceServingRecipe,
ManagedInferenceTopologyQualification,
ManagedInferenceTopologyRequirement,
ResolvedHostLocalInferenceSelection,
ResolvedManagedInferenceSelection,
ServingReadinessComparison,
} from "./types.js";

export const MANAGED_INFERENCE_READINESS_MAX_AGE_MS = 30_000;
Expand Down Expand Up @@ -162,6 +163,42 @@ function readinessScopeMatches(
return false;
}

function compareNumericDottedVersions(left: string, right: string): number | undefined {
const parse = (value: string): number[] | undefined => {
if (!/^\d+(?:\.\d+)+$/u.test(value)) return undefined;
const parts = value.split(".").map(Number);
return parts.every(Number.isSafeInteger) ? parts : undefined;
};
const leftParts = parse(left);
const rightParts = parse(right);
if (!leftParts || !rightParts) return undefined;
const length = Math.max(leftParts.length, rightParts.length);
for (let index = 0; index < length; index += 1) {
const difference = (leftParts[index] ?? 0) - (rightParts[index] ?? 0);
if (difference !== 0) return difference < 0 ? -1 : 1;
}
return 0;
}

function readinessComparisonMatches(
actual: unknown,
comparison: ServingReadinessComparison,
): boolean {
switch (comparison.operator) {
case "equals":
return scalarEquals(actual, comparison.value);
case "one-of":
return comparison.values.some((candidate) => scalarEquals(actual, candidate));
case "at-least":
return typeof actual === "number" && actual >= comparison.value;
case "version-at-least": {
if (typeof actual !== "string") return false;
const order = compareNumericDottedVersions(actual, comparison.value);
return order !== undefined && order >= 0;
}
}
}

function readinessRequirementMatches(
requirement: ManagedInferenceReadinessRequirement["readiness"],
reports: readonly ManagedInferenceReadinessSource[],
Expand All @@ -171,7 +208,14 @@ function readinessRequirementMatches(
const matches = report.qualifications.filter(({ id }) => id === requirement.id);
return matches.length === 1 && matches[0]!.status === requirement.status;
}
if ("comparison" in requirement) return false;
if ("comparison" in requirement) {
const matches = report.observations.filter(({ id }) => id === requirement.id);
return (
matches.length === 1 &&
matches[0]!.state === "present" &&
readinessComparisonMatches(matches[0]!.value, requirement.comparison)
);
}
const collection =
requirement.kind === "observation" ? report.observations : report.capabilities;
const matches = collection.filter(({ id }) => id === requirement.id);
Expand Down
16 changes: 16 additions & 0 deletions src/lib/inference/vllm-models.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,22 @@ import {
} from "./vllm-models";

describe("vllm model registry", () => {
it("starts directly with setup when the serving environment is empty (#8246)", () => {
const command = buildVllmServeCommand({
id: "test/model",
label: "Test model",
envValue: "test-model",
downloadSizeBytes: 1,
maxModelLen: 4096,
modelArgs: [],
gated: false,
platforms: ["spark"],
serveEnv: {},
});

expect(command).toMatch(/^pip install vllm\[fastsafetensors\] && vllm serve/u);
});

it("records a finite positive Hugging Face file size for every model", () => {
for (const model of VLLM_MODELS) {
expect(Number.isFinite(model.downloadSizeBytes)).toBe(true);
Expand Down
21 changes: 11 additions & 10 deletions src/lib/inference/vllm-models.ts
Original file line number Diff line number Diff line change
Expand Up @@ -675,16 +675,17 @@ export function buildVllmServeCommand(
model: VllmModelDef,
env: NodeJS.ProcessEnv = process.env,
): string {
const envPrefix = model.serveEnv
? `${Object.entries(model.serveEnv)
.map(([key, value]) => {
if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key)) {
throw new Error(`Invalid vLLM serving environment variable name: ${key}`);
}
return `export ${key}=${shellQuote(value)}`;
})
.join(" && ")} && `
: "";
const envPrefix =
model.serveEnv && Object.keys(model.serveEnv).length > 0
? `${Object.entries(model.serveEnv)
.map(([key, value]) => {
if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key)) {
throw new Error(`Invalid vLLM serving environment variable name: ${key}`);
}
return `export ${key}=${shellQuote(value)}`;
})
.join(" && ")} && `
: "";
const args = [
...SHARED_VLLM_ARGS,
"--max-model-len",
Expand Down
Loading