QVAC-17481 feat[api]: integrate @qvac/classification-ggml into SDK

packages/sdk/client/api/classify.ts

@@ -0,0 +1,53 @@
+import {
+  classifyResponseSchema,
+  type ClassifyRequest,
+  type ClassifyClientParams,
+  type ClassificationResult,
+} from "@/schemas";
+import { stream as streamRpc } from "@/client/rpc/rpc-client";
+import { encodeBase64 } from "@/utils/encoding";
+
+/**
+ * Classifies an image using a loaded classification model.
+ *
+ * The bundled MobileNetV3-Small model produces 3 labels: `"food"`, `"report"`, `"other"`.
+ * Custom models may emit different labels sourced from the GGUF metadata.
+ *
+ * @param params.modelId - The identifier of the loaded classification model
+ * @param params.image - JPEG or PNG buffer; raw RGB bytes also accepted with `width`, `height`, `channels`
+ * @param params.topK - Limit results to top-K classes (default: all)
+ * @returns Sorted classification results, highest confidence first
+ *
+ * @example
+ * ```typescript
+ * const modelId = await loadModel({ modelType: "ggml-classification" });
+ * const jpeg = fs.readFileSync("photo.jpg");
+ * const results = await classify({ modelId, image: jpeg });
+ * // [ { label: "food", confidence: 0.93 }, { label: "other", confidence: 0.05 }, ... ]
+ * await unloadModel({ modelId });
+ * ```
+ */
+export async function classify(
+  params: ClassifyClientParams,
+): Promise<ClassificationResult[]> {
+  const request: ClassifyRequest = {
+    type: "classify",
+    modelId: params.modelId,
+    image: encodeBase64(params.image),
+    ...(params.topK !== undefined && { topK: params.topK }),
+    ...(params.width !== undefined && { width: params.width }),
+    ...(params.height !== undefined && { height: params.height }),
+    ...(params.channels !== undefined && { channels: params.channels }),
+  };
+
+  for await (const response of streamRpc(request)) {
+    if (response && typeof response === "object" && "type" in response && response.type === "classify") {
+      const parsed = classifyResponseSchema.parse(response);
+      if (parsed.done) {
+        return parsed.results;
+      }
+    }
+  }
+
+  return [];
+}

packages/sdk/client/api/index.ts

@@ -29,6 +29,7 @@ export { getLoadedModelInfo } from "./get-loaded-model-info";
 export { ocr } from "./ocr";
 export { invokePlugin, invokePluginStream } from "./invoke-plugin";
 export { diffusion, type DiffusionProgressTick } from "./diffusion";
+export { classify } from "./classify";
 export { video, type VideoProgressTick } from "./video";
 export { upscale } from "./upscale";
 export {

packages/sdk/e2e/tests/classification-tests.ts

@@ -0,0 +1,172 @@
+import type { TestDefinition } from "@tetherto/qvac-test-suite";
+
+// The bundled MobileNetV3-Small classifier returns 3 classes
+// ("food", "report", "other") with softmax probabilities. These tests
+// exercise the SDK's `classify()` client function end-to-end against
+// the bundled weights. Numerical correctness lives in the addon's own
+// integration suite — here we focus on the SDK shape contract.
+
+const createClassificationTest = (
+  testId: string,
+  params: Record<string, unknown>,
+  expectation:
+    | { validation: "type"; expectedType: "string" | "number" | "array" }
+    | {
+        validation: "function";
+        fn: (result: unknown) => { passed: boolean; output?: string };
+      },
+  estimatedDurationMs: number = 60000,
+  suites?: string[],
+): TestDefinition => ({
+  testId,
+  params,
+  expectation,
+  ...(suites && { suites }),
+  metadata: {
+    category: "classification",
+    dependency: "classification",
+    estimatedDurationMs,
+  },
+});
+
+// Result shape: an array of `{label, confidence}` objects sorted by
+// descending confidence. With the bundled MobileNetV3-Small all three
+// canonical labels must appear and every confidence must be in [0, 1].
+export const classificationResultsShape = createClassificationTest(
+  "classification-results-shape",
+  {},
+  {
+    validation: "function",
+    fn: (result: unknown) => {
+      const r = result as {
+        results?: { label?: string; confidence?: number }[];
+      };
+      if (!Array.isArray(r.results)) {
+        return { passed: false, output: "results is not an array" };
+      }
+      if (r.results.length === 0) {
+        return { passed: false, output: "results array is empty" };
+      }
+      for (const item of r.results) {
+        if (typeof item.label !== "string" || item.label.length === 0) {
+          return {
+            passed: false,
+            output: `result item missing string label (got: ${JSON.stringify(item)})`,
+          };
+        }
+        if (
+          typeof item.confidence !== "number" ||
+          item.confidence < 0 ||
+          item.confidence > 1
+        ) {
+          return {
+            passed: false,
+            output: `confidence out of [0,1] for label '${item.label}' (got ${item.confidence})`,
+          };
+        }
+      }
+      // Descending-confidence ordering invariant.
+      for (let i = 1; i < r.results.length; i++) {
+        const prev = r.results[i - 1]?.confidence ?? 0;
+        const cur = r.results[i]?.confidence ?? 0;
+        if (cur > prev) {
+          return {
+            passed: false,
+            output: `results not sorted by descending confidence at index ${i}`,
+          };
+        }
+      }
+      return { passed: true };
+    },
+  },
+  60000,
+  ["smoke"],
+);
+
+// Softmax invariant: probabilities sum to approximately 1 when topK is not
+// applied. Allow 1e-3 slack for FP16 / accumulator noise.
+export const classificationConfidenceSum = createClassificationTest(
+  "classification-confidence-sum",
+  {},
+  {
+    validation: "function",
+    fn: (result: unknown) => {
+      const r = result as {
+        results?: { confidence?: number }[];
+      };
+      if (!Array.isArray(r.results) || r.results.length === 0) {
+        return { passed: false, output: "results missing or empty" };
+      }
+      const sum = r.results.reduce((acc, x) => acc + (x.confidence ?? 0), 0);
+      if (Math.abs(sum - 1) > 1e-3) {
+        return {
+          passed: false,
+          output: `confidence sum ${sum} not within 1e-3 of 1`,
+        };
+      }
+      return { passed: true };
+    },
+  },
+);
+
+// `topK: 1` must truncate to exactly one result.
+export const classificationTopK = createClassificationTest(
+  "classification-topk",
+  { topK: 1 },
+  {
+    validation: "function",
+    fn: (result: unknown) => {
+      const r = result as { results?: unknown[] };
+      if (!Array.isArray(r.results)) {
+        return { passed: false, output: "results is not an array" };
+      }
+      if (r.results.length !== 1) {
+        return {
+          passed: false,
+          output: `topK:1 expected 1 result, got ${r.results.length}`,
+        };
+      }
+      return { passed: true };
+    },
+  },
+  60000,
+  ["smoke"],
+);
+
+// An invalid image buffer (too small to decode as JPEG/PNG) must reject
+// cleanly, and the model must remain usable for a follow-up valid call —
+// proves the addon does not wedge on the rejection path.
+export const classificationInvalidImage = createClassificationTest(
+  "classification-invalid-image",
+  { inputs: "invalid" },
+  {
+    validation: "function",
+    fn: (result: unknown) => {
+      const r = result as {
+        rejected?: boolean;
+        recoveryRan?: boolean;
+        errorMsg?: string;
+      };
+      if (!r.rejected) {
+        return {
+          passed: false,
+          output: "expected classify() to reject on invalid image bytes",
+        };
+      }
+      if (!r.recoveryRan) {
+        return {
+          passed: false,
+          output: "follow-up classify() did not succeed after rejection",
+        };
+      }
+      return { passed: true };
+    },
+  },
+);
+
+export const classificationTests: TestDefinition[] = [
+  classificationResultsShape,
+  classificationConfidenceSum,
+  classificationTopK,
+  classificationInvalidImage,
+];

packages/sdk/e2e/tests/desktop/consumer.ts

@@ -55,6 +55,7 @@ import { TranscribeStreamEventsExecutor } from "./executors/transcribe-stream-ev
 import { RagExecutor } from "./executors/rag-executor.js";
 import { OcrExecutor } from "./executors/ocr-executor.js";
 import { VlaExecutor } from "./executors/vla-executor.js";
+import { ClassificationExecutor } from "./executors/classification-executor.js";
 import { ConfigReloadExecutor } from "./executors/config-reload-executor.js";
 import { DesktopLoggingExecutor } from "./executors/logging-executor.js";
 import { RegistryExecutor } from "../shared/executors/registry-executor.js";
@@ -156,6 +157,12 @@ resources.define("vla", {
   config: { backend: "cpu" },
 });
 
+// Classification ships bundled weights inside @qvac/classification-ggml,
+// so no registry constant / pre-download is required.
+resources.define("classification", {
+  type: "classification",
+});
+
 resources.define("sharded-embeddings", {
   constant: GTE_LARGE_335M_FP16_SHARD,
   type: "embeddings",
@@ -425,6 +432,7 @@ export const executor = createExecutor({
     new ShardedModelExecutor(resources),
     new OcrExecutor(resources),
     new VlaExecutor(resources),
+    new ClassificationExecutor(resources),
     new TtsExecutor(resources),
     new ConfigReloadExecutor(resources),
     new DesktopLoggingExecutor(resources),

packages/sdk/e2e/tests/desktop/executors/classification-executor.ts

@@ -0,0 +1,109 @@
+import fs from "node:fs";
+import path from "node:path";
+import { classify } from "@qvac/sdk";
+import {
+  ValidationHelpers,
+  type TestResult,
+  type Expectation,
+} from "@tetherto/qvac-test-suite";
+import { AbstractModelExecutor } from "../../shared/executors/abstract-model-executor.js";
+import { classificationTests } from "../../classification-tests.js";
+
+interface ClassificationParams {
+  topK?: number;
+  inputs?: "invalid";
+}
+
+const SAMPLE_IMAGE_PATH = path.resolve(
+  process.cwd(),
+  "assets/images/elephant.jpg",
+);
+
+export class ClassificationExecutor extends AbstractModelExecutor<
+  typeof classificationTests
+> {
+  pattern = /^classification-/;
+
+  protected handlers = Object.fromEntries(
+    classificationTests.map((test) => {
+      switch (test.testId) {
+        case "classification-invalid-image":
+          return [test.testId, this.runInvalidImage.bind(this)];
+        default:
+          return [test.testId, this.runClassify.bind(this)];
+      }
+    }),
+  ) as never;
+
+  private async ensureModel() {
+    return this.resources.ensureLoaded("classification");
+  }
+
+  private readSampleImage(): Uint8Array {
+    return new Uint8Array(fs.readFileSync(SAMPLE_IMAGE_PATH));
+  }
+
+  async runClassify(
+    params: ClassificationParams,
+    expectation: Expectation,
+  ): Promise<TestResult> {
+    try {
+      const modelId = await this.ensureModel();
+      const image = this.readSampleImage();
+      const results = await classify({
+        modelId,
+        image,
+        ...(params.topK !== undefined && { topK: params.topK }),
+      });
+      return ValidationHelpers.validate({ results }, expectation);
+    } catch (error) {
+      return {
+        passed: false,
+        output: `classify failed: ${error instanceof Error ? error.message : String(error)}`,
+      };
+    }
+  }
+
+  async runInvalidImage(
+    _params: ClassificationParams,
+    expectation: Expectation,
+  ): Promise<TestResult> {
+    try {
+      const modelId = await this.ensureModel();
+      // 4 bytes is too small to decode as JPEG/PNG — addon should reject.
+      const badImage = new Uint8Array([0x00, 0x01, 0x02, 0x03]);
+
+      let rejected = false;
+      let errorMsg = "";
+      try {
+        await classify({ modelId, image: badImage });
+      } catch (e) {
+        rejected = true;
+        errorMsg = e instanceof Error ? e.message : String(e);
+      }
+
+      // After the rejection, a fresh valid classify() must succeed —
+      // proves the addon does not wedge on the error path.
+      let recoveryRan = false;
+      try {
+        const goodResults = await classify({
+          modelId,
+          image: this.readSampleImage(),
+        });
+        recoveryRan = Array.isArray(goodResults) && goodResults.length > 0;
+      } catch {
+        recoveryRan = false;
+      }
+
+      return ValidationHelpers.validate(
+        { rejected, recoveryRan, errorMsg },
+        expectation,
+      );
+    } catch (error) {
+      return {
+        passed: false,
+        output: `classification invalid-image test failed: ${error instanceof Error ? error.message : String(error)}`,
+      };
+    }
+  }
+}