diff --git a/API_REFERENCE.md b/API_REFERENCE.md index f8ddfbea..b053aa33 100644 --- a/API_REFERENCE.md +++ b/API_REFERENCE.md @@ -95,6 +95,14 @@ consistently: - [Re-exported SAP AI SDK Classes](#re-exported-sap-ai-sdk-classes) - [Re-exported SAP AI SDK Types](#re-exported-sap-ai-sdk-types) - [`DeploymentConfig`](#deploymentconfig) +- [Model Capabilities Detection](#model-capabilities-detection) + - [Exported Types](#exported-types) + - [`SAPAIModelVendor`](#sapaimodelvendor) + - [`SAPAIModelCapabilities`](#sapaimodelcapabilities) + - [`getSAPAIModelCapabilities(modelId)`](#getsapaimodelcapabilitiesmodelid) + - [`getModelVendor(modelId)`](#getmodelvendormodelid) + - [`modelSupports(modelId, capability)`](#modelsupportsmodelid-capability) + - [Vendor Capability Summary](#vendor-capability-summary) - [Utility Functions](#utility-functions) - [`getProviderName(providerIdentifier)`](#getprovidernameprovideridentifier) - [`buildDpiMaskingProvider(config)`](#builddpimaskingproviderconfig) @@ -2181,18 +2189,23 @@ Implementation of Vercel AI SDK's `LanguageModelV3` interface. **Properties:** -| Property | Type | Description | -| ----------------------------- | -------------------------- | --------------------------------------------------- | -| `specificationVersion` | `'v3'` | API specification version (readonly) | -| `modelId` | `SAPAIModelId` | Current model identifier (readonly) | -| `provider` | `string` | Provider identifier (getter, e.g., `'sap-ai.chat'`) | -| `supportedUrls` | `Record` | URL patterns for supported media (getter) | -| `supportsImageUrls` | `true` | Image URL support flag (readonly) | -| `supportsMultipleCompletions` | `true` | Multiple completions support (readonly) | -| `supportsParallelToolCalls` | `true` | Parallel tool calls support (readonly) | -| `supportsStreaming` | `true` | Streaming support (readonly) | -| `supportsStructuredOutputs` | `true` | Structured output support (readonly) | -| `supportsToolCalls` | `true` | Tool calling support (readonly) | +| Property | Type | Description | +| ----------------------------- | -------------------------- | ------------------------------------------------------ | +| `specificationVersion` | `'v3'` | API specification version (readonly) | +| `modelId` | `SAPAIModelId` | Current model identifier (readonly) | +| `provider` | `string` | Provider identifier (getter, e.g., `'sap-ai.chat'`) | +| `capabilities` | `SAPAIModelCapabilities` | Dynamic model capabilities (getter, cached) | +| `supportedUrls` | `Record` | URL patterns for supported media (getter) | +| `supportsImageUrls` | `boolean` | Image URL support (getter, model-dependent) | +| `supportsMultipleCompletions` | `boolean` | Multiple completions support (getter, model-dependent) | +| `supportsParallelToolCalls` | `boolean` | Parallel tool calls support (getter, model-dependent) | +| `supportsStreaming` | `boolean` | Streaming support (getter, model-dependent) | +| `supportsStructuredOutputs` | `boolean` | Structured output support (getter, model-dependent) | +| `supportsToolCalls` | `boolean` | Tool calling support (getter, model-dependent) | + +> **Note:** Model capabilities are now dynamically determined based on the model vendor +> and type. Use `getSAPAIModelCapabilities(modelId)` to query capabilities programmatically. +> See [Model Capabilities Detection](#model-capabilities-detection) for details. **Methods:** @@ -2600,6 +2613,146 @@ advanced use cases. --- +## Model Capabilities Detection + +The provider includes dynamic capability detection based on model vendor and type, +following SAP AI Core's naming convention: `vendor--model-name`. + +### Exported Types + +#### `SAPAIModelVendor` + +Union type of supported vendor prefixes: + +```typescript +type SAPAIModelVendor = "aicore" | "amazon" | "anthropic" | "azure" | "cohere" | "google" | "meta" | "mistral" | "mistralai"; +``` + +#### `SAPAIModelCapabilities` + +Interface describing the capabilities of a model: + +```typescript +interface SAPAIModelCapabilities { + readonly defaultSystemMessageMode: "system" | "developer" | "user"; + readonly supportsImageInputs: boolean; + readonly supportsN: boolean; + readonly supportsParallelToolCalls: boolean; + readonly supportsStreaming: boolean; + readonly supportsStructuredOutputs: boolean; + readonly supportsToolCalls: boolean; + readonly vendor: SAPAIModelVendor | "unknown"; +} +``` + +### `getSAPAIModelCapabilities(modelId)` + +Returns capability information for a specific SAP AI Core model. + +**Signature:** + +```typescript +function getSAPAIModelCapabilities(modelId: string): SAPAIModelCapabilities; +``` + +**Parameters:** + +- `modelId`: The full model identifier (e.g., `"anthropic--claude-3.5-sonnet"`) + +**Returns:** `SAPAIModelCapabilities` object with the following properties: + +| Property | Type | Description | +| --------------------------- | ----------------------------------- | --------------------------------------------------------- | +| `supportsN` | `boolean` | Multiple completions support (`n` parameter) | +| `supportsImageInputs` | `boolean` | Vision/image input support | +| `supportsParallelToolCalls` | `boolean` | Parallel tool calls in single response | +| `supportsStreaming` | `boolean` | Streaming response support | +| `supportsStructuredOutputs` | `boolean` | JSON schema response format support | +| `supportsToolCalls` | `boolean` | Tool/function calling support | +| `defaultSystemMessageMode` | `"system" \| "developer" \| "user"` | System message mode (`"system"`, `"developer"`, `"user"`) | +| `vendor` | `SAPAIModelVendor \| "unknown"` | Detected vendor or `"unknown"` | + +**Example:** + +```typescript +import { getSAPAIModelCapabilities } from "@jerome-benoit/sap-ai-provider"; + +const capabilities = getSAPAIModelCapabilities("amazon--nova-pro"); +// { +// supportsN: false, // Amazon doesn't support n parameter +// supportsImageInputs: true, +// supportsParallelToolCalls: true, +// supportsStreaming: true, +// supportsStructuredOutputs: true, +// supportsToolCalls: true, +// defaultSystemMessageMode: "system", +// vendor: "amazon" +// } +``` + +### `getModelVendor(modelId)` + +Extracts the vendor prefix from a SAP AI Core model ID. + +**Signature:** + +```typescript +function getModelVendor(modelId: string): SAPAIModelVendor | "unknown"; +``` + +**Parameters:** + +- `modelId`: The full model identifier (e.g., `"anthropic--claude-3.5-sonnet"`) + +**Returns:** The vendor prefix (`"aicore"`, `"amazon"`, `"anthropic"`, `"azure"`, +`"cohere"`, `"google"`, `"meta"`, `"mistral"`, `"mistralai"`) or `"unknown"` if +not recognized. + +**Example:** + +```typescript +import { getModelVendor } from "@jerome-benoit/sap-ai-provider"; + +getModelVendor("anthropic--claude-3.5-sonnet"); // "anthropic" +getModelVendor("gpt-4o"); // "unknown" +``` + +### `modelSupports(modelId, capability)` + +Convenience function to check if a model supports a specific capability. + +**Signature:** + +```typescript +function modelSupports(modelId: string, capability: keyof Omit): boolean; +``` + +**Example:** + +```typescript +import { modelSupports } from "@jerome-benoit/sap-ai-provider"; + +if (modelSupports("amazon--nova-pro", "supportsN")) { + // Use n parameter for multiple completions +} +``` + +### Vendor Capability Summary + +| Vendor | `supportsN` | `supportsStructuredOutputs` | Notes | +| ----------- | ----------- | --------------------------- | ------------------------------- | +| `azure` | ✅ | ✅ | Full capabilities | +| `google` | ✅ | ✅ | Gemini 1.0 has limited outputs | +| `mistral` | ✅ | ✅ | Small/Tiny have limited outputs | +| `mistralai` | ✅ | ✅ | Small/Tiny have limited outputs | +| `cohere` | ✅ | ✅ | Full capabilities | +| `amazon` | ❌ | ✅ | Titan models very limited | +| `anthropic` | ❌ | ✅ | Claude 2.x has limitations | +| `meta` | ✅ | ❌ | Llama 2 lacks tools | +| `aicore` | ✅ | ❌ | Open source models | + +--- + ### Re-exported SAP AI SDK Classes The following classes are re-exported from `@sap-ai-sdk/orchestration` for diff --git a/src/index.ts b/src/index.ts index 66272511..de4c77bc 100644 --- a/src/index.ts +++ b/src/index.ts @@ -32,6 +32,20 @@ export { ApiSwitchError, UnsupportedFeatureError } from "./sap-ai-error.js"; */ export { SAPAILanguageModel } from "./sap-ai-language-model.js"; +/** + * Dynamic model capability detection for SAP AI Core models. + * + * Functions for determining model capabilities based on model ID prefix + * following the SAP AI Core naming convention: `vendor--model-name`. + */ +export { + getModelVendor, + getSAPAIModelCapabilities, + modelSupports, +} from "./sap-ai-model-capabilities.js"; + +export type { SAPAIModelCapabilities, SAPAIModelVendor } from "./sap-ai-model-capabilities.js"; + /** * Provider options for per-call configuration. * diff --git a/src/sap-ai-language-model.test.ts b/src/sap-ai-language-model.test.ts index bace2457..ab0c90dc 100644 --- a/src/sap-ai-language-model.test.ts +++ b/src/sap-ai-language-model.test.ts @@ -1020,27 +1020,99 @@ describe("SAPAILanguageModel", () => { expect(urls["image/*"]?.[1]?.test("data:image/png;base64,Zm9v")).toBe(true); }); - describe("model capabilities", () => { - const expectedCapabilities = { - supportsImageUrls: true, - supportsMultipleCompletions: true, - supportsParallelToolCalls: true, - supportsStreaming: true, - supportsStructuredOutputs: true, - supportsToolCalls: true, - }; + it("should return empty supportedUrls for models without image support", () => { + const model = createModelForApi(api, "meta--llama-3.1-70b"); + expect(model.supportedUrls).toEqual({}); + }); + it("should return false from supportsUrl for models without image support", () => { + const model = createModelForApi(api, "meta--llama-3.1-70b"); + expect(model.supportsUrl(new URL("https://example.com/image.png"))).toBe(false); + expect(model.supportsUrl(new URL("data:image/png;base64,Zm9v"))).toBe(false); + }); + + describe("model capabilities", () => { it.each([ - "any-model", - "gpt-4o", - "anthropic--claude-3.5-sonnet", - "gemini-2.0-flash", - "amazon--nova-pro", - "mistralai--mistral-large-instruct", - "unknown-future-model", - ])("should have consistent capabilities for model %s", (modelId) => { + { + expected: { + supportsImageUrls: true, + supportsMultipleCompletions: true, + supportsParallelToolCalls: true, + supportsStreaming: true, + supportsStructuredOutputs: true, + supportsToolCalls: true, + }, + modelId: "any-model", + }, + { + expected: { + supportsImageUrls: true, + supportsMultipleCompletions: true, + supportsParallelToolCalls: true, + supportsStreaming: true, + supportsStructuredOutputs: true, + supportsToolCalls: true, + }, + modelId: "gpt-4o", + }, + { + expected: { + supportsImageUrls: true, + supportsMultipleCompletions: false, // Anthropic models don't support n parameter + supportsParallelToolCalls: true, + supportsStreaming: true, + supportsStructuredOutputs: true, + supportsToolCalls: true, + }, + modelId: "anthropic--claude-3.5-sonnet", + }, + { + expected: { + supportsImageUrls: true, + supportsMultipleCompletions: true, + supportsParallelToolCalls: true, + supportsStreaming: true, + supportsStructuredOutputs: true, + supportsToolCalls: true, + }, + modelId: "gemini-2.0-flash", + }, + { + expected: { + supportsImageUrls: true, + supportsMultipleCompletions: false, // Amazon models don't support n parameter + supportsParallelToolCalls: true, + supportsStreaming: true, + supportsStructuredOutputs: true, + supportsToolCalls: true, + }, + modelId: "amazon--nova-pro", + }, + { + expected: { + supportsImageUrls: true, + supportsMultipleCompletions: true, + supportsParallelToolCalls: true, + supportsStreaming: true, + supportsStructuredOutputs: true, + supportsToolCalls: true, + }, + modelId: "mistralai--mistral-large-instruct", + }, + { + expected: { + supportsImageUrls: true, + supportsMultipleCompletions: true, + supportsParallelToolCalls: true, + supportsStreaming: true, + supportsStructuredOutputs: true, + supportsToolCalls: true, + }, + modelId: "unknown-future-model", + }, + ])("should have correct capabilities for model $modelId", ({ expected, modelId }) => { const model = createModelForApi(api, modelId); - expect(model).toMatchObject(expectedCapabilities); + expect(model).toMatchObject(expected); }); }); }, diff --git a/src/sap-ai-language-model.ts b/src/sap-ai-language-model.ts index 266b5fe2..999ae49a 100644 --- a/src/sap-ai-language-model.ts +++ b/src/sap-ai-language-model.ts @@ -19,6 +19,10 @@ import { parseProviderOptions } from "@ai-sdk/provider-utils"; import type { SAPAIApiType, SAPAIModelId, SAPAISettings } from "./sap-ai-settings.js"; +import { + getSAPAIModelCapabilities, + type SAPAIModelCapabilities, +} from "./sap-ai-model-capabilities.js"; import { getProviderName, sapAILanguageModelProviderOptions, @@ -78,23 +82,57 @@ interface SAPAILanguageModelConfig { export class SAPAILanguageModel implements LanguageModelV3 { readonly modelId: SAPAIModelId; readonly specificationVersion = "v3"; - readonly supportsImageUrls: boolean = true; - readonly supportsMultipleCompletions: boolean = true; - readonly supportsParallelToolCalls: boolean = true; - readonly supportsStreaming: boolean = true; - readonly supportsStructuredOutputs: boolean = true; - readonly supportsToolCalls: boolean = true; + + /** + * Gets the model capabilities for the current model. + * Cached after first access for performance. + * @returns The model capabilities. + */ + get capabilities(): SAPAIModelCapabilities { + this._capabilities ??= getSAPAIModelCapabilities(this.modelId); + return this._capabilities; + } get provider(): string { return this.config.provider; } get supportedUrls(): Record { + if (!this.capabilities.supportsImageInputs) { + return {}; + } return { "image/*": [/^https:\/\/.+$/i, /^data:image\/.*$/], }; } + get supportsImageUrls(): boolean { + return this.capabilities.supportsImageInputs; + } + + get supportsMultipleCompletions(): boolean { + return this.capabilities.supportsN; + } + + get supportsParallelToolCalls(): boolean { + return this.capabilities.supportsParallelToolCalls; + } + + get supportsStreaming(): boolean { + return this.capabilities.supportsStreaming; + } + + get supportsStructuredOutputs(): boolean { + return this.capabilities.supportsStructuredOutputs; + } + + get supportsToolCalls(): boolean { + return this.capabilities.supportsToolCalls; + } + + /** @internal */ + private _capabilities?: SAPAIModelCapabilities; + /** @internal */ private readonly config: SAPAILanguageModelConfig; @@ -127,6 +165,9 @@ export class SAPAILanguageModel implements LanguageModelV3 { } supportsUrl(url: URL): boolean { + if (!this.capabilities.supportsImageInputs) { + return false; + } if (url.protocol === "https:") return true; if (url.protocol === "data:") { return /^data:image\//i.test(url.href); diff --git a/src/sap-ai-model-capabilities.test.ts b/src/sap-ai-model-capabilities.test.ts new file mode 100644 index 00000000..e8ffbaca --- /dev/null +++ b/src/sap-ai-model-capabilities.test.ts @@ -0,0 +1,370 @@ +/** Unit tests for SAP AI Model Capabilities detection. */ + +import { describe, expect, it } from "vitest"; + +import { + getModelVendor, + getSAPAIModelCapabilities, + modelSupports, + type SAPAIModelCapabilities, + type SAPAIModelVendor, +} from "./sap-ai-model-capabilities"; + +describe("sap-ai-model-capabilities", () => { + describe("getModelVendor", () => { + it.each<{ expected: "unknown" | SAPAIModelVendor; modelId: string }>([ + { expected: "aicore", modelId: "aicore--llama-3.1-70b" }, + { expected: "amazon", modelId: "amazon--nova-pro" }, + { expected: "amazon", modelId: "amazon--titan-text-express" }, + { expected: "anthropic", modelId: "anthropic--claude-3.5-sonnet" }, + { expected: "anthropic", modelId: "anthropic--claude-2.1" }, + { expected: "azure", modelId: "azure--gpt-4o" }, + { expected: "azure", modelId: "azure--gpt-4-turbo" }, + { expected: "google", modelId: "google--gemini-2.0-flash" }, + { expected: "google", modelId: "google--gemini-1.0-pro" }, + { expected: "meta", modelId: "meta--llama-3.1-70b" }, + { expected: "meta", modelId: "meta--llama-2-70b" }, + { expected: "mistral", modelId: "mistral--mistral-large" }, + { expected: "mistral", modelId: "mistral--mistral-small" }, + { expected: "mistralai", modelId: "mistralai--mistral-large-instruct" }, + { expected: "mistralai", modelId: "mistralai--mistral-small" }, + { expected: "mistralai", modelId: "mistralai--mistral-small-instruct" }, + { expected: "cohere", modelId: "cohere--command-a-reasoning" }, + { expected: "cohere", modelId: "cohere--command-r-plus" }, + { expected: "meta", modelId: "meta--llama3.1-70b-instruct" }, + ])("should return '$expected' for model '$modelId'", ({ expected, modelId }) => { + expect(getModelVendor(modelId)).toBe(expected); + }); + + it.each([ + "gpt-4o", + "claude-3.5-sonnet", + "gemini-2.0-flash", + "unknown-model", + "", + "no-double-dash", + ])("should return 'unknown' for model without vendor prefix: '%s'", (modelId) => { + expect(getModelVendor(modelId)).toBe("unknown"); + }); + + it("should be case-insensitive for vendor extraction", () => { + expect(getModelVendor("Amazon--Nova-Pro")).toBe("amazon"); + expect(getModelVendor("ANTHROPIC--CLAUDE-3")).toBe("anthropic"); + }); + + it("should return 'unknown' for unrecognized vendor prefixes", () => { + expect(getModelVendor("foobar--some-model")).toBe("unknown"); + expect(getModelVendor("openai--gpt-4")).toBe("unknown"); + }); + }); + + describe("getSAPAIModelCapabilities", () => { + describe("default capabilities", () => { + it("should return all capabilities enabled for unknown models", () => { + const capabilities = getSAPAIModelCapabilities("unknown-model"); + + expect(capabilities).toEqual({ + defaultSystemMessageMode: "system", + supportsImageInputs: true, + supportsN: true, + supportsParallelToolCalls: true, + supportsStreaming: true, + supportsStructuredOutputs: true, + supportsToolCalls: true, + vendor: "unknown", + } satisfies SAPAIModelCapabilities); + }); + }); + + describe("vendor-specific capabilities", () => { + it("should disable supportsN for Amazon models", () => { + const capabilities = getSAPAIModelCapabilities("amazon--nova-pro"); + + expect(capabilities.supportsN).toBe(false); + expect(capabilities.vendor).toBe("amazon"); + expect(capabilities.supportsToolCalls).toBe(true); + expect(capabilities.supportsStreaming).toBe(true); + }); + + it("should disable supportsN for Anthropic models", () => { + const capabilities = getSAPAIModelCapabilities("anthropic--claude-3.5-sonnet"); + + expect(capabilities.supportsN).toBe(false); + expect(capabilities.vendor).toBe("anthropic"); + expect(capabilities.supportsToolCalls).toBe(true); + expect(capabilities.supportsParallelToolCalls).toBe(true); + }); + + it("should disable supportsStructuredOutputs for AI Core models", () => { + const capabilities = getSAPAIModelCapabilities("aicore--llama-3.1-70b"); + + expect(capabilities.supportsStructuredOutputs).toBe(false); + expect(capabilities.vendor).toBe("aicore"); + expect(capabilities.supportsN).toBe(true); + }); + + it("should disable supportsStructuredOutputs for Meta models", () => { + const capabilities = getSAPAIModelCapabilities("meta--llama-3.1-70b"); + + expect(capabilities.supportsStructuredOutputs).toBe(false); + expect(capabilities.vendor).toBe("meta"); + }); + + it("should have all capabilities enabled for Azure models", () => { + const capabilities = getSAPAIModelCapabilities("azure--gpt-4o"); + + expect(capabilities.supportsN).toBe(true); + expect(capabilities.supportsStructuredOutputs).toBe(true); + expect(capabilities.supportsToolCalls).toBe(true); + expect(capabilities.supportsParallelToolCalls).toBe(true); + expect(capabilities.vendor).toBe("azure"); + }); + + it("should have all capabilities enabled for Google models", () => { + const capabilities = getSAPAIModelCapabilities("google--gemini-2.0-flash"); + + expect(capabilities.supportsN).toBe(true); + expect(capabilities.supportsStructuredOutputs).toBe(true); + expect(capabilities.vendor).toBe("google"); + }); + + it("should have all capabilities enabled for Mistral models", () => { + const capabilities = getSAPAIModelCapabilities("mistral--mistral-large"); + + expect(capabilities.supportsN).toBe(true); + expect(capabilities.vendor).toBe("mistral"); + }); + + it("should have all capabilities enabled for MistralAI models", () => { + const capabilities = getSAPAIModelCapabilities("mistralai--mistral-large-instruct"); + + expect(capabilities.supportsN).toBe(true); + expect(capabilities.supportsStructuredOutputs).toBe(true); + expect(capabilities.supportsToolCalls).toBe(true); + expect(capabilities.vendor).toBe("mistralai"); + }); + + it("should have all capabilities enabled for Cohere models", () => { + const capabilities = getSAPAIModelCapabilities("cohere--command-a-reasoning"); + + expect(capabilities.supportsN).toBe(true); + expect(capabilities.supportsStructuredOutputs).toBe(true); + expect(capabilities.supportsToolCalls).toBe(true); + expect(capabilities.vendor).toBe("cohere"); + }); + }); + + describe("model-specific overrides", () => { + describe("Claude 2.x models", () => { + it.each(["anthropic--claude-2", "anthropic--claude-2.0", "anthropic--claude-2.1"])( + "should have limited capabilities for %s", + (modelId) => { + const capabilities = getSAPAIModelCapabilities(modelId); + + expect(capabilities.supportsN).toBe(false); + expect(capabilities.supportsParallelToolCalls).toBe(false); + expect(capabilities.supportsStructuredOutputs).toBe(false); + expect(capabilities.supportsToolCalls).toBe(true); + }, + ); + + it("should not match hypothetical claude-20 models", () => { + const capabilities = getSAPAIModelCapabilities("anthropic--claude-20-turbo"); + + // Should fall back to vendor defaults (anthropic), not claude-2 specific + expect(capabilities.supportsParallelToolCalls).toBe(true); + expect(capabilities.supportsStructuredOutputs).toBe(true); + }); + }); + + describe("Amazon Titan models", () => { + it.each([ + "amazon--titan-text-express", + "amazon--titan-text-lite", + "amazon--titan-embed-text-v1", + "amazon--titan-embed-text-v2", + ])("should have limited capabilities for %s", (modelId) => { + const capabilities = getSAPAIModelCapabilities(modelId); + + expect(capabilities.supportsN).toBe(false); + expect(capabilities.supportsImageInputs).toBe(false); + expect(capabilities.supportsParallelToolCalls).toBe(false); + expect(capabilities.supportsStructuredOutputs).toBe(false); + expect(capabilities.supportsToolCalls).toBe(false); + }); + + it("should not match other Amazon models", () => { + const capabilities = getSAPAIModelCapabilities("amazon--nova-pro"); + + // Should fall back to vendor defaults (amazon), not titan-specific + expect(capabilities.supportsN).toBe(false); // Amazon vendor default + expect(capabilities.supportsToolCalls).toBe(true); + expect(capabilities.supportsImageInputs).toBe(true); + }); + }); + + describe("Llama 2 models", () => { + it.each([ + "meta--llama-2-70b", + "meta--llama-2-13b", + "aicore--llama-2-70b-chat", + "aicore--llama-2-13b-chat", + ])("should have limited capabilities for %s", (modelId) => { + const capabilities = getSAPAIModelCapabilities(modelId); + + expect(capabilities.supportsImageInputs).toBe(false); + expect(capabilities.supportsStructuredOutputs).toBe(false); + expect(capabilities.supportsToolCalls).toBe(false); + }); + }); + + describe("Llama 3.1+ models", () => { + it.each([ + "meta--llama-3.1-70b", + "meta--llama-3.2-90b", + "aicore--llama-3.1-8b", + "meta--llama-3.10-70b", + "meta--llama-3.11-128b", + "aicore--llama-3.15-70b", + // Without dash format (llama3.1 vs llama-3.1) - documented in API_REFERENCE.md + "meta--llama3.1-70b-instruct", + "meta--llama3.2-90b-instruct", + "aicore--llama3.1-8b", + ])("should support tools for %s", (modelId) => { + const capabilities = getSAPAIModelCapabilities(modelId); + + expect(capabilities.supportsToolCalls).toBe(true); + expect(capabilities.supportsImageInputs).toBe(false); + expect(capabilities.supportsStructuredOutputs).toBe(false); + }); + }); + + describe("Llama 3.2 Vision models", () => { + it.each([ + "meta--llama-3.2-11b-vision", + "meta--llama-3.2-90b-vision", + "meta--llama-3.2-11b-vision-instruct", + "meta--llama-3.2-90b-vision-instruct", + "aicore--llama-3.2-11b-vision", + "aicore--llama3.2-90b-vision", + // Case insensitive + "meta--llama-3.2-90b-Vision", + "meta--llama-3.2-90b-VISION-instruct", + ])("should support image inputs for %s", (modelId) => { + const capabilities = getSAPAIModelCapabilities(modelId); + + expect(capabilities.supportsImageInputs).toBe(true); + expect(capabilities.supportsToolCalls).toBe(true); + expect(capabilities.supportsStructuredOutputs).toBe(false); + }); + }); + + describe("Llama 3.0 models", () => { + it.each([ + "meta--llama-3.0-70b", + "meta--llama-3.0-8b", + "aicore--llama-3.0-70b-chat", + "meta--llama-3-70b", + "aicore--llama-3-8b", + ])("should fall to vendor defaults for %s", (modelId) => { + const capabilities = getSAPAIModelCapabilities(modelId); + + expect(capabilities.supportsStructuredOutputs).toBe(false); + expect(capabilities.supportsImageInputs).toBe(true); + expect(capabilities.supportsToolCalls).toBe(true); + }); + }); + + describe("Gemini 1.0 models", () => { + it.each(["google--gemini-1.0-pro", "google--gemini-1.0-ultra"])( + "should have limited structured output for %s", + (modelId) => { + const capabilities = getSAPAIModelCapabilities(modelId); + + expect(capabilities.supportsStructuredOutputs).toBe(false); + expect(capabilities.supportsN).toBe(true); + }, + ); + + it("should have full capabilities for Gemini 1.5+", () => { + const capabilities = getSAPAIModelCapabilities("google--gemini-1.5-pro"); + + expect(capabilities.supportsStructuredOutputs).toBe(true); + expect(capabilities.supportsN).toBe(true); + }); + }); + + describe("Mistral Small/Tiny models", () => { + it.each([ + "mistral--mistral-small", + "mistral--mistral-tiny", + "mistralai--mistral-small", + "mistralai--mistral-tiny", + ])("should have limited structured output for %s", (modelId) => { + const capabilities = getSAPAIModelCapabilities(modelId); + + expect(capabilities.supportsStructuredOutputs).toBe(false); + expect(capabilities.supportsN).toBe(true); + }); + + it("should have full capabilities for Mistral Large", () => { + const capabilities = getSAPAIModelCapabilities("mistral--mistral-large"); + + expect(capabilities.supportsStructuredOutputs).toBe(true); + }); + + it("should have full capabilities for MistralAI Large Instruct", () => { + const capabilities = getSAPAIModelCapabilities("mistralai--mistral-large-instruct"); + + expect(capabilities.supportsStructuredOutputs).toBe(true); + }); + }); + + describe("MistralAI Instruct models", () => { + it("should have full capabilities for mistralai--mistral-small-instruct", () => { + const capabilities = getSAPAIModelCapabilities("mistralai--mistral-small-instruct"); + + // Note: mistral-small has limited structured outputs, but -instruct suffix + // doesn't change this - the pattern matches on mistral-small prefix + expect(capabilities.supportsStructuredOutputs).toBe(false); + expect(capabilities.supportsN).toBe(true); + expect(capabilities.vendor).toBe("mistralai"); + }); + }); + }); + + describe("case insensitivity", () => { + it("should handle mixed case model IDs", () => { + const capabilities = getSAPAIModelCapabilities("Amazon--Nova-Pro"); + + expect(capabilities.vendor).toBe("amazon"); + expect(capabilities.supportsN).toBe(false); + }); + }); + }); + + describe("modelSupports", () => { + it("should return true for supported capabilities", () => { + expect(modelSupports("azure--gpt-4o", "supportsN")).toBe(true); + expect(modelSupports("azure--gpt-4o", "supportsToolCalls")).toBe(true); + expect(modelSupports("azure--gpt-4o", "supportsStructuredOutputs")).toBe(true); + }); + + it("should return false for unsupported capabilities", () => { + expect(modelSupports("amazon--nova-pro", "supportsN")).toBe(false); + expect(modelSupports("anthropic--claude-3.5-sonnet", "supportsN")).toBe(false); + expect(modelSupports("amazon--titan-text-express", "supportsToolCalls")).toBe(false); + }); + + it("should work with all boolean capability keys", () => { + const modelId = "azure--gpt-4o"; + + expect(typeof modelSupports(modelId, "supportsImageInputs")).toBe("boolean"); + expect(typeof modelSupports(modelId, "supportsN")).toBe("boolean"); + expect(typeof modelSupports(modelId, "supportsParallelToolCalls")).toBe("boolean"); + expect(typeof modelSupports(modelId, "supportsStreaming")).toBe("boolean"); + expect(typeof modelSupports(modelId, "supportsStructuredOutputs")).toBe("boolean"); + expect(typeof modelSupports(modelId, "supportsToolCalls")).toBe("boolean"); + }); + }); +}); diff --git a/src/sap-ai-model-capabilities.ts b/src/sap-ai-model-capabilities.ts new file mode 100644 index 00000000..d6fed2cc --- /dev/null +++ b/src/sap-ai-model-capabilities.ts @@ -0,0 +1,267 @@ +/** + * Dynamic model capability detection for SAP AI Core models. + * + * Provides model-specific capability information based on the model ID prefix, + * following the SAP AI Core naming convention: `vendor--model-name`. + * @example + * ```typescript + * const capabilities = getSAPAIModelCapabilities("anthropic--claude-3.5-sonnet"); + * // { supportsN: false, supportsParallelToolCalls: true, ... } + * ``` + */ + +/** + * Capability information for a SAP AI Core model. + * Used to determine which features are available for a specific model. + */ +export interface SAPAIModelCapabilities { + /** + * Default system message mode for this model. + * - 'system': Standard system role (most models) + * - 'developer': Developer role (some reasoning models) + * - 'user': Prepend to first user message (legacy models) + */ + readonly defaultSystemMessageMode: "developer" | "system" | "user"; + + /** + * Whether the model supports image inputs (vision capability). + * @default true + */ + readonly supportsImageInputs: boolean; + + /** + * Whether the model supports the `n` parameter for multiple completions. + * Amazon Bedrock and Anthropic models do not support this parameter. + * @default true + */ + readonly supportsN: boolean; + + /** + * Whether the model supports parallel tool calls in a single response. + * Most modern models support this, but some older or specialized models may not. + * @default true + */ + readonly supportsParallelToolCalls: boolean; + + /** + * Whether the model supports streaming responses. + * @default true + */ + readonly supportsStreaming: boolean; + + /** + * Whether the model supports structured JSON output (json_schema response format). + * @default true + */ + readonly supportsStructuredOutputs: boolean; + + /** + * Whether the model supports tool/function calling. + * @default true + */ + readonly supportsToolCalls: boolean; + + /** + * The detected vendor for this model. + */ + readonly vendor: "unknown" | SAPAIModelVendor; +} + +/** + * Model vendor prefixes used in SAP AI Core Orchestration Service. + * Models are identified as `vendor--model-name`. + */ +export type SAPAIModelVendor = + | "aicore" + | "amazon" + | "anthropic" + | "azure" + | "cohere" + | "google" + | "meta" + | "mistral" + | "mistralai"; + +/** + * Module-level cache for model capabilities to avoid repeated computation. + * @internal + */ +const capabilitiesCache = new Map(); + +/** + * Default capabilities for models without specific overrides. + * @internal + */ +const DEFAULT_CAPABILITIES: SAPAIModelCapabilities = { + defaultSystemMessageMode: "system", + supportsImageInputs: true, + supportsN: true, + supportsParallelToolCalls: true, + supportsStreaming: true, + supportsStructuredOutputs: true, + supportsToolCalls: true, + vendor: "unknown", +}; + +/** + * Vendor-specific capability overrides. + * @internal + */ +const VENDOR_CAPABILITIES: Record> = { + aicore: { supportsStructuredOutputs: false }, + amazon: { supportsN: false }, + anthropic: { supportsN: false }, + azure: {}, + cohere: {}, + google: {}, + meta: { supportsStructuredOutputs: false }, + mistral: {}, + mistralai: {}, +}; + +/** + * Model-specific capability overrides for known model patterns. + * These take precedence over vendor defaults. + * + * Patterns are evaluated in array order (first match wins). + * More specific patterns should precede general ones. + * @internal + */ +const MODEL_SPECIFIC_CAPABILITIES: { + capabilities: Partial; + pattern: RegExp; +}[] = [ + { + // Matches claude-2, claude-2.0, claude-2.1, but not claude-20 + capabilities: { supportsParallelToolCalls: false, supportsStructuredOutputs: false }, + pattern: /^anthropic--claude-2($|[.-])/, + }, + { + capabilities: { + supportsImageInputs: false, + supportsParallelToolCalls: false, + supportsStructuredOutputs: false, + supportsToolCalls: false, + }, + pattern: /^amazon--titan-(text|embed)/, + }, + { + capabilities: { + supportsImageInputs: false, + supportsStructuredOutputs: false, + supportsToolCalls: false, + }, + pattern: /^(meta--llama-2|aicore--llama-2)/, + }, + { + // Llama 3.2+ Vision models support image inputs + capabilities: { supportsImageInputs: true, supportsToolCalls: true }, + pattern: /^(meta--llama-?3\.[2-9][0-9]*|aicore--llama-?3\.[2-9][0-9]*).*vision/i, + }, + { + // Matches both "llama-3.1" and "llama3.1" formats for version 3.1+ + capabilities: { supportsImageInputs: false, supportsToolCalls: true }, + pattern: /^(meta--llama-?3\.[1-9][0-9]*|aicore--llama-?3\.[1-9][0-9]*)/, + }, + { + capabilities: { supportsStructuredOutputs: false }, + pattern: /^google--gemini-1\.0/, + }, + { + capabilities: { supportsStructuredOutputs: false }, + pattern: /^mistral(ai)?--(mistral-small|mistral-tiny)/, + }, +]; + +/** + * Extracts the vendor prefix from a SAP AI Core model ID. + * + * SAP AI Core uses the convention `vendor--model-name` for model identification. + * @param modelId - The full model identifier (e.g., "anthropic--claude-3.5-sonnet"). + * @returns The vendor prefix, or "unknown" if not recognized. + * @example + * ```typescript + * getModelVendor("anthropic--claude-3.5-sonnet"); // "anthropic" + * getModelVendor("gpt-4o"); // "unknown" + * ``` + */ +export function getModelVendor(modelId: string): "unknown" | SAPAIModelVendor { + const vendorMatch = /^([a-z]+)--/.exec(modelId.toLowerCase()); + if (!vendorMatch) { + return "unknown"; + } + + const vendor = vendorMatch[1] as SAPAIModelVendor; + if (vendor in VENDOR_CAPABILITIES) { + return vendor; + } + + return "unknown"; +} + +/** + * Gets the capability information for a specific SAP AI Core model. + * + * Capabilities are built up by applying overrides in this order: + * 1. Global defaults (base capabilities) + * 2. Vendor defaults (override globals) + * 3. Model-specific patterns (highest priority, override vendor defaults) + * @param modelId - The full model identifier (e.g., "anthropic--claude-3.5-sonnet"). + * @returns The model's capabilities. + * @example + * ```typescript + * const capabilities = getSAPAIModelCapabilities("amazon--nova-pro"); + * if (!capabilities.supportsN) { + * // Don't use n parameter for this model + * } + * ``` + */ +export function getSAPAIModelCapabilities(modelId: string): SAPAIModelCapabilities { + const normalizedModelId = modelId.toLowerCase(); + + const cached = capabilitiesCache.get(normalizedModelId); + if (cached) { + return cached; + } + + const vendor = getModelVendor(modelId); + + let capabilities: SAPAIModelCapabilities = { ...DEFAULT_CAPABILITIES }; + + if (vendor !== "unknown") { + capabilities = { ...capabilities, ...VENDOR_CAPABILITIES[vendor] }; + } + + for (const { capabilities: modelCapabilities, pattern } of MODEL_SPECIFIC_CAPABILITIES) { + if (pattern.test(normalizedModelId)) { + capabilities = { ...capabilities, ...modelCapabilities }; + break; + } + } + + const result = Object.freeze({ ...capabilities, vendor }); + capabilitiesCache.set(normalizedModelId, result); + return result; +} + +/** + * Checks if a model supports a specific capability. + * + * Convenience function for checking individual capabilities. + * @param modelId - The full model identifier. + * @param capability - The capability to check. + * @returns True if the model supports the capability. + * @example + * ```typescript + * if (modelSupports("anthropic--claude-3.5-sonnet", "supportsN")) { + * // Use n parameter + * } + * ``` + */ +export function modelSupports( + modelId: string, + capability: keyof Omit, +): boolean { + const capabilities = getSAPAIModelCapabilities(modelId); + return capabilities[capability]; +}