diff --git a/.changeset/brave-bikes-happen.md b/.changeset/brave-bikes-happen.md new file mode 100644 index 000000000000..1da73e228570 --- /dev/null +++ b/.changeset/brave-bikes-happen.md @@ -0,0 +1,5 @@ +--- +'@ai-sdk/anthropic': patch +--- + +feat (provider/anthropic): pdf support diff --git a/content/docs/02-foundations/03-prompts.mdx b/content/docs/02-foundations/03-prompts.mdx index 0cd862cf46e4..9a1ee1a3ab30 100644 --- a/content/docs/02-foundations/03-prompts.mdx +++ b/content/docs/02-foundations/03-prompts.mdx @@ -199,9 +199,10 @@ const result = await generateText({ Only a few providers and models currently support file parts: [Google Generative AI](/providers/ai-sdk-providers/google-generative-ai), [Google - Vertex AI](/providers/ai-sdk-providers/google-vertex), and + Vertex AI](/providers/ai-sdk-providers/google-vertex), [OpenAI](/providers/ai-sdk-providers/openai) (for `wav` and `mp3` audio with - `gpt-4o-audio-preview) + `gpt-4o-audio-preview), [Anthropic](/providers/ai-sdk-providers/anthropic) + (for `pdf`). User messages can include file parts. A file can be one of the following: diff --git a/content/providers/01-ai-sdk-providers/05-anthropic.mdx b/content/providers/01-ai-sdk-providers/05-anthropic.mdx index e47c6f16708e..792c6863977a 100644 --- a/content/providers/01-ai-sdk-providers/05-anthropic.mdx +++ b/content/providers/01-ai-sdk-providers/05-anthropic.mdx @@ -280,6 +280,38 @@ Parameters: These tools can be used in conjunction with the `sonnet-3-5-sonnet-20240620` model to enable more complex interactions and tasks. +### PDF support + +Anthropic Sonnet `claude-3-5-sonnet-20241022` supports reading PDF files. +You can pass PDF files as part of the message content using the `file` type: + +```ts +const result = await generateText({ + model: anthropic('claude-3-5-sonnet-20241022'), + messages: [ + { + role: 'user', + content: [ + { + type: 'text', + text: 'What is an embedding model according to this document?', + }, + { + type: 'file', + data: fs.readFileSync('./data/ai.pdf'), + mimeType: 'application/pdf', + }, + ], + }, + ], +}); +``` + +The model will have access to the contents of the PDF file and +respond to questions about it. +The PDF file should be passed using the `data` field, +and the `mimeType` should be set to `'application/pdf'`. + ### Model Capabilities See also [Anthropic Model Comparison](https://docs.anthropic.com/en/docs/about-claude/models#model-comparison). diff --git a/examples/ai-core/src/generate-text/anthropic-image.ts b/examples/ai-core/src/generate-text/anthropic-image.ts index b9a52feeda93..c4e5c3f79053 100644 --- a/examples/ai-core/src/generate-text/anthropic-image.ts +++ b/examples/ai-core/src/generate-text/anthropic-image.ts @@ -6,7 +6,6 @@ import fs from 'node:fs'; async function main() { const result = await generateText({ model: anthropic('claude-3-5-sonnet-20240620'), - maxTokens: 512, messages: [ { role: 'user', diff --git a/examples/ai-core/src/generate-text/anthropic-pdf.ts b/examples/ai-core/src/generate-text/anthropic-pdf.ts new file mode 100644 index 000000000000..1c7e94a2dc44 --- /dev/null +++ b/examples/ai-core/src/generate-text/anthropic-pdf.ts @@ -0,0 +1,30 @@ +import { anthropic } from '@ai-sdk/anthropic'; +import { generateText } from 'ai'; +import 'dotenv/config'; +import fs from 'node:fs'; + +async function main() { + const result = await generateText({ + model: anthropic('claude-3-5-sonnet-20241022'), + messages: [ + { + role: 'user', + content: [ + { + type: 'text', + text: 'What is an embedding model according to this document?', + }, + { + type: 'file', + data: fs.readFileSync('./data/ai.pdf'), + mimeType: 'application/pdf', + }, + ], + }, + ], + }); + + console.log(result.text); +} + +main().catch(console.error); diff --git a/examples/ai-core/src/stream-text/anthropic-image.ts b/examples/ai-core/src/stream-text/anthropic-image.ts index 5b36b4ef7a0a..32bf60d60126 100644 --- a/examples/ai-core/src/stream-text/anthropic-image.ts +++ b/examples/ai-core/src/stream-text/anthropic-image.ts @@ -6,7 +6,6 @@ import fs from 'node:fs'; async function main() { const result = await streamText({ model: anthropic('claude-3-5-sonnet-20240620'), - maxTokens: 512, messages: [ { role: 'user', diff --git a/examples/ai-core/src/stream-text/anthropic-pdf.ts b/examples/ai-core/src/stream-text/anthropic-pdf.ts new file mode 100644 index 000000000000..4a2d916ab27f --- /dev/null +++ b/examples/ai-core/src/stream-text/anthropic-pdf.ts @@ -0,0 +1,32 @@ +import { anthropic } from '@ai-sdk/anthropic'; +import { streamText } from 'ai'; +import 'dotenv/config'; +import fs from 'node:fs'; + +async function main() { + const result = await streamText({ + model: anthropic('claude-3-5-sonnet-20241022'), + messages: [ + { + role: 'user', + content: [ + { + type: 'text', + text: 'What is an embedding model according to this document?', + }, + { + type: 'file', + data: fs.readFileSync('./data/ai.pdf'), + mimeType: 'application/pdf', + }, + ], + }, + ], + }); + + for await (const textPart of result.textStream) { + process.stdout.write(textPart); + } +} + +main().catch(console.error); diff --git a/packages/anthropic/src/anthropic-api-types.ts b/packages/anthropic/src/anthropic-api-types.ts index 5714a918b872..ed2e0d8b73da 100644 --- a/packages/anthropic/src/anthropic-api-types.ts +++ b/packages/anthropic/src/anthropic-api-types.ts @@ -12,7 +12,10 @@ export type AnthropicCacheControl = { type: 'ephemeral' }; export interface AnthropicUserMessage { role: 'user'; content: Array< - AnthropicTextContent | AnthropicImageContent | AnthropicToolResultContent + | AnthropicTextContent + | AnthropicImageContent + | AnthropicDocumentContent + | AnthropicToolResultContent >; } @@ -37,6 +40,16 @@ export interface AnthropicImageContent { cache_control: AnthropicCacheControl | undefined; } +export interface AnthropicDocumentContent { + type: 'document'; + source: { + type: 'base64'; + media_type: 'application/pdf'; + data: string; + }; + cache_control: AnthropicCacheControl | undefined; +} + export interface AnthropicToolCallContent { type: 'tool_use'; id: string; diff --git a/packages/anthropic/src/anthropic-messages-language-model.ts b/packages/anthropic/src/anthropic-messages-language-model.ts index 69d7d21ef148..f720eed14a7b 100644 --- a/packages/anthropic/src/anthropic-messages-language-model.ts +++ b/packages/anthropic/src/anthropic-messages-language-model.ts @@ -102,10 +102,11 @@ export class AnthropicMessagesLanguageModel implements LanguageModelV1 { }); } - const messagesPrompt = convertToAnthropicMessagesPrompt({ - prompt, - cacheControl: this.settings.cacheControl ?? false, - }); + const { prompt: messagesPrompt, betas: messagesBetas } = + convertToAnthropicMessagesPrompt({ + prompt, + cacheControl: this.settings.cacheControl ?? false, + }); const baseArgs = { // model id: @@ -127,12 +128,17 @@ export class AnthropicMessagesLanguageModel implements LanguageModelV1 { switch (type) { case 'regular': { - const { tools, tool_choice, toolWarnings, betas } = prepareTools(mode); + const { + tools, + tool_choice, + toolWarnings, + betas: toolsBetas, + } = prepareTools(mode); return { args: { ...baseArgs, tools, tool_choice }, warnings: [...warnings, ...toolWarnings], - betas, + betas: new Set([...messagesBetas, ...toolsBetas]), }; } @@ -152,7 +158,7 @@ export class AnthropicMessagesLanguageModel implements LanguageModelV1 { tool_choice: { type: 'tool', name }, }, warnings, - betas: new Set(), + betas: messagesBetas, }; } diff --git a/packages/anthropic/src/convert-to-anthropic-messages-prompt.test.ts b/packages/anthropic/src/convert-to-anthropic-messages-prompt.test.ts index 9be0658e0a28..eb7f10704de7 100644 --- a/packages/anthropic/src/convert-to-anthropic-messages-prompt.test.ts +++ b/packages/anthropic/src/convert-to-anthropic-messages-prompt.test.ts @@ -8,8 +8,11 @@ describe('system messages', () => { }); expect(result).toEqual({ - messages: [], - system: [{ type: 'text', text: 'This is a system message' }], + prompt: { + messages: [], + system: [{ type: 'text', text: 'This is a system message' }], + }, + betas: new Set(), }); }); @@ -23,11 +26,14 @@ describe('system messages', () => { }); expect(result).toEqual({ - messages: [], - system: [ - { type: 'text', text: 'This is a system message' }, - { type: 'text', text: 'This is another system message' }, - ], + prompt: { + messages: [], + system: [ + { type: 'text', text: 'This is a system message' }, + { type: 'text', text: 'This is another system message' }, + ], + }, + betas: new Set(), }); }); }); @@ -51,23 +57,106 @@ describe('user messages', () => { }); expect(result).toEqual({ - messages: [ + prompt: { + messages: [ + { + role: 'user', + content: [ + { + type: 'image', + source: { + data: 'AAECAw==', + media_type: 'image/png', + type: 'base64', + }, + }, + ], + }, + ], + system: undefined, + }, + betas: new Set(), + }); + }); + + it('should add PDF file parts', async () => { + const result = convertToAnthropicMessagesPrompt({ + prompt: [ { role: 'user', content: [ { - type: 'image', - source: { - data: 'AAECAw==', - media_type: 'image/png', - type: 'base64', - }, + type: 'file', + data: 'base64PDFdata', + mimeType: 'application/pdf', }, ], }, ], - system: undefined, + cacheControl: false, }); + + expect(result).toEqual({ + prompt: { + messages: [ + { + role: 'user', + content: [ + { + type: 'document', + source: { + type: 'base64', + media_type: 'application/pdf', + data: 'base64PDFdata', + }, + }, + ], + }, + ], + system: undefined, + }, + betas: new Set(['pdfs-2024-09-25']), + }); + }); + + it('should throw error for non-PDF file types', async () => { + expect(() => + convertToAnthropicMessagesPrompt({ + prompt: [ + { + role: 'user', + content: [ + { + type: 'file', + data: 'base64data', + mimeType: 'text/plain', + }, + ], + }, + ], + cacheControl: false, + }), + ).toThrow('Non-PDF files in user messages'); + }); + + it('should throw error for URL-based file parts', async () => { + expect(() => + convertToAnthropicMessagesPrompt({ + prompt: [ + { + role: 'user', + content: [ + { + type: 'file', + data: 'base64data', + mimeType: 'text/plain', + }, + ], + }, + ], + cacheControl: false, + }), + ).toThrow('Non-PDF files in user messages'); }); }); @@ -91,20 +180,23 @@ describe('tool messages', () => { }); expect(result).toEqual({ - messages: [ - { - role: 'user', - content: [ - { - type: 'tool_result', - tool_use_id: 'tool-call-1', - is_error: undefined, - content: JSON.stringify({ test: 'This is a tool message' }), - }, - ], - }, - ], - system: undefined, + prompt: { + messages: [ + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'tool-call-1', + is_error: undefined, + content: JSON.stringify({ test: 'This is a tool message' }), + }, + ], + }, + ], + system: undefined, + }, + betas: new Set(), }); }); @@ -133,26 +225,29 @@ describe('tool messages', () => { }); expect(result).toEqual({ - messages: [ - { - role: 'user', - content: [ - { - type: 'tool_result', - tool_use_id: 'tool-call-1', - is_error: undefined, - content: JSON.stringify({ test: 'This is a tool message' }), - }, - { - type: 'tool_result', - tool_use_id: 'tool-call-2', - is_error: undefined, - content: JSON.stringify({ something: 'else' }), - }, - ], - }, - ], - system: undefined, + prompt: { + messages: [ + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'tool-call-1', + is_error: undefined, + content: JSON.stringify({ test: 'This is a tool message' }), + }, + { + type: 'tool_result', + tool_use_id: 'tool-call-2', + is_error: undefined, + content: JSON.stringify({ something: 'else' }), + }, + ], + }, + ], + system: undefined, + }, + betas: new Set(), }); }); @@ -179,21 +274,24 @@ describe('tool messages', () => { }); expect(result).toEqual({ - messages: [ - { - role: 'user', - content: [ - { - type: 'tool_result', - tool_use_id: 'tool-call-1', - is_error: undefined, - content: JSON.stringify({ test: 'This is a tool message' }), - }, - { type: 'text', text: 'This is a user message' }, - ], - }, - ], - system: undefined, + prompt: { + messages: [ + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'tool-call-1', + is_error: undefined, + content: JSON.stringify({ test: 'This is a tool message' }), + }, + { type: 'text', text: 'This is a user message' }, + ], + }, + ], + system: undefined, + }, + betas: new Set(), }); }); @@ -227,30 +325,32 @@ describe('tool messages', () => { }); expect(result).toEqual({ - messages: [ - { - role: 'user', - content: [ - { - type: 'tool_result', - tool_use_id: 'image-gen-1', - is_error: undefined, - content: [ - { type: 'text', text: 'Image generated successfully' }, - { - type: 'image', - source: { - type: 'base64', - data: 'AAECAw==', - media_type: 'image/png', + prompt: { + messages: [ + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'image-gen-1', + is_error: undefined, + content: [ + { type: 'text', text: 'Image generated successfully' }, + { + type: 'image', + source: { + type: 'base64', + data: 'AAECAw==', + media_type: 'image/png', + }, }, - }, - ], - }, - ], - }, - ], - system: undefined, + ], + }, + ], + }, + ], + }, + betas: new Set(), }); }); }); @@ -272,17 +372,19 @@ describe('assistant messages', () => { }); expect(result).toEqual({ - messages: [ - { - role: 'user', - content: [{ type: 'text', text: 'user content' }], - }, - { - role: 'assistant', - content: [{ type: 'text', text: 'assistant content' }], - }, - ], - system: undefined, + prompt: { + messages: [ + { + role: 'user', + content: [{ type: 'text', text: 'user content' }], + }, + { + role: 'assistant', + content: [{ type: 'text', text: 'assistant content' }], + }, + ], + }, + betas: new Set(), }); }); @@ -306,21 +408,23 @@ describe('assistant messages', () => { }); expect(result).toEqual({ - messages: [ - { - role: 'user', - content: [{ type: 'text', text: 'user content' }], - }, - { - role: 'assistant', - content: [{ type: 'text', text: 'assistant content ' }], - }, - { - role: 'user', - content: [{ type: 'text', text: 'user content 2' }], - }, - ], - system: undefined, + prompt: { + messages: [ + { + role: 'user', + content: [{ type: 'text', text: 'user content' }], + }, + { + role: 'assistant', + content: [{ type: 'text', text: 'assistant content ' }], + }, + { + role: 'user', + content: [{ type: 'text', text: 'user content 2' }], + }, + ], + }, + betas: new Set(), }); }); @@ -336,18 +440,20 @@ describe('assistant messages', () => { }); expect(result).toEqual({ - messages: [ - { role: 'user', content: [{ type: 'text', text: 'Hi!' }] }, - { - role: 'assistant', - content: [ - { type: 'text', text: 'Hello' }, - { type: 'text', text: 'World' }, - { type: 'text', text: '!' }, - ], - }, - ], - system: undefined, + prompt: { + messages: [ + { role: 'user', content: [{ type: 'text', text: 'Hi!' }] }, + { + role: 'assistant', + content: [ + { type: 'text', text: 'Hello' }, + { type: 'text', text: 'World' }, + { type: 'text', text: '!' }, + ], + }, + ], + }, + betas: new Set(), }); }); }); @@ -369,14 +475,17 @@ describe('cache control', () => { }); expect(result).toEqual({ - messages: [], - system: [ - { - type: 'text', - text: 'system message', - cache_control: { type: 'ephemeral' }, - }, - ], + prompt: { + messages: [], + system: [ + { + type: 'text', + text: 'system message', + cache_control: { type: 'ephemeral' }, + }, + ], + }, + betas: new Set(), }); }); }); @@ -404,19 +513,21 @@ describe('cache control', () => { }); expect(result).toEqual({ - messages: [ - { - role: 'user', - content: [ - { - type: 'text', - text: 'test', - cache_control: { type: 'ephemeral' }, - }, - ], - }, - ], - system: undefined, + prompt: { + messages: [ + { + role: 'user', + content: [ + { + type: 'text', + text: 'test', + cache_control: { type: 'ephemeral' }, + }, + ], + }, + ], + }, + betas: new Set(), }); }); @@ -440,24 +551,26 @@ describe('cache control', () => { }); expect(result).toEqual({ - messages: [ - { - role: 'user', - content: [ - { - type: 'text', - text: 'part1', - cache_control: undefined, - }, - { - type: 'text', - text: 'part2', - cache_control: { type: 'ephemeral' }, - }, - ], - }, - ], - system: undefined, + prompt: { + messages: [ + { + role: 'user', + content: [ + { + type: 'text', + text: 'part1', + cache_control: undefined, + }, + { + type: 'text', + text: 'part2', + cache_control: { type: 'ephemeral' }, + }, + ], + }, + ], + }, + betas: new Set(), }); }); }); @@ -486,20 +599,22 @@ describe('cache control', () => { }); expect(result).toEqual({ - messages: [ - { role: 'user', content: [{ type: 'text', text: 'user-content' }] }, - { - role: 'assistant', - content: [ - { - type: 'text', - text: 'test', - cache_control: { type: 'ephemeral' }, - }, - ], - }, - ], - system: undefined, + prompt: { + messages: [ + { role: 'user', content: [{ type: 'text', text: 'user-content' }] }, + { + role: 'assistant', + content: [ + { + type: 'text', + text: 'test', + cache_control: { type: 'ephemeral' }, + }, + ], + }, + ], + }, + betas: new Set(), }); }); @@ -528,22 +643,24 @@ describe('cache control', () => { }); expect(result).toEqual({ - messages: [ - { role: 'user', content: [{ type: 'text', text: 'user-content' }] }, - { - role: 'assistant', - content: [ - { - type: 'tool_use', - name: 'test-tool', - id: 'test-id', - input: { some: 'arg' }, - cache_control: { type: 'ephemeral' }, - }, - ], - }, - ], - system: undefined, + prompt: { + messages: [ + { role: 'user', content: [{ type: 'text', text: 'user-content' }] }, + { + role: 'assistant', + content: [ + { + type: 'tool_use', + name: 'test-tool', + id: 'test-id', + input: { some: 'arg' }, + cache_control: { type: 'ephemeral' }, + }, + ], + }, + ], + }, + betas: new Set(), }); }); @@ -568,25 +685,27 @@ describe('cache control', () => { }); expect(result).toEqual({ - messages: [ - { role: 'user', content: [{ type: 'text', text: 'user-content' }] }, - { - role: 'assistant', - content: [ - { - type: 'text', - text: 'part1', - cache_control: undefined, - }, - { - type: 'text', - text: 'part2', - cache_control: { type: 'ephemeral' }, - }, - ], - }, - ], - system: undefined, + prompt: { + messages: [ + { role: 'user', content: [{ type: 'text', text: 'user-content' }] }, + { + role: 'assistant', + content: [ + { + type: 'text', + text: 'part1', + cache_control: undefined, + }, + { + type: 'text', + text: 'part2', + cache_control: { type: 'ephemeral' }, + }, + ], + }, + ], + }, + betas: new Set(), }); }); }); @@ -616,21 +735,23 @@ describe('cache control', () => { }); expect(result).toEqual({ - messages: [ - { - role: 'user', - content: [ - { - type: 'tool_result', - content: '{"test":"test"}', - is_error: undefined, - tool_use_id: 'test', - cache_control: { type: 'ephemeral' }, - }, - ], - }, - ], - system: undefined, + prompt: { + messages: [ + { + role: 'user', + content: [ + { + type: 'tool_result', + content: '{"test":"test"}', + is_error: undefined, + tool_use_id: 'test', + cache_control: { type: 'ephemeral' }, + }, + ], + }, + ], + }, + betas: new Set(), }); }); @@ -664,28 +785,30 @@ describe('cache control', () => { }); expect(result).toEqual({ - messages: [ - { - role: 'user', - content: [ - { - type: 'tool_result', - tool_use_id: 'part1', - content: '{"test":"part1"}', - is_error: undefined, - cache_control: undefined, - }, - { - type: 'tool_result', - tool_use_id: 'part2', - content: '{"test":"part2"}', - is_error: undefined, - cache_control: { type: 'ephemeral' }, - }, - ], - }, - ], - system: undefined, + prompt: { + messages: [ + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'part1', + content: '{"test":"part1"}', + is_error: undefined, + cache_control: undefined, + }, + { + type: 'tool_result', + tool_use_id: 'part2', + content: '{"test":"part2"}', + is_error: undefined, + cache_control: { type: 'ephemeral' }, + }, + ], + }, + ], + }, + betas: new Set(), }); }); }); @@ -713,19 +836,21 @@ describe('cache control', () => { }); expect(result).toEqual({ - messages: [ - { - role: 'user', - content: [ - { - type: 'text', - text: 'test', - cache_control: undefined, - }, - ], - }, - ], - system: undefined, + prompt: { + messages: [ + { + role: 'user', + content: [ + { + type: 'text', + text: 'test', + cache_control: undefined, + }, + ], + }, + ], + }, + betas: new Set(), }); }); }); diff --git a/packages/anthropic/src/convert-to-anthropic-messages-prompt.ts b/packages/anthropic/src/convert-to-anthropic-messages-prompt.ts index ee6c9bf15f03..5a3f8b5dd315 100644 --- a/packages/anthropic/src/convert-to-anthropic-messages-prompt.ts +++ b/packages/anthropic/src/convert-to-anthropic-messages-prompt.ts @@ -18,7 +18,11 @@ export function convertToAnthropicMessagesPrompt({ }: { prompt: LanguageModelV1Prompt; cacheControl: boolean; -}): AnthropicMessagesPrompt { +}): { + prompt: AnthropicMessagesPrompt; + betas: Set; +} { + const betas = new Set(); const blocks = groupIntoBlocks(prompt); let system: AnthropicMessagesPrompt['system'] = undefined; @@ -95,6 +99,7 @@ export function convertToAnthropicMessagesPrompt({ }); break; } + case 'image': { if (part.image instanceof URL) { // The AI SDK automatically downloads images for user image parts with URLs @@ -115,6 +120,35 @@ export function convertToAnthropicMessagesPrompt({ break; } + + case 'file': { + if (part.data instanceof URL) { + // The AI SDK automatically downloads files for user file parts with URLs + throw new UnsupportedFunctionalityError({ + functionality: 'Image URLs in user messages', + }); + } + + if (part.mimeType !== 'application/pdf') { + throw new UnsupportedFunctionalityError({ + functionality: 'Non-PDF files in user messages', + }); + } + + betas.add('pdfs-2024-09-25'); + + anthropicContent.push({ + type: 'document', + source: { + type: 'base64', + media_type: 'application/pdf', + data: part.data, + }, + cache_control: cacheControl, + }); + + break; + } } } @@ -247,8 +281,8 @@ export function convertToAnthropicMessagesPrompt({ } return { - system, - messages, + prompt: { system, messages }, + betas, }; }