diff --git a/pingpong/ai.py b/pingpong/ai.py index e91bafeab..23e40babd 100644 --- a/pingpong/ai.py +++ b/pingpong/ai.py @@ -743,7 +743,8 @@ async def build_response_input_item_list( case WebSearchActionType.FIND: action = ActionFind( type="find", - query=action_rec.query or "", + pattern=action_rec.pattern or "", + url=action_rec.url or "", ) case _: action = None @@ -1346,6 +1347,33 @@ async def add_cached_message_part_on_output_text_url_citation_added( await add_cached_message_part_on_output_text_url_citation_added() + self.enqueue( + { + "type": "message_delta", + "delta": { + "content": [ + { + "index": 0, + "type": "text", + "text": { + "value": "", + "annotations": [ + { + "type": "url_citation", + "end_index": data["end_index"], + "start_index": data["start_index"], + "url": data["url"], + "title": data["title"], + } + ], + }, + }, + ], + "role": None, + }, + } + ) + async def on_output_text_part_done(self, data: ResponseOutputText): if not self.message_part_id: logger.exception( @@ -1715,6 +1743,35 @@ async def add_cached_tool_call_on_code_interpreter_tool_call_done( self.tool_call_id = None self.tool_call_external_id = None + def get_action_payload( + self, + action: ActionSearch | ActionFind | ActionOpenPage | None, + ): + if not action: + return None + + match action.type: + case "search": + return { + "type": WebSearchActionType.SEARCH.value, + "query": action.query, + "sources": [{"url": source.url} for source in action.sources or []], + } + case "find": + return { + "type": WebSearchActionType.FIND.value, + "pattern": action.pattern, + "url": action.url, + } + case "open_page": + return { + "type": WebSearchActionType.OPEN_PAGE.value, + "url": action.url, + } + case _: + return None + return None + async def on_web_search_call_created(self, data: ResponseFunctionWebSearch): if not self.run_id: logger.exception( @@ -1751,6 +1808,21 @@ async def add_cached_tool_call_on_web_search_call_created( self.tool_call_id = tool_call.id self.tool_call_external_id = tool_call.tool_call_id + self.enqueue( + { + "type": "tool_call_created", + "tool_call": { + "id": str(data.id), + "index": self.prev_output_index, + "output_index": self.prev_output_index, + "type": "web_search", + "web_search": { + "action": self.get_action_payload(data.action), + }, + }, + } + ) + async def on_web_search_call_in_progress( self, data: ResponseWebSearchCallInProgressEvent ): @@ -1933,6 +2005,20 @@ async def add_cached_tool_call_on_web_search_call_done( ) await session_.commit() + self.enqueue( + { + "type": "tool_call_delta", + "delta": { + "type": "web_search", + "id": data.id, + "index": self.prev_output_index, + "run_id": str(self.run_id), + "status": data.status, + "action": self.get_action_payload(data.action), + }, + } + ) + await add_cached_tool_call_on_web_search_call_done() self.tool_call_id = None self.tool_call_external_id = None diff --git a/pingpong/schemas.py b/pingpong/schemas.py index 580a66411..26eee9a97 100644 --- a/pingpong/schemas.py +++ b/pingpong/schemas.py @@ -1,10 +1,26 @@ from datetime import date, datetime from enum import Enum, StrEnum, auto from typing import Generic, Literal, NotRequired, TypeVar, Union -from typing_extensions import TypedDict - +from typing_extensions import TypedDict, Annotated, TypeAlias + +from openai._utils import PropertyInfo +from openai.types.beta.threads import ( + ImageFileContentBlock, + TextContentBlock, + RefusalContentBlock, + ImageURLContentBlock, +) +from openai.types.beta.threads.text import Text as OpenAIText +from openai.types.beta.threads.annotation import ( + FileCitationAnnotation, + FilePathAnnotation, +) from openai.types.beta.assistant_tool import AssistantTool as Tool from openai.types.beta.threads import Message as OpenAIMessage +from openai.types.responses.response_output_text import AnnotationURLCitation +from openai.types.responses.response_function_web_search import ( + Action as WebSearchAction, +) from pydantic import ( BaseModel, Field, @@ -1422,6 +1438,33 @@ class FileSearchMessage(BaseModel): output_index: int | None = None +class WebSearchActionType(StrEnum): + SEARCH = "search" + FIND = "find" + OPEN_PAGE = "open_page" + + +class WebSearchCall(BaseModel): + step_id: str + type: Literal["web_search_call"] + action: WebSearchAction | None = None + status: Literal["in_progress", "searching", "completed", "incomplete", "failed"] + + +class WebSearchMessage(BaseModel): + id: str + assistant_id: str + created_at: float + content: list[WebSearchCall] + metadata: dict[str, str] + object: Literal["thread.message"] + message_type: Literal["web_search_call"] + role: Literal["assistant"] + run_id: str + thread_id: str + output_index: int | None = None + + class CodeInterpreterMessage(BaseModel): id: str assistant_id: str @@ -1439,7 +1482,7 @@ class CodeInterpreterMessage(BaseModel): class CodeInterpreterMessages(BaseModel): - ci_messages: list[CodeInterpreterMessage] + ci_messages: list[CodeInterpreterMessage] = [] class ThreadRun(BaseModel): @@ -1455,6 +1498,31 @@ class ThreadParticipants(BaseModel): assistant: dict[int, str] +ThreadAnnotation: TypeAlias = Annotated[ + Union[FileCitationAnnotation, FilePathAnnotation, AnnotationURLCitation], + PropertyInfo(discriminator="type"), +] + + +class ThreadText(OpenAIText): + annotations: list[ThreadAnnotation] + + +class ThreadTextContentBlock(TextContentBlock): + text: ThreadText + + +ThreadMessageContent: TypeAlias = Annotated[ + Union[ + ImageFileContentBlock, + ImageURLContentBlock, + ThreadTextContentBlock, + RefusalContentBlock, + ], + PropertyInfo(discriminator="type"), +] + + class ThreadMessage(OpenAIMessage): status: Literal["in_progress", "incomplete", "completed"] | None """ @@ -1472,6 +1540,9 @@ class ThreadMessage(OpenAIMessage): output_index: int | None = None """The output index of the message, if applicable for Next-Gen Assistants.""" + content: list[ThreadMessageContent] + """The content of the message in array of text and/or images.""" + metadata: dict[str, str | bool] | None = None """Set of 16 key-value pairs that can be attached to an object. @@ -1489,9 +1560,10 @@ class ThreadMessage(OpenAIMessage): class ThreadMessages(BaseModel): limit: int messages: list[ThreadMessage] - fs_messages: list[FileSearchMessage] | None = None - ci_messages: list[CodeInterpreterMessage] | None - reasoning_messages: list["ReasoningMessage"] | None = None + fs_messages: list[FileSearchMessage] = [] + ci_messages: list[CodeInterpreterMessage] = [] + ws_messages: list[WebSearchMessage] = [] + reasoning_messages: list["ReasoningMessage"] = [] has_more: bool @@ -1512,6 +1584,7 @@ class ThreadWithMeta(BaseModel): limit: int ci_messages: list[CodeInterpreterMessage] | None fs_messages: list[FileSearchMessage] | None = None + ws_messages: list[WebSearchMessage] | None = None reasoning_messages: list["ReasoningMessage"] | None = None attachments: dict[str, File] | None instructions: str | None @@ -1810,12 +1883,6 @@ class ToolCallStatus(StrEnum): FAILED = "failed" -class WebSearchActionType(StrEnum): - SEARCH = "search" - FIND = "find" - OPEN_PAGE = "open_page" - - class MessagePartType(StrEnum): INPUT_TEXT = "input_text" INPUT_IMAGE = "input_image" diff --git a/pingpong/server.py b/pingpong/server.py index f224a4440..31d09450e 100644 --- a/pingpong/server.py +++ b/pingpong/server.py @@ -26,13 +26,18 @@ ) from fastapi.responses import JSONResponse, RedirectResponse, StreamingResponse from pydantic import PositiveInt +from openai.types.responses.response_output_text import AnnotationURLCitation +from openai.types.responses.response_function_web_search import ( + ActionFind, + ActionOpenPage, + ActionSearch, + ActionSearchSource, +) from openai.types.beta.threads.message import Attachment -from openai.types.beta.threads.text_content_block import TextContentBlock from openai.types.beta.threads.image_file_content_block import ImageFileContentBlock from openai.types.beta.threads.image_file import ImageFile from openai.types.beta.threads.annotation import Annotation from openai.types.beta.threads.file_path_annotation import FilePathAnnotation, FilePath -from openai.types.beta.threads.text import Text from openai.types.beta.threads.file_citation_annotation import ( FileCitationAnnotation, FileCitation, @@ -2438,6 +2443,8 @@ async def get_thread( "messages": list(messages.data), "limit": 20, "ci_messages": placeholder_ci_calls, + "fs_messages": [], + "ws_messages": [], "reasoning_messages": [], "attachments": all_files, "instructions": thread.instructions if can_view_prompt else None, @@ -2506,6 +2513,7 @@ async def get_thread( placeholder_ci_calls = [] file_search_calls: list[schemas.FileSearchMessage] = [] file_search_results: dict[str, schemas.FileSearchToolAnnotationResult] = {} + web_search_calls: list[schemas.WebSearchMessage] = [] reasoning_messages: list[schemas.ReasoningMessage] = [] for tool_call in tool_calls_v3: if tool_call.type == schemas.ToolCallType.CODE_INTERPRETER: @@ -2585,6 +2593,69 @@ async def get_thread( output_index=tool_call.output_index, ) ) + elif tool_call.type == schemas.ToolCallType.WEB_SEARCH: + action = ( + tool_call.web_search_actions[0] + if tool_call.web_search_actions + else None + ) + + if not action or not action.type: + action_obj = None + else: + match action.type: + case schemas.WebSearchActionType.SEARCH: + sources = ( + [ + ActionSearchSource(url=source.url or "", type="url") + for source in action.sources + ] + if action and action.sources + else [] + ) + action_obj = ActionSearch( + query=action.query or "", + type="search", + sources=sources, + ) + case schemas.WebSearchActionType.FIND: + action_obj = ActionFind( + url=action.url or "", + pattern=action.pattern or "", + type="find", + ) + case schemas.WebSearchActionType.OPEN_PAGE: + action_obj = ActionOpenPage( + url=action.url or "", + type="open_page", + ) + case _: + action_obj = None + + web_search_calls.append( + schemas.WebSearchMessage( + id=str(tool_call.id), + assistant_id=str(thread.assistant_id) + if thread.assistant_id + else "", + created_at=tool_call.created.timestamp(), + content=[ + schemas.WebSearchCall( + step_id=str(tool_call.id), + type="web_search_call", + status=tool_call.status.value, + action=action_obj, + ) + ], + metadata={}, + object="thread.message", + role="assistant", + run_id=str(tool_call.run_id), + thread_id=str(thread.id), + output_index=tool_call.output_index, + message_type="web_search_call", + ) + ) for reasoning_step in reasoning_steps_v3: reasoning_messages.append( @@ -2673,8 +2744,10 @@ async def get_thread( match content.type: case schemas.MessagePartType.INPUT_TEXT: _message.content.append( - TextContentBlock( - text=Text(value=content.text, annotations=[]), + schemas.ThreadTextContentBlock( + text=schemas.ThreadText( + value=content.text, annotations=[] + ), type="text", ) ) @@ -2688,13 +2761,14 @@ async def get_thread( ) ) case schemas.MessagePartType.OUTPUT_TEXT: - _annotations: list[Annotation] = [] + _annotations: list[schemas.ThreadAnnotation] = [] _file_ids_file_citation_annotation: set[str] = set() - if content.annotations and show_file_search_document_names: + if content.annotations: for annotation in content.annotations: if ( annotation.type == schemas.AnnotationType.FILE_CITATION + and show_file_search_document_names ): _file_record = file_search_results.get( annotation.file_id @@ -2757,11 +2831,25 @@ async def get_thread( type="image_file", ), ) + elif ( + annotation.type + == schemas.AnnotationType.URL_CITATION + ): + _annotations.append( + AnnotationURLCitation( + type="url_citation", + end_index=annotation.end_index or 0, + start_index=annotation.start_index or 0, + url=annotation.url or "", + title=annotation.title or "", + text=annotation.text or "", + ) + ) _message.content.append( - TextContentBlock( + schemas.ThreadTextContentBlock( type="text", - text=Text( + text=schemas.ThreadText( value=content.text, annotations=_annotations, ), @@ -2845,6 +2933,7 @@ async def get_thread( "limit": 20, "ci_messages": placeholder_ci_calls, "fs_messages": file_search_calls, + "ws_messages": web_search_calls, "reasoning_messages": reasoning_messages, "attachments": all_files, "instructions": thread.instructions if can_view_prompt else None, @@ -3245,7 +3334,7 @@ async def list_thread_messages( "messages": list(messages.data), "ci_messages": placeholder_ci_calls, "fs_messages": [], - "reasoning_messages": [], + "ws_messages": [], "limit": limit, "has_more": messages.has_more, } @@ -3274,7 +3363,7 @@ async def list_thread_messages( "messages": [], "ci_messages": [], "fs_messages": [], - "reasoning_messages": [], + "ws_messages": [], "limit": limit, "has_more": False, } @@ -3345,6 +3434,7 @@ async def get_assistant( file_search_calls: list[schemas.FileSearchMessage] = [] file_search_results: dict[str, schemas.FileSearchToolAnnotationResult] = {} reasoning_messages: list[schemas.ReasoningMessage] = [] + web_search_calls: list[schemas.WebSearchMessage] = [] for tool_call in tool_calls_v3: if tool_call.type == schemas.ToolCallType.CODE_INTERPRETER: tool_content: list[schemas.CodeInterpreterMessageContent] = [] @@ -3425,6 +3515,69 @@ async def get_assistant( output_index=tool_call.output_index, ) ) + elif tool_call.type == schemas.ToolCallType.WEB_SEARCH: + action = ( + tool_call.web_search_actions[0] + if tool_call.web_search_actions + else None + ) + + if not action or not action.type: + action_obj = None + else: + match action.type: + case schemas.WebSearchActionType.SEARCH: + sources = ( + [ + ActionSearchSource(url=source.url or "", type="url") + for source in action.sources + ] + if action and action.sources + else [] + ) + action_obj = ActionSearch( + query=action.query or "", + type="search", + sources=sources, + ) + case schemas.WebSearchActionType.FIND: + action_obj = ActionFind( + url=action.url or "", + pattern=action.pattern or "", + type="find", + ) + case schemas.WebSearchActionType.OPEN_PAGE: + action_obj = ActionOpenPage( + url=action.url or "", + type="open_page", + ) + case _: + action_obj = None + + web_search_calls.append( + schemas.WebSearchMessage( + id=str(tool_call.id), + assistant_id=str(thread.assistant_id) + if thread.assistant_id + else "", + created_at=tool_call.created.timestamp(), + content=[ + schemas.WebSearchCall( + step_id=str(tool_call.id), + type="web_search_call", + status=tool_call.status.value, + action=action_obj, + ) + ], + metadata={}, + object="thread.message", + role="assistant", + run_id=str(tool_call.run_id), + thread_id=str(thread.id), + output_index=tool_call.output_index, + message_type="web_search_call", + ) + ) for reasoning_step in reasoning_steps_v3: reasoning_messages.append( @@ -3512,8 +3665,10 @@ async def get_assistant( match content.type: case schemas.MessagePartType.INPUT_TEXT: _message.content.append( - TextContentBlock( - text=Text(value=content.text, annotations=[]), + schemas.ThreadTextContentBlock( + text=schemas.ThreadText( + value=content.text, annotations=[] + ), type="text", ) ) @@ -3529,11 +3684,12 @@ async def get_assistant( case schemas.MessagePartType.OUTPUT_TEXT: _annotations: list[Annotation] = [] _file_ids_file_citation_annotation: set[str] = set() - if content.annotations and show_file_search_document_names: + if content.annotations: for annotation in content.annotations: if ( annotation.type == schemas.AnnotationType.FILE_CITATION + and show_file_search_document_names ): _file_record = file_search_results.get( annotation.file_id @@ -3596,11 +3752,25 @@ async def get_assistant( type="image_file", ), ) + elif ( + annotation.type + == schemas.AnnotationType.URL_CITATION + ): + _annotations.append( + AnnotationURLCitation( + type="url_citation", + end_index=annotation.end_index or 0, + start_index=annotation.start_index or 0, + url=annotation.url or "", + title=annotation.title or "", + text=annotation.text or "", + ) + ) _message.content.append( - TextContentBlock( + schemas.ThreadTextContentBlock( type="text", - text=Text( + text=schemas.ThreadText( value=content.text, annotations=_annotations, ), @@ -3636,6 +3806,7 @@ async def get_assistant( "messages": thread_messages, "ci_messages": [], "fs_messages": file_search_calls, + "ws_messages": web_search_calls, "reasoning_messages": reasoning_messages, "limit": limit, "has_more": has_more_runs, diff --git a/web/pingpong/package.json b/web/pingpong/package.json index 2aa899f63..6d89b7f1b 100644 --- a/web/pingpong/package.json +++ b/web/pingpong/package.json @@ -54,7 +54,8 @@ "katex": "^0.16.21", "marked": "^11.2.0", "marked-highlight": "^2.1.4", - "svelte-copy": "^1.4.2" + "svelte-copy": "^1.4.2", + "tldts": "^7.0.18" }, "pnpm": { "overrides": { diff --git a/web/pingpong/pnpm-lock.yaml b/web/pingpong/pnpm-lock.yaml index c8928fb78..b99567429 100644 --- a/web/pingpong/pnpm-lock.yaml +++ b/web/pingpong/pnpm-lock.yaml @@ -47,6 +47,9 @@ importers: svelte-copy: specifier: ^1.4.2 version: 1.4.2(svelte@4.2.19) + tldts: + specifier: ^7.0.18 + version: 7.0.18 devDependencies: '@sveltejs/adapter-auto': specifier: ^3.2.4 @@ -2212,6 +2215,13 @@ packages: resolution: {integrity: sha512-KYad6Vy5VDWV4GH3fjpseMQ/XU2BhIYP7Vzd0LG44qRWm/Yt2WCOTicFdvmgo6gWaqooMQCawTtILVQJupKu7A==} engines: {node: '>=14.0.0'} + tldts-core@7.0.18: + resolution: {integrity: sha512-jqJC13oP4FFAahv4JT/0WTDrCF9Okv7lpKtOZUGPLiAnNbACcSg8Y8T+Z9xthOmRBqi/Sob4yi0TE0miRCvF7Q==} + + tldts@7.0.18: + resolution: {integrity: sha512-lCcgTAgMxQ1JKOWrVGo6E69Ukbnx4Gc1wiYLRf6J5NN4HRYJtCby1rPF8rkQ4a6qqoFBK5dvjJ1zJ0F7VfDSvw==} + hasBin: true + to-fast-properties@2.0.0: resolution: {integrity: sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==} engines: {node: '>=4'} @@ -4506,6 +4516,12 @@ snapshots: tinyspy@2.2.1: {} + tldts-core@7.0.18: {} + + tldts@7.0.18: + dependencies: + tldts-core: 7.0.18 + to-fast-properties@2.0.0: {} to-regex-range@5.0.1: diff --git a/web/pingpong/src/lib/api.ts b/web/pingpong/src/lib/api.ts index 1305e3814..4add76cc1 100644 --- a/web/pingpong/src/lib/api.ts +++ b/web/pingpong/src/lib/api.ts @@ -2139,7 +2139,18 @@ export type TextAnnotationFileCitation = { type: 'file_citation'; }; -export type TextAnnotation = TextAnnotationFilePath | TextAnnotationFileCitation; +export type TextAnnotationURLCitation = { + end_index: number; + start_index: number; + title: string; + type: 'url_citation'; + url: string; +}; + +export type TextAnnotation = + | TextAnnotationFilePath + | TextAnnotationFileCitation + | TextAnnotationURLCitation; export type Text = { annotations: TextAnnotation[]; @@ -2193,6 +2204,19 @@ export type FileSearchCallItem = { status?: 'in_progress' | 'searching' | 'completed' | 'incomplete' | 'failed'; }; +export type WebSearchSource = { + url?: string | null; + title?: string | null; + type: 'url'; +}; + +export type WebSearchCallItem = { + step_id: string; + type: 'web_search_call'; + action?: WebSearchAction | null; + status: 'in_progress' | 'completed' | 'incomplete' | 'searching' | 'failed'; +}; + export type ReasoningSummaryPart = { id?: number; part_index: number; @@ -2216,6 +2240,7 @@ export type Content = | MessageContentCodeOutputLogs | CodeInterpreterCallPlaceholder | FileSearchCallItem + | WebSearchCallItem | ReasoningCallItem; export type OpenAIMessage = { @@ -2255,6 +2280,7 @@ export type ThreadWithMeta = { messages: OpenAIMessage[]; ci_messages: OpenAIMessage[]; fs_messages: OpenAIMessage[]; + ws_messages: OpenAIMessage[]; reasoning_messages: OpenAIMessage[]; attachments: Record; instructions: string | null; @@ -2325,6 +2351,7 @@ export type ThreadMessages = { messages: OpenAIMessage[]; ci_messages: OpenAIMessage[]; fs_messages: OpenAIMessage[]; + ws_messages: OpenAIMessage[]; reasoning_messages: OpenAIMessage[]; limit: number; has_more: boolean; @@ -2347,6 +2374,7 @@ export const getThreadMessages = async ( limit: null, messages: [], fs_messages: [], + ws_messages: [], ci_messages: [], reasoning_messages: [], has_more: false, @@ -2360,6 +2388,7 @@ export const getThreadMessages = async ( messages: expanded.data.messages, ci_messages: expanded.data.ci_messages, fs_messages: expanded.data.fs_messages, + ws_messages: expanded.data.ws_messages, reasoning_messages: expanded.data.reasoning_messages, limit: expanded.data.limit, has_more: hasMore, @@ -2440,6 +2469,40 @@ export type FileSearchCall = { status: 'in_progress' | 'searching' | 'completed' | 'incomplete' | 'failed'; }; +export type WebSearchActionSearchSource = { + url: string; + type: 'url'; +}; + +export type WebSearchActionSearch = { + type: 'search'; + query: string; + sources: WebSearchActionSearchSource[]; +}; + +export type WebSearchActionOpenPage = { + type: 'open_page'; + url: string; +}; + +export type WebSearchActionFind = { + type: 'find'; + pattern: string; + url: string; +}; + +export type WebSearchAction = WebSearchActionSearch | WebSearchActionOpenPage | WebSearchActionFind; + +export type WebSearchCall = { + type: 'web_search'; + id: string; + index: number; + output_index?: number; + run_id: string | null; + action: WebSearchAction; + status: 'in_progress' | 'completed' | 'incomplete' | 'failed' | 'searching'; +}; + export type ReasoningStepSummaryPartChunk = { reasoning_step_id: number; part_index: number; @@ -2458,7 +2521,7 @@ export type ReasoningCall = { }; // TODO(jnu): support function calling, updates for v2 -export type ToolCallDelta = CodeInterpreterCall | FileSearchCall; +export type ToolCallDelta = CodeInterpreterCall | FileSearchCall | WebSearchCall; export type ThreadStreamToolCallCreatedChunk = { type: 'tool_call_created'; diff --git a/web/pingpong/src/lib/components/FileSearchCallItem.svelte b/web/pingpong/src/lib/components/FileSearchCallItem.svelte index 312d8b5d4..16c8dcb0f 100644 --- a/web/pingpong/src/lib/components/FileSearchCallItem.svelte +++ b/web/pingpong/src/lib/components/FileSearchCallItem.svelte @@ -50,6 +50,10 @@
{#if content.status === 'completed'} Searched files + {:else if content.status === 'failed'} + File search failed + {:else if content.status === 'incomplete'} + File search was canceled {:else} Searching files... {/if} diff --git a/web/pingpong/src/lib/components/Markdown.svelte b/web/pingpong/src/lib/components/Markdown.svelte index 1ce7ea5ec..353f7f121 100644 --- a/web/pingpong/src/lib/components/Markdown.svelte +++ b/web/pingpong/src/lib/components/Markdown.svelte @@ -1,13 +1,69 @@ -
+
diff --git a/web/pingpong/src/lib/components/ThreadDetailPage.svelte b/web/pingpong/src/lib/components/ThreadDetailPage.svelte index 8c06aa0c8..eb3553b41 100644 --- a/web/pingpong/src/lib/components/ThreadDetailPage.svelte +++ b/web/pingpong/src/lib/components/ThreadDetailPage.svelte @@ -59,6 +59,7 @@ import StatusErrors from './StatusErrors.svelte'; import FileSearchCallItem from './FileSearchCallItem.svelte'; import ReasoningCallItem from './ReasoningCallItem.svelte'; + import WebSearchCallItem from './WebSearchCallItem.svelte'; export let data; let userTimezone = Intl.DateTimeFormat().resolvedOptions().timeZone; @@ -942,14 +943,16 @@ {@const { clean_string, images } = processString(content.text.value)} {@const imageInfo = convertImageProxyToInfo(images)} {@const quoteCitations = (content.text.annotations ?? []).filter(isFileCitation)} + {@const parsedTextContent = parseTextContent( + { value: clean_string, annotations: content.text.annotations }, + $version, + api.fullPath(`/class/${classId}/thread/${threadId}`) + )}
@@ -1016,6 +1019,8 @@ > {:else if content.type === 'file_search_call'} + {:else if content.type === 'web_search_call'} + {:else if content.type === 'reasoning'} {:else if content.type === 'code_output_image_file'} diff --git a/web/pingpong/src/lib/components/WebSearchCallItem.svelte b/web/pingpong/src/lib/components/WebSearchCallItem.svelte new file mode 100644 index 000000000..4129681df --- /dev/null +++ b/web/pingpong/src/lib/components/WebSearchCallItem.svelte @@ -0,0 +1,179 @@ + + +
+ {#if content.action && content.action.type === 'search' && (uniqueSources.length > 0 || content.action.query)} +
+ + +
+ {#if open} +
+ {#if uniqueSources.length === 0} +
+ No sources found. +
+ {:else} + {#each uniqueSources as source, i (source.url)} +
+ +
+ {/each} + {/if} +
+ {/if} + {:else if content.action && content.action.type} +
+ +
+ {#if content.action.type === 'search'} + {#if content.status === 'completed'} + Searched web + {:else if content.status === 'failed'} + Web search failed + {:else if content.status === 'incomplete'} + Web search was canceled + {:else} + Searching web... + {/if} + {:else if content.action.type === 'find'} + {#if content.status === 'completed'} + Looked closer through{#if content.action.url}{:else}web sources{/if}{#if content.action.pattern} + for {content.action.pattern}{/if} + {:else if content.status === 'failed'} + Web search failed + {:else if content.status === 'incomplete'} + Web search was canceled + {:else} + Digging through web sources... + {/if} + {:else if content.action.type === 'open_page'} + {#if content.status === 'completed'} + {#if content.action.url}Opened{:else}Looked through web sources{/if} + {:else if content.status === 'failed'} + Web search failed + {:else if content.status === 'incomplete'} + Web search was canceled + {:else} + Looking through web sources... + {/if} + {/if} +
+
+ {:else} +
+ +
+ {#if content.status === 'completed'} + Web search completed + {:else if content.status === 'failed'} + Web search failed + {:else if content.status === 'incomplete'} + Web search was canceled + {:else} + Searching web... + {/if} +
+
+ {/if} +
+ + diff --git a/web/pingpong/src/lib/components/WebSourceChip.svelte b/web/pingpong/src/lib/components/WebSourceChip.svelte new file mode 100644 index 000000000..f1d015f9f --- /dev/null +++ b/web/pingpong/src/lib/components/WebSourceChip.svelte @@ -0,0 +1,104 @@ + + + + +
{label}
+ {#if url} +
+ {#if faviconUrl && showFavicon} + Favicon (showFavicon = false)} + /> + {/if} +
{url}
+
+ {/if} + diff --git a/web/pingpong/src/lib/content.ts b/web/pingpong/src/lib/content.ts index 2f40278f4..d58db968b 100644 --- a/web/pingpong/src/lib/content.ts +++ b/web/pingpong/src/lib/content.ts @@ -1,4 +1,4 @@ -import { type Text, join } from '$lib/api'; +import { join, type Text, type WebSearchSource } from '$lib/api'; type Replacement = { start: number; @@ -6,12 +6,28 @@ type Replacement = { newValue: string; }; +export type InlineWebSource = { + index: number; + source: WebSearchSource; +}; + +export type ParsedTextContent = { + content: string; + inlineWebSources: InlineWebSource[]; +}; + /** * Rewrite OpenAI text content to incorporate annotations. */ -export const parseTextContent = (text: Text, threadVersion: number = 2, baseUrl: string = '') => { +export const parseTextContent = ( + text: Text, + threadVersion: number = 2, + baseUrl: string = '' +): ParsedTextContent => { let content = text.value; const replacements: Replacement[] = []; + const inlineWebSources: InlineWebSource[] = []; + let urlCitationIndex = 0; if (text.annotations) { for (const annotation of text.annotations) { if ( @@ -26,6 +42,24 @@ export const parseTextContent = (text: Text, threadVersion: number = 2, baseUrl: const { start_index, end_index, file_citation } = annotation; const fileName = ` (${file_citation.file_name})`; replacements.push({ start: start_index, end: end_index, newValue: fileName }); + } else if (annotation.type === 'url_citation') { + // Drop a placeholder that the Markdown component swaps for an inline WebSourceChip. + inlineWebSources.push({ + index: urlCitationIndex, + source: { + url: annotation.url, + title: annotation.title || undefined, + type: 'url' + } + }); + const needsLeadingSpace = + annotation.start_index > 0 && !/\s/.test(text.value[annotation.start_index - 1]); + replacements.push({ + start: annotation.start_index, + end: annotation.end_index, + newValue: `${needsLeadingSpace ? ' ' : ''}` + }); + urlCitationIndex += 1; } } } @@ -38,7 +72,7 @@ export const parseTextContent = (text: Text, threadVersion: number = 2, baseUrl: content = content.slice(0, start) + newValue + content.slice(end); } - return content; + return { content, inlineWebSources }; }; /** diff --git a/web/pingpong/src/lib/stores/thread.ts b/web/pingpong/src/lib/stores/thread.ts index 061d007e5..575b9ad7f 100644 --- a/web/pingpong/src/lib/stores/thread.ts +++ b/web/pingpong/src/lib/stores/thread.ts @@ -210,6 +210,11 @@ export class ThreadManager { error: null, persisted: true })); + const ws_messages = ($data.data?.ws_messages || []).map((message) => ({ + data: message, + error: null, + persisted: true + })); const reasoning_messages = ($data.data?.reasoning_messages || []).map((message) => ({ data: message, error: null, @@ -224,6 +229,7 @@ export class ThreadManager { const allMessages = realMessages .concat(ci_messages) .concat(fs_messages) + .concat(ws_messages) .concat(reasoning_messages) .concat(optimisticMessages) .sort(compareMessageDataAsc); @@ -522,10 +528,11 @@ export class ThreadManager { ...d, data: { ...d.data, - ci_messages: [...response.ci_messages, ...d.data.ci_messages], - fs_messages: [...response.fs_messages, ...d.data.fs_messages], + ci_messages: [...(response.ci_messages || []), ...d.data.ci_messages], + fs_messages: [...(response.fs_messages || []), ...d.data.fs_messages], + ws_messages: [...(response.ws_messages || []), ...d.data.ws_messages], reasoning_messages: [ - ...response.reasoning_messages, + ...(response.reasoning_messages || []), ...(d.data.reasoning_messages || []) ], messages: [...response.messages, ...d.data.messages].sort(compareApiMessagesAsc) @@ -775,6 +782,7 @@ export class ThreadManager { ...(state.data?.messages ?? []), ...(state.data?.ci_messages ?? []), ...(state.data?.fs_messages ?? []), + ...(state.data?.ws_messages ?? []), ...(state.data?.reasoning_messages ?? []), ...state.optimistic ]; @@ -1107,7 +1115,9 @@ export class ThreadManager { if ( lastMessage.data.role !== 'assistant' && - (call.type === 'code_interpreter' || call.type === 'file_search') + (call.type === 'code_interpreter' || + call.type === 'file_search' || + call.type === 'web_search') ) { const version = get(this.version); const callOutputIndex = @@ -1204,8 +1214,23 @@ export class ThreadManager { queries: chunk.queries || [] }); } - } + } else if (chunk.type === 'web_search') { + const placeholder = lastMessage.content.find( + (c) => c.type === 'web_search_call' && c.step_id === chunk.id + ) as api.WebSearchCallItem | undefined; + if (placeholder) { + placeholder.action = chunk.action || placeholder.action; + placeholder.status = chunk.status; + } else { + lastMessage.content.push({ + type: 'web_search_call', + step_id: chunk.id, + action: chunk.action || null, + status: chunk.status + }); + } + } return { ...d }; }); }