webui: support video files as input

tools/ui/src/app.d.ts

@@ -39,6 +39,7 @@ import type {
 	DatabaseMessage,
 	DatabaseMessageExtra,
 	DatabaseMessageExtraAudioFile,
+	DatabaseMessageExtraVideoFile,
 	DatabaseMessageExtraImageFile,
 	DatabaseMessageExtraTextFile,
 	DatabaseMessageExtraPdfFile,
@@ -102,6 +103,7 @@ declare global {
 		DatabaseMessage,
 		DatabaseMessageExtra,
 		DatabaseMessageExtraAudioFile,
+		DatabaseMessageExtraVideoFile,
 		DatabaseMessageExtraImageFile,
 		DatabaseMessageExtraTextFile,
 		DatabaseMessageExtraPdfFile,

tools/ui/src/lib/components/app/badges/BadgesModality.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-	import { Eye, Mic } from '@lucide/svelte';
+	import { Eye, Mic, Video } from '@lucide/svelte';
 	import { ModelModality } from '$lib/enums';
 
 	interface Props {
@@ -11,7 +11,7 @@
 </script>
 
 {#each modalities as modality (modality)}
-	{#if modality === ModelModality.VISION || modality === ModelModality.AUDIO}
+	{#if modality === ModelModality.VISION || modality === ModelModality.AUDIO || modality === ModelModality.VIDEO}
 		<span
 			class={[
 				'inline-flex items-center gap-1 rounded-md bg-muted px-2 py-1 text-xs font-medium',
@@ -21,7 +21,11 @@
 			{#if modality === ModelModality.VISION}
 				<Eye class="h-3 w-3" />
 
-				Vision
+				Vision (Image)
+			{:else if modality === ModelModality.VIDEO}
+				<Video class="h-3 w-3" />
+
+				Vision (Video)
 			{:else}
 				<Mic class="h-3 w-3" />
 

tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemThumbnailFile.svelte

@@ -1,10 +1,12 @@
 <script lang="ts">
-	import { X } from '@lucide/svelte';
+	import { X, Music, Video } from '@lucide/svelte';
 	import {
 		formatFileSize,
 		getFileTypeLabel,
 		getPreviewText,
 		isPdfFile,
+		isAudioFile,
+		isVideoFile,
 		isTextFile
 	} from '$lib/utils';
 	import { ActionIcon } from '$lib/components/app';
@@ -38,6 +40,8 @@
 	}: Props = $props();
 
 	let isPdf = $derived(isPdfFile(attachment, uploadedFile));
+	let isAudio = $derived(isAudioFile(attachment, uploadedFile));
+	let isVideo = $derived(isVideoFile(attachment, uploadedFile));
 	let isPdfWithContent = $derived(isPdf && !!textContent);
 
 	let isText = $derived(isTextFile(attachment, uploadedFile));
@@ -102,7 +106,13 @@
 	<div
 		class="flex h-8 w-8 items-center justify-center rounded bg-primary/10 text-xs font-medium text-primary"
 	>
-		{fileTypeLabel}
+		{#if isAudio}
+			<Music class="h-4 w-4 text-white/70" />
+		{:else if isVideo}
+			<Video class="h-4 w-4 text-white/70" />
+		{:else}
+			{fileTypeLabel}
+		{/if}
 	</div>
 {/snippet}
 

tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview.svelte

@@ -12,6 +12,7 @@
 		getAttachmentDisplayItems,
 		getLanguageFromFilename,
 		isAudioFile,
+		isVideoFile,
 		isImageFile,
 		isMcpPrompt,
 		isMcpResource,
@@ -29,6 +30,7 @@
 		textContent?: string;
 		isImage: boolean;
 		isAudio: boolean;
+		isVideo: boolean;
 	}
 
 	interface Props {
@@ -54,7 +56,8 @@
 				(item): PreviewItem => ({
 					...item,
 					isImage: isImageFile(item.attachment, item.uploadedFile),
-					isAudio: isAudioFile(item.attachment, item.uploadedFile)
+					isAudio: isAudioFile(item.attachment, item.uploadedFile),
+					isVideo: isVideoFile(item.attachment, item.uploadedFile)
 				})
 			)
 	);
@@ -102,6 +105,9 @@
 	let isAudio = $derived(
 		currentItem ? isAudioFile(currentItem.attachment, currentItem.uploadedFile) : false
 	);
+	let isVideo = $derived(
+		currentItem ? isVideoFile(currentItem.attachment, currentItem.uploadedFile) : false
+	);
 	let isImage = $derived(
 		currentItem ? isImageFile(currentItem.attachment, currentItem.uploadedFile) : false
 	);
@@ -148,6 +154,20 @@
 			: null
 	);
 
+	let videoSrc = $derived(
+		isVideo && currentItem
+			? (currentItem.uploadedFile?.preview ??
+					(currentItem.attachment &&
+					'mimeType' in currentItem.attachment &&
+					'base64Data' in currentItem.attachment
+						? createBase64DataUrl(
+								currentItem.attachment.mimeType,
+								currentItem.attachment.base64Data
+							)
+						: null))
+			: null
+	);
+
 	export function prev() {
 		currentIndex = currentIndex > 0 ? currentIndex - 1 : allItems.length - 1;
 	}
@@ -173,11 +193,13 @@
 					{currentItem}
 					{isImage}
 					{isAudio}
+					{isVideo}
 					{isPdf}
 					{isText}
 					{displayPreview}
 					{displayTextContent}
 					{audioSrc}
+					{videoSrc}
 					{language}
 					{hasVisionModality}
 					{activeModelId}

tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItem.svelte

@@ -1,21 +1,24 @@
 <script lang="ts">
 	import type { ChatAttachmentDisplayItem } from '$lib/types';
-	import { Image, Music, FileText, FileIcon } from '@lucide/svelte';
+	import { Image, Music, Video, FileText, FileIcon } from '@lucide/svelte';
 	import ChatAttachmentsPreviewCurrentItemPdf from './ChatAttachmentsPreviewCurrentItemPdf.svelte';
 	import ChatAttachmentsPreviewCurrentItemImage from './ChatAttachmentsPreviewCurrentItemImage.svelte';
 	import ChatAttachmentsPreviewCurrentItemAudio from './ChatAttachmentsPreviewCurrentItemAudio.svelte';
+	import ChatAttachmentsPreviewCurrentItemVideo from './ChatAttachmentsPreviewCurrentItemVideo.svelte';
 	import ChatAttachmentsPreviewCurrentItemText from './ChatAttachmentsPreviewCurrentItemText.svelte';
 	import ChatAttachmentsPreviewCurrentItemUnavailable from './ChatAttachmentsPreviewCurrentItemUnavailable.svelte';
 
 	interface Props {
 		currentItem: ChatAttachmentDisplayItem | null;
 		isImage: boolean;
 		isAudio: boolean;
+		isVideo: boolean;
 		isPdf: boolean;
 		isText: boolean;
 		displayPreview: string | undefined;
 		displayTextContent: string | undefined;
 		audioSrc: string | null;
+		videoSrc: string | null;
 		language: string;
 		hasVisionModality: boolean;
 		activeModelId?: string;
@@ -25,21 +28,25 @@
 		currentItem,
 		isImage,
 		isAudio,
+		isVideo,
 		isPdf,
 		isText,
 		displayPreview,
 		displayTextContent,
 		audioSrc,
+		videoSrc,
 		language,
 		hasVisionModality,
 		activeModelId
 	}: Props = $props();
 
 	let IconComponent = $derived(
-		isImage ? Image : isText || isPdf ? FileText : isAudio ? Music : FileIcon
+		isImage ? Image : isText || isPdf ? FileText : isAudio ? Music : isVideo ? Video : FileIcon
 	);
 
-	let isUnavailable = $derived(!isPdf && !isImage && !(isText && displayTextContent) && !isAudio);
+	let isUnavailable = $derived(
+		!isPdf && !isImage && !(isText && displayTextContent) && !isAudio && !isVideo
+	);
 </script>
 
 {#if currentItem}
@@ -58,6 +65,8 @@
 			<ChatAttachmentsPreviewCurrentItemText {displayTextContent} {language} />
 		{:else if isAudio}
 			<ChatAttachmentsPreviewCurrentItemAudio {currentItem} {audioSrc} />
+		{:else if isVideo}
+			<ChatAttachmentsPreviewCurrentItemVideo {currentItem} {videoSrc} />
 		{:else if isUnavailable}
 			<ChatAttachmentsPreviewCurrentItemUnavailable {IconComponent} />
 		{/if}

tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemVideo.svelte

@@ -0,0 +1,26 @@
+<script lang="ts">
+	import { Video } from '@lucide/svelte';
+
+	interface Props {
+		currentItem: { name?: string } | null;
+		videoSrc: string | null;
+	}
+
+	let { currentItem, videoSrc }: Props = $props();
+</script>
+
+<div class="flex flex-1 items-center justify-center p-8">
+	<div class="w-full max-w-md text-center">
+		<Video class="mx-auto mb-4 h-16 w-16 text-white/50" />
+
+		{#if videoSrc}
+			<video controls class="mb-4 w-full" src={videoSrc}>
+				Your browser does not support the video element.
+			</video>
+		{:else}
+			<p class="mb-4 text-white/70">Video preview not available</p>
+		{/if}
+
+		<p class="text-sm text-white/50">{currentItem?.name || 'Video'}</p>
+	</div>
+</div>

tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewThumbnailStrip.svelte

@@ -1,12 +1,13 @@
 <script lang="ts">
-	import { Music, FileText } from '@lucide/svelte';
+	import { Music, Video, FileText } from '@lucide/svelte';
 	import { HorizontalScrollCarousel } from '$lib/components/app/misc';
 
 	interface PreviewItem {
 		id: string;
 		name: string;
 		isImage: boolean;
 		isAudio: boolean;
+		isVideo: boolean;
 		preview?: string;
 	}
 
@@ -49,6 +50,8 @@
 						>
 							{#if item.isAudio}
 								<Music class="h-4 w-4 text-white/70" />
+							{:else if item.isVideo}
+								<Video class="h-4 w-4 text-white/70" />
 							{:else}
 								<FileText class="h-4 w-4 text-white/70" />
 							{/if}

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte

@@ -23,6 +23,7 @@
 		class?: string;
 		disabled?: boolean;
 		hasAudioModality?: boolean;
+		hasVideoModality?: boolean;
 		hasVisionModality?: boolean;
 		hasMcpPromptsSupport?: boolean;
 		hasMcpResourcesSupport?: boolean;
@@ -37,6 +38,7 @@
 		class: className = '',
 		disabled = false,
 		hasAudioModality = false,
+		hasVideoModality = false,
 		hasVisionModality = false,
 		hasMcpPromptsSupport = false,
 		hasMcpResourcesSupport = false,
@@ -58,6 +60,7 @@
 		() => ({
 			hasVisionModality,
 			hasAudioModality,
+			hasVideoModality,
 			hasMcpPromptsSupport,
 			hasMcpResourcesSupport
 		}),

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte

@@ -19,6 +19,7 @@
 		class?: string;
 		disabled?: boolean;
 		hasAudioModality?: boolean;
+		hasVideoModality?: boolean;
 		hasVisionModality?: boolean;
 		hasMcpPromptsSupport?: boolean;
 		hasMcpResourcesSupport?: boolean;
@@ -34,6 +35,7 @@
 		disabled = false,
 		hasAudioModality = false,
 		hasVisionModality = false,
+		hasVideoModality = false,
 		hasMcpPromptsSupport = false,
 		hasMcpResourcesSupport = false,
 		onFileUpload,
@@ -49,6 +51,7 @@
 		() => ({
 			hasVisionModality,
 			hasAudioModality,
+			hasVideoModality,
 			hasMcpPromptsSupport,
 			hasMcpResourcesSupport
 		}),

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionsAdd.svelte

@@ -7,6 +7,7 @@
 	interface Props {
 		disabled?: boolean;
 		hasAudioModality?: boolean;
+		hasVideoModality?: boolean;
 		hasMcpPromptsSupport?: boolean;
 		hasMcpResourcesSupport?: boolean;
 		hasVisionModality?: boolean;
@@ -20,6 +21,7 @@
 	let {
 		disabled = false,
 		hasAudioModality = false,
+		hasVideoModality = false,
 		hasMcpPromptsSupport = false,
 		hasMcpResourcesSupport = false,
 		hasVisionModality = false,
@@ -37,6 +39,7 @@
 	<ChatFormActionAddSheet
 		{disabled}
 		{hasAudioModality}
+		{hasVideoModality}
 		{hasVisionModality}
 		{hasMcpPromptsSupport}
 		{hasMcpResourcesSupport}
@@ -52,6 +55,7 @@
 	<ChatFormActionAddDropdown
 		{disabled}
 		{hasAudioModality}
+		{hasVideoModality}
 		{hasVisionModality}
 		{hasMcpPromptsSupport}
 		{hasMcpResourcesSupport}

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte

@@ -11,6 +11,7 @@
 		disabled?: boolean;
 		forceForegroundText?: boolean;
 		hasAudioModality?: boolean;
+		hasVideoModality?: boolean;
 		hasVisionModality?: boolean;
 		hasModelSelected?: boolean;
 		isSelectedModelInCache?: boolean;
@@ -23,6 +24,7 @@
 		disabled = false,
 		forceForegroundText = false,
 		hasAudioModality = $bindable(false),
+		hasVideoModality = $bindable(false),
 		hasVisionModality = $bindable(false),
 		hasModelSelected = $bindable(false),
 		isSelectedModelInCache = $bindable(true),
@@ -95,6 +97,10 @@
 		hasAudioModality = activeModelId ? modelsStore.modelSupportsAudio(activeModelId) : false;
 	});
 
+	$effect(() => {
+		hasVideoModality = activeModelId ? modelsStore.modelSupportsVideo(activeModelId) : false;
+	});
+
 	$effect(() => {
 		void modelPropsVersion;
 

tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte

@@ -66,6 +66,7 @@
 	});
 
 	let hasAudioModality = $state(false);
+	let hasVideoModality = $state(false);
 	let hasVisionModality = $state(false);
 	let hasModelSelected = $state(false);
 	let isSelectedModelInCache = $state(true);
@@ -94,6 +95,7 @@
 			<ChatFormActionsAdd
 				{disabled}
 				{hasAudioModality}
+				{hasVideoModality}
 				{hasVisionModality}
 				{hasMcpPromptsSupport}
 				{hasMcpResourcesSupport}
@@ -111,6 +113,7 @@
 			{disabled}
 			bind:this={selectorModelRef}
 			bind:hasAudioModality
+			bind:hasVideoModality
 			bind:hasVisionModality
 			bind:hasModelSelected
 			bind:isSelectedModelInCache