fix(media): preview host files and stream videos across clients

apps/desktop/src/electron/ElectronProtocol.test.ts

@@ -113,6 +113,71 @@ describe("ElectronProtocol", () => {
     }).pipe(Effect.provide(ElectronProtocol.layer)),
   );
 
+  it.effect("forwards video byte ranges and streams the response without buffering the file", () =>
+    Effect.gen(function* () {
+      let handler: ((request: Request) => Promise<Response>) | undefined;
+      let controller: ReadableStreamDefaultController<Uint8Array> | undefined;
+      handleMock.mockImplementation((_scheme, nextHandler) => {
+        handler = nextHandler;
+      });
+      netFetchMock.mockResolvedValue(
+        new Response(
+          new ReadableStream<Uint8Array>({
+            start(nextController) {
+              controller = nextController;
+              controller.enqueue(new Uint8Array([2, 3]));
+            },
+          }),
+          {
+            status: 206,
+            headers: {
+              "content-type": "video/mp4",
+              "content-range": "bytes 2-5/10",
+              "accept-ranges": "bytes",
+            },
+          },
+        ),
+      );
+
+      yield* Effect.scoped(
+        Effect.gen(function* () {
+          const protocol = yield* ElectronProtocol.ElectronProtocol;
+          yield* protocol.registerDesktopProtocol({
+            scheme: "t3code",
+            targetOrigin: new URL("http://127.0.0.1:3773/"),
+            backendOrigin: new URL("http://127.0.0.1:3773/"),
+            clerkFrontendApiHostname: undefined,
+          });
+          const response = yield* Effect.promise(() =>
+            handler!(
+              new Request("t3code://app/api/assets/signed-token/recording.mp4", {
+                headers: { range: "bytes=2-5" },
+              }),
+            ),
+          );
+          const forwardedHeaders = new Headers(netFetchMock.mock.calls[0]?.[1]?.headers);
+          assert.equal(forwardedHeaders.get("range"), "bytes=2-5");
+          assert.equal(response.status, 206);
+          assert.equal(response.headers.get("content-range"), "bytes 2-5/10");
+          assert.equal(response.headers.get("accept-ranges"), "bytes");
+          assert.equal(response.headers.get("content-type"), "video/mp4");
+          const reader = response.body!.getReader();
+          assert.deepEqual(yield* Effect.promise(() => reader.read()), {
+            done: false,
+            value: new Uint8Array([2, 3]),
+          });
+          controller!.enqueue(new Uint8Array([4, 5]));
+          controller!.close();
+          assert.deepEqual(yield* Effect.promise(() => reader.read()), {
+            done: false,
+            value: new Uint8Array([4, 5]),
+          });
+          assert.isTrue((yield* Effect.promise(() => reader.read())).done);
+        }),
+      );
+    }).pipe(Effect.provide(ElectronProtocol.layer)),
+  );
+
   it.effect("retries transient renderer target failures", () =>
     Effect.gen(function* () {
       let handler: ((request: Request) => Promise<Response>) | undefined;
@@ -225,7 +290,7 @@ describe("ElectronProtocol", () => {
       "http:",
       "https:",
     ]);
-    assert.deepEqual(directives["media-src"], ["'self'", "t3code:", "blob:"]);
+    assert.deepEqual(directives["media-src"], ["'self'", "t3code:", "blob:", "http:", "https:"]);
     assert.deepEqual(directives["font-src"], ["'self'", "t3code:", "data:"]);
   });
 });

apps/desktop/src/electron/ElectronProtocol.ts

@@ -87,7 +87,7 @@ export function makeDesktopContentSecurityPolicy(input: DesktopProtocolRegistrat
     `script-src ${scriptSources.join(" ")}`,
     `connect-src ${connectSources.join(" ")}`,
     `img-src 'self' ${input.scheme}: blob: data: http: https:`,
-    `media-src 'self' ${input.scheme}: blob:`,
+    `media-src 'self' ${input.scheme}: blob: http: https:`,
     "style-src 'self' 'unsafe-inline'",
     `font-src 'self' ${input.scheme}: data:`,
     "worker-src 'self' blob:",
@@ -118,6 +118,7 @@ export function registerDesktopSchemePrivilegesSync(): void {
         secure: true,
         supportFetchAPI: true,
         corsEnabled: true,
+        stream: true,
       },
     },
     {
@@ -127,6 +128,7 @@ export function registerDesktopSchemePrivilegesSync(): void {
         secure: true,
         supportFetchAPI: true,
         corsEnabled: true,
+        stream: true,
       },
     },
   ]);

apps/mobile/modules/t3-markdown-text/package.json

@@ -26,6 +26,7 @@
     "./types": "./src/SelectableMarkdownText.types.ts"
   },
   "peerDependencies": {
+    "@t3tools/client-runtime": "*",
     "expo-asset": "*",
     "expo-clipboard": "*",
     "expo-haptics": "*",

apps/mobile/modules/t3-markdown-text/src/markdownLinks.ts

@@ -1,3 +1,8 @@
+import {
+  inlineCodeFilePathCandidate,
+  isConventionalFilePosition,
+} from "@t3tools/client-runtime/markdown-links";
+
 import type { MARKDOWN_FILE_ICON_SOURCES } from "./markdownFileIcons.generated";
 
 const WINDOWS_DRIVE_PATH_PATTERN = /^[A-Za-z]:[\\/]/;
@@ -327,6 +332,7 @@ function looksLikeFilePath(value: string): boolean {
   if (FILE_ICON_BY_NAME[value.replace(POSITION_SUFFIX_PATTERN, "").toLowerCase()]) {
     return true;
   }
+  if (isConventionalFilePosition(value)) return true;
   return RELATIVE_FILE_PATH_PATTERN.test(value) || RELATIVE_FILE_NAME_PATTERN.test(value);
 }
 
@@ -399,3 +405,13 @@ export function resolveMarkdownLinkPresentation(href: string): MarkdownLinkPrese
     href: /^(?:mailto|tel):/i.test(normalized) ? normalized : null,
   };
 }
+
+/** Backticks become file references only when the shared path heuristic recognizes the whole span. */
+export function resolveMarkdownInlineCodePresentation(
+  content: string,
+): Extract<MarkdownLinkPresentation, { readonly kind: "file" }> | null {
+  const candidate = inlineCodeFilePathCandidate(content);
+  if (candidate === null) return null;
+  const presentation = resolveMarkdownLinkPresentation(candidate);
+  return presentation.kind === "file" ? presentation : null;
+}

apps/mobile/modules/t3-markdown-text/src/nativeMarkdownText.ts

@@ -1,7 +1,11 @@
 import type { MarkdownNode } from "react-native-nitro-markdown/headless";
 
 import type { SelectableMarkdownSkill } from "./SelectableMarkdownText.types";
-import { resolveMarkdownLinkPresentation, type MarkdownFileIcon } from "./markdownLinks";
+import {
+  resolveMarkdownInlineCodePresentation,
+  resolveMarkdownLinkPresentation,
+  type MarkdownFileIcon,
+} from "./markdownLinks";
 
 export interface NativeMarkdownTextRun {
   readonly text: string;
@@ -283,8 +287,17 @@ function appendNode(
       return appendRun(runs, textNodeContent(nodeTextContent(node)), context);
     case "html_inline":
       return appendRun(runs, inlineHtmlText(nodeTextContent(node)), context);
-    case "code_inline":
-      return appendRun(runs, nodeTextContent(node), { ...context, code: true });
+    case "code_inline": {
+      const content = nodeTextContent(node);
+      const presentation = context.href ? null : resolveMarkdownInlineCodePresentation(content);
+      return presentation
+        ? appendRun(runs, presentation.label, {
+            ...context,
+            href: presentation.href,
+            fileIcon: presentation.icon,
+          })
+        : appendRun(runs, content, { ...context, code: true });
+    }
     case "soft_break":
       return appendRun(runs, " ", context);
     case "line_break":

apps/mobile/src/components/FilePreviewModal.tsx

@@ -14,6 +14,7 @@ export interface ResolvedFilePreviewSource {
   readonly uri: string;
   readonly name?: string;
   readonly sourceIdentifier?: string;
+  readonly srcFragment?: string;
 }
 
 export type FilePreviewSource = Omit<ResolvedFilePreviewSource, "uri"> &
@@ -40,13 +41,18 @@ function ResolvedFilePreview(props: {
     (connection._tag === "None" || asset._tag === "Failure");
   useEffect(() => Keyboard.dismiss(), []);
   useEffect(() => {
-    if (uri === null && asset._tag === "Success") setUri(asset.url);
-  }, [uri, asset]);
+    if (uri === null && asset._tag === "Success") setUri(asset.url + (source.srcFragment ?? ""));
+  }, [uri, asset, source.srcFragment]);
   useEffect(() => {
     if (!failed) return;
-    Alert.alert("Could not open preview", "Reconnect to this environment and try again.");
+    Alert.alert(
+      "Could not open preview",
+      connection._tag === "None"
+        ? "Reconnect to this environment and try again."
+        : "The file could not be loaded. It may have been moved or deleted.",
+    );
     onRequestClose();
-  }, [failed]);
+  }, [failed, connection._tag]);
   useEffect(() => {
     if (!("attachment" in source)) return;
     const controller = new AbortController();

apps/mobile/src/components/MediaVideoPlayer.tsx

@@ -0,0 +1,144 @@
+import { useIsFocused } from "@react-navigation/native";
+import { useEvent } from "expo";
+import { useVideoPlayer, VideoView } from "expo-video";
+import { useEffect, useRef, useState } from "react";
+import { ActivityIndicator, AppState, Pressable, View } from "react-native";
+
+import { loadMediaVideoSource } from "../lib/mediaVideoPlayback";
+import { AppText } from "./AppText";
+import { SymbolView } from "./AppSymbol";
+
+/** Loads only after Play or opening the viewer. Source replacement never starts playback itself. */
+function LoadedMediaVideo(props: {
+  readonly uri: string;
+  readonly playRequested: boolean;
+  readonly paused: boolean;
+}) {
+  const focused = useIsFocused();
+  const active = useRef(focused && AppState.currentState === "active");
+  const [attempt, setAttempt] = useState(0);
+  const [loadError, setLoadError] = useState(false);
+  const player = useVideoPlayer(null, (player) => {
+    player.staysActiveInBackground = false;
+    player.bufferOptions = { preferredForwardBufferDuration: 5 };
+  });
+  const { status } = useEvent(player, "statusChange", { status: player.status });
+
+  useEffect(() => {
+    active.current = focused && !props.paused && AppState.currentState === "active";
+    if (!active.current) player.pause();
+    const subscription = AppState.addEventListener("change", (state) => {
+      active.current = focused && !props.paused && state === "active";
+      if (!active.current) player.pause();
+    });
+    return () => subscription.remove();
+  }, [focused, player, props.paused]);
+
+  useEffect(() => {
+    const controller = new AbortController();
+    setLoadError(false);
+    void loadMediaVideoSource(player, {
+      uri: props.uri,
+      signal: controller.signal,
+      playRequested: props.playRequested,
+      isActive: () => active.current,
+    }).catch(() => {
+      if (!controller.signal.aborted) setLoadError(true);
+    });
+    return () => controller.abort();
+  }, [player, props.uri, props.playRequested, attempt]);
+
+  return (
+    <View collapsable={false} style={{ flex: 1 }}>
+      <VideoView
+        player={player}
+        style={{ width: "100%", height: "100%" }}
+        surfaceType="textureView"
+        nativeControls
+        contentFit="contain"
+        fullscreenOptions={{ enable: true }}
+        allowsPictureInPicture={false}
+      />
+      {loadError || status === "error" ? (
+        <View className="absolute inset-0 items-center justify-center gap-2 bg-black px-4">
+          <AppText className="text-center text-sm text-white/80">Video unavailable</AppText>
+          <Pressable
+            accessibilityRole="button"
+            accessibilityLabel="Retry video"
+            onPress={() => setAttempt((value) => value + 1)}
+            className="min-h-11 justify-center px-4"
+          >
+            <AppText className="text-sm text-white">Retry</AppText>
+          </Pressable>
+        </View>
+      ) : status === "loading" || status === "idle" ? (
+        <View pointerEvents="none" className="absolute inset-0 items-center justify-center">
+          <ActivityIndicator color="#ffffff" accessibilityLabel="Loading video" />
+        </View>
+      ) : null}
+    </View>
+  );
+}
+
+export function MediaVideoPlayer(props: {
+  readonly uri: string | null;
+  readonly name: string;
+  readonly unavailable?: boolean;
+  readonly expanded?: boolean;
+  readonly paused?: boolean;
+  readonly onExpand?: () => void;
+}) {
+  const [playbackUri, setPlaybackUri] = useState<string | null>(null);
+  const uri = playbackUri ?? (props.expanded ? props.uri : null);
+
+  return (
+    <View
+      collapsable={false}
+      className="overflow-hidden rounded-[10px] bg-black"
+      style={props.expanded ? { flex: 1 } : { width: "100%", maxWidth: 480, aspectRatio: 16 / 9 }}
+    >
+      {uri ? (
+        <LoadedMediaVideo
+          uri={uri}
+          playRequested={playbackUri !== null}
+          paused={props.paused ?? false}
+        />
+      ) : (
+        <Pressable
+          accessibilityRole="button"
+          accessibilityLabel={`Play ${props.name}`}
+          accessibilityState={{ disabled: props.uri === null || props.unavailable === true }}
+          disabled={props.uri === null || props.unavailable === true}
+          onPress={() => setPlaybackUri(props.uri)}
+          className="flex-1 items-center justify-center gap-2 px-4"
+        >
+          {props.unavailable ? (
+            <AppText className="text-sm text-white/80">Video unavailable</AppText>
+          ) : props.uri === null ? (
+            <ActivityIndicator color="#ffffff" accessibilityLabel="Loading video" />
+          ) : (
+            <>
+              <SymbolView name="play" size={28} tintColor="#ffffff" type="monochrome" />
+              <AppText className="text-center text-xs text-white/80" numberOfLines={2}>
+                {props.name}
+              </AppText>
+            </>
+          )}
+        </Pressable>
+      )}
+      {props.onExpand ? (
+        <Pressable
+          accessibilityRole="button"
+          accessibilityLabel={`Expand ${props.name}`}
+          onPress={() => {
+            setPlaybackUri(null);
+            props.onExpand?.();
+          }}
+          className="absolute right-1 top-1 min-h-11 min-w-11 items-center justify-center rounded-md bg-black/60 px-2"
+        >
+          <AppText className="text-xs text-white">Expand</AppText>
+        </Pressable>
+      ) : null}
+    </View>
+  );
+}