diff --git a/apps/mobile/src/lib/composerFiles.test.ts b/apps/mobile/src/lib/composerFiles.test.ts index bdcd35c36ff75..7f16b644d3748 100644 --- a/apps/mobile/src/lib/composerFiles.test.ts +++ b/apps/mobile/src/lib/composerFiles.test.ts @@ -590,6 +590,9 @@ describe("composer file attachments", () => { header.writeUInt32BE(3000, 20); const original = header.toString("base64"); mocks.readBase64.mockResolvedValue(original); + const readBytes = vi.fn().mockResolvedValueOnce(header).mockResolvedValue(new Uint8Array()); + const close = vi.fn(); + mocks.open.mockReturnValue({ readBytes, close }); const resize = vi.fn(); const renderAsync = vi.fn(async () => ({ width: 4000, @@ -620,6 +623,16 @@ describe("composer file attachments", () => { const result = await pickComposerFiles({ existingCount: 0 }); expect(resize).toHaveBeenCalledWith({ width: 2000, height: 1500 }); + expect(mocks.readBase64).toHaveBeenCalledTimes(oversized ? 1 : 0); + expect(readBytes.mock.calls.map((call) => call[0])).toEqual([ + 256 * 1024, + 256 * 1024 - header.length, + ]); + expect(close).toHaveBeenCalledExactlyOnceWith(); + expect(mocks.delete.mock.calls.map(([uri]) => uri)).toEqual([ + ...(oversized ? Array.from({ length: 3 }, () => "file:///cache/fitted.png") : []), + "file:///documents/t3-composer-attachments/attachment-id-photo.png", + ]); expect(result).toEqual({ files: [ expect.objectContaining({ diff --git a/apps/mobile/src/lib/composerImageFit.ts b/apps/mobile/src/lib/composerImageFit.ts index 18cd93b3fe723..0bcb066de4752 100644 --- a/apps/mobile/src/lib/composerImageFit.ts +++ b/apps/mobile/src/lib/composerImageFit.ts @@ -87,3 +87,51 @@ export async function fitComposerImage(image: { return null; } } + +async function readFileImageDimensions(uri: string) { + const { File, FileMode } = await import("expo-file-system"); + const reader = new File(uri).open(FileMode.ReadOnly); + try { + const header = new Uint8Array(IMAGE_DIMENSIONS_HEADER_BYTES); + let length = 0; + while (length < header.length) { + const chunk = await reader.readBytes(header.length - length); + if (chunk.byteLength === 0) break; + header.set(chunk, length); + length += chunk.byteLength; + } + return readImageDimensions(header.subarray(0, length)); + } finally { + reader.close(); + } +} + +/** File imports only need original base64 when it will be kept or a header read is unsupported. */ +export async function fitComposerImageFile(image: { + readonly uri: string; + readonly mimeType: string; +}): Promise<{ base64: string; uri: string; fitted: boolean }> { + const { File } = await import("expo-file-system"); + const mimeType = downscalableImageMimeType(image.mimeType); + // Some content providers allow a full read but do not expose file handles. + const dimensions = + mimeType === null ? null : await readFileImageDimensions(image.uri).catch(() => null); + if (dimensions === null) { + const base64 = await new File(image.uri).base64(); + const fitted = await fitComposerImage({ ...image, base64 }); + return fitted === null + ? { base64, uri: image.uri, fitted: false } + : { ...fitted, fitted: true }; + } + if (fitImageAttachmentSize(dimensions) !== null) { + try { + return { + ...(await renderPhoto(image.uri, mimeType === "image/png" ? "png" : "jpeg")), + fitted: true, + }; + } catch (error) { + console.warn("[composer-images] could not scale an oversized image", error); + } + } + return { base64: await new File(image.uri).base64(), uri: image.uri, fitted: false }; +} diff --git a/apps/mobile/src/lib/composerImages.test.ts b/apps/mobile/src/lib/composerImages.test.ts index d082e47f958a2..f99f77832cca3 100644 --- a/apps/mobile/src/lib/composerImages.test.ts +++ b/apps/mobile/src/lib/composerImages.test.ts @@ -4,10 +4,36 @@ import { PROVIDER_SEND_TURN_MAX_IMAGE_BYTES, } from "@t3tools/contracts"; -const files = new Map(); +const files = new Map< + string, + { + base64: string; + deleted: boolean; + text?: string; + readChunkBytes?: number; + openError?: boolean; + readError?: boolean; + } +>(); +const fileReads = { + base64: vi.fn(), + readBytes: vi.fn(), + close: vi.fn(), +}; + +const picker = vi.hoisted(() => ({ launchImageLibraryAsync: vi.fn() })); +vi.mock("expo-image-picker", () => picker); const manipulator = vi.hoisted(() => { - const state = { sourceSize: { width: 0, height: 0 } }; + const state = { + sourceSize: { width: 0, height: 0 }, + failure: null as "render" | "resize" | "save" | null, + }; + const release = vi.fn(); + const saveAsync = vi.fn(async (options: { format: string }) => { + if (state.failure === "save") throw new Error("could not save image"); + return { uri: `file:///cache/rendered.${options.format}`, base64: "cmVuZGVyZWQ=" }; + }); const manipulate = vi.fn((source: string | { width: number; height: number }) => { let size = typeof source === "string" ? state.sourceSize : source; const context = { @@ -15,18 +41,16 @@ const manipulator = vi.hoisted(() => { size = next; return context; }), - renderAsync: vi.fn(async () => ({ - ...size, - release: vi.fn(), - saveAsync: vi.fn(async (options: { format: string }) => ({ - uri: `file:///cache/rendered.${options.format}`, - base64: `cmVuZGVyZWQ=`, - })), - })), + renderAsync: vi.fn(async () => { + if (state.failure === (typeof source === "string" ? "render" : "resize")) { + throw new Error("could not render image"); + } + return { ...size, release, saveAsync }; + }), }; return context; }); - return { state, manipulate }; + return { state, manipulate, release, saveAsync }; }); vi.mock("expo-image-manipulator", () => ({ @@ -51,7 +75,33 @@ vi.mock("expo-file-system", () => ({ return files.has(this.uri) && files.get(this.uri)?.deleted === false; } + get size(): number | null { + const entry = files.get(this.uri); + return entry ? Buffer.from(entry.base64, "base64").byteLength : null; + } + + open() { + const uri = this.uri; + if (files.get(uri)?.openError) throw new Error("unsupported file handle"); + let offset = 0; + return { + readBytes: async (count: number) => { + const bytes = Buffer.from(files.get(uri)?.base64 ?? "", "base64"); + if (files.get(uri)?.readError) throw new Error("could not read header"); + const chunk = bytes.subarray( + offset, + offset + Math.min(count, files.get(uri)?.readChunkBytes ?? count), + ); + offset += chunk.byteLength; + fileReads.readBytes(uri, count, chunk.byteLength); + return chunk; + }, + close: () => fileReads.close(uri), + }; + } + async base64(): Promise { + fileReads.base64(this.uri); const entry = files.get(this.uri); if (!entry || entry.deleted) { throw new Error("missing file"); @@ -90,6 +140,7 @@ vi.mock("expo-file-system", () => ({ create(): void {} }, + FileMode: { ReadOnly: "r" }, Paths: { document: "file:///documents" }, })); @@ -97,7 +148,9 @@ vi.mock("./uuid", () => ({ uuidv4: () => "attachment-id", })); -import { convertPastedImagesToAttachments } from "./composerImages"; +import { convertPastedImagesToAttachments, pickComposerMedia } from "./composerImages"; +import { IMAGE_DIMENSIONS_HEADER_BYTES } from "@t3tools/shared/imageDimensions"; +import { fitComposerImage, fitComposerImageFile } from "./composerImageFit"; describe("native pasted image cleanup", () => { beforeEach(() => { @@ -194,3 +247,245 @@ describe("composerStripAttachments", () => { expect(isComposerImageAttachment(oversizedImage)).toBe(false); }); }); + +describe("file image header fitting", () => { + function imageBytes(format = "png", width = 3024, height = 1964, length = 1024 * 1024) { + const bytes = Buffer.alloc(length); + if (format === "png") { + bytes.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + bytes.writeUInt32BE(13, 8); + bytes.write("IHDR", 12); + bytes.writeUInt32BE(width, 16); + bytes.writeUInt32BE(height, 20); + } else { + bytes.set([0xff, 0xd8, 0xff, 0xc0, 0, 17, 8]); + bytes.writeUInt16BE(height, 7); + bytes.writeUInt16BE(width, 9); + } + return bytes.toString("base64"); + } + + async function importImage(entryPoint: string, uri: string, mimeType = "image/png") { + if (entryPoint === "native paste") { + return convertPastedImagesToAttachments({ uris: [uri], existingCount: 0 }); + } + picker.launchImageLibraryAsync.mockResolvedValue({ + canceled: false, + assets: [{ uri, type: "image", mimeType, fileName: "photo" }], + }); + const result = await pickComposerMedia({ existingCount: 0 }); + expect(result.error).toBeNull(); + return result.attachments; + } + + beforeEach(() => { + files.clear(); + vi.clearAllMocks(); + manipulator.state.sourceSize = { width: 3024, height: 1964 }; + manipulator.state.failure = null; + }); + + it.each([ + ["picker", "png"], + ["native paste", "png"], + ["picker", "jpeg"], + ["native paste", "jpeg"], + ])( + "%s resizes from the URI without reading full original base64 (%s)", + async (entryPoint, format) => { + const uri = `file:///photos/photo.${format}`; + const mimeType = `image/${format}`; + const base64 = imageBytes(format); + files.set(uri, { base64, deleted: false }); + + const attachments = await importImage(entryPoint, uri, mimeType); + + expect(attachments).toEqual([ + expect.objectContaining({ + mimeType, + sizeBytes: 8, + dataUrl: `data:${mimeType};base64,cmVuZGVyZWQ=`, + previewUri: `file:///cache/rendered.${format}`, + }), + ]); + expect(fileReads.base64).not.toHaveBeenCalled(); + expect(fileReads.readBytes).toHaveBeenCalledExactlyOnceWith( + uri, + IMAGE_DIMENSIONS_HEADER_BYTES, + IMAGE_DIMENSIONS_HEADER_BYTES, + ); + expect(fileReads.close).toHaveBeenCalledExactlyOnceWith(uri); + expect(manipulator.manipulate).toHaveBeenCalledWith(uri); + expect( + manipulator.manipulate.mock.results.at(-1)?.value.resize, + ).toHaveBeenCalledExactlyOnceWith({ + width: 2000, + height: 1299, + }); + expect(manipulator.saveAsync).toHaveBeenCalledExactlyOnceWith({ + format, + compress: 0.85, + base64: true, + }); + expect(manipulator.release).toHaveBeenCalledTimes(2); + expect(await fitComposerImage({ uri, mimeType, base64 })).toEqual({ + base64: "cmVuZGVyZWQ=", + uri: `file:///cache/rendered.${format}`, + }); + }, + ); + + it("fills partial header reads without exceeding the bound and closes before rendering", async () => { + const uri = "file:///photos/partial.png"; + files.set(uri, { base64: imageBytes(), deleted: false, readChunkBytes: 64 * 1024 }); + + await importImage("native paste", uri); + + expect(fileReads.readBytes.mock.calls.map((call) => call[2])).toEqual(Array(4).fill(64 * 1024)); + expect(fileReads.readBytes.mock.calls.map((call) => call[1])).toEqual([ + 256 * 1024, + 192 * 1024, + 128 * 1024, + 64 * 1024, + ]); + expect(fileReads.close).toHaveBeenCalledExactlyOnceWith(uri); + expect(fileReads.close.mock.invocationCallOrder[0]).toBeLessThan( + manipulator.manipulate.mock.invocationCallOrder[0]!, + ); + expect(fileReads.base64).not.toHaveBeenCalled(); + }); + + it.each(["picker", "native paste"])( + "%s keeps small and corrupt originals byte-identical", + async (entryPoint) => { + for (const base64 of [ + imageBytes("png", 2000, 1200, 24), + "aGVsbG8=", + imageBytes("png", 0, 0, 24), + ]) { + const uri = "file:///photos/original.png"; + files.set(uri, { base64, deleted: false }); + const readsBefore = fileReads.base64.mock.calls.length; + const closesBefore = fileReads.close.mock.calls.length; + + const attachments = await importImage(entryPoint, uri); + + expect(attachments).toEqual([ + expect.objectContaining({ + dataUrl: `data:image/png;base64,${base64}`, + previewUri: uri, + }), + ]); + expect(fileReads.base64).toHaveBeenCalledTimes(readsBefore + 1); + expect(fileReads.close).toHaveBeenCalledTimes(closesBefore + 1); + } + expect(manipulator.manipulate).not.toHaveBeenCalled(); + }, + ); + + it.each(["openError", "readError"] as const)( + "falls back for content URI %s and preserves fitting", + async (failure) => { + const uri = "content://photos/oversized.png"; + files.set(uri, { base64: imageBytes(), deleted: false, [failure]: true }); + + const attachments = await importImage("native paste", uri); + + expect(attachments).toEqual([ + expect.objectContaining({ + dataUrl: "data:image/png;base64,cmVuZGVyZWQ=", + }), + ]); + expect(fileReads.base64).toHaveBeenCalledExactlyOnceWith(uri); + expect(fileReads.close).toHaveBeenCalledTimes(failure === "readError" ? 1 : 0); + expect(manipulator.saveAsync).toHaveBeenCalledExactlyOnceWith({ + format: "png", + compress: 0.85, + base64: true, + }); + }, + ); + + it.each(["render", "resize", "save"] as const)( + "retains original bytes after native %s failure and releases images", + async (failure) => { + using _warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const uri = "file:///photos/failure.png"; + const base64 = imageBytes(); + files.set(uri, { base64, deleted: false }); + manipulator.state.failure = failure; + + const attachments = await importImage("native paste", uri); + + expect(attachments).toEqual([ + expect.objectContaining({ + dataUrl: `data:image/png;base64,${base64}`, + previewUri: uri, + }), + ]); + expect(fileReads.base64).toHaveBeenCalledExactlyOnceWith(uri); + expect(fileReads.close).toHaveBeenCalledExactlyOnceWith(uri); + expect(manipulator.manipulate).toHaveBeenCalledTimes(failure === "render" ? 1 : 2); + expect(manipulator.release).toHaveBeenCalledTimes( + failure === "render" ? 0 : failure === "resize" ? 1 : 2, + ); + }, + ); + + it.each(["gif", "webp"])( + "keeps animated %s bytes without reading a header or rendering", + async (format) => { + const uri = `file:///photos/animated.${format}`; + const base64 = "YW5pbWF0aW9u"; + files.set(uri, { base64, deleted: false }); + + for (const entryPoint of ["picker", "native paste"]) { + const attachments = await importImage(entryPoint, uri, `image/${format}`); + expect(attachments).toEqual([ + expect.objectContaining({ dataUrl: `data:image/${format};base64,${base64}` }), + ]); + } + + expect(fileReads.base64).toHaveBeenCalledTimes(2); + expect(fileReads.readBytes).not.toHaveBeenCalled(); + expect(manipulator.manipulate).not.toHaveBeenCalled(); + }, + ); + + it("leaves unsupported formats to the existing picker conversion and file pass-through", async () => { + const uri = "file:///photos/photo.heic"; + const base64 = "aGVsbG8="; + files.set(uri, { base64, deleted: false }); + + expect(await fitComposerImageFile({ uri, mimeType: "image/heic" })).toEqual({ + base64, + uri, + fitted: false, + }); + expect(fileReads.readBytes).not.toHaveBeenCalled(); + expect(manipulator.manipulate).not.toHaveBeenCalled(); + expect(await importImage("picker", uri, "image/heic")).toEqual([ + expect.objectContaining({ + mimeType: "image/jpeg", + dataUrl: "data:image/jpeg;base64,cmVuZGVyZWQ=", + }), + ]); + expect(fileReads.base64).toHaveBeenCalledExactlyOnceWith(uri); + expect(fileReads.readBytes).not.toHaveBeenCalled(); + }); + + it("retains the owned paste cleanup and data-backed preview after fitting", async () => { + const uri = "file:///tmp/t3-composer-paste/id.png"; + files.set(uri, { base64: imageBytes(), deleted: false }); + + expect(await importImage("native paste", uri)).toEqual([ + expect.objectContaining({ + dataUrl: "data:image/png;base64,cmVuZGVyZWQ=", + previewUri: "data:image/png;base64,cmVuZGVyZWQ=", + }), + ]); + expect(files.get(uri)?.deleted).toBe(true); + expect(fileReads.base64).not.toHaveBeenCalled(); + expect(fileReads.close).toHaveBeenCalledExactlyOnceWith(uri); + }); +}); diff --git a/apps/mobile/src/lib/composerImages.ts b/apps/mobile/src/lib/composerImages.ts index cf28b437725ca..1dbe25b9453f8 100644 --- a/apps/mobile/src/lib/composerImages.ts +++ b/apps/mobile/src/lib/composerImages.ts @@ -20,7 +20,7 @@ import { } from "./composerAttachmentFiles"; import { imageMimeType } from "@t3tools/shared/image"; import { videoMimeType } from "@t3tools/shared/video"; -import { fitComposerImage, renderPhoto } from "./composerImageFit"; +import { fitComposerImageFile, renderPhoto } from "./composerImageFit"; import { beginForegroundHandoff } from "./foreground-handoff"; import { uuidv4 } from "./uuid"; import { writeFileAtomically } from "./atomic-file"; @@ -315,19 +315,22 @@ export async function pickComposerFiles(input: { continue; } try { - const { File } = await import("expo-file-system"); - const original = await new File(attachment.fileUri).base64(); - const candidate = await fitComposerImage({ + let image = await fitComposerImageFile({ uri: attachment.fileUri, mimeType, - base64: original, }); - const fitted = - candidate !== null && - estimateBase64ByteSize(candidate.base64) <= PROVIDER_SEND_TURN_MAX_IMAGE_BYTES - ? candidate - : null; - const base64 = fitted?.base64 ?? original; + if ( + image.fitted && + estimateBase64ByteSize(image.base64) > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + ) { + const { File } = await import("expo-file-system"); + image = { + base64: await new File(attachment.fileUri).base64(), + uri: attachment.fileUri, + fitted: false, + }; + } + const base64 = image.base64; const sizeBytes = estimateBase64ByteSize(base64); if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) { throw new Error(`'${name}' exceeds the 10 MB attachment limit.`); @@ -340,7 +343,7 @@ export async function pickComposerFiles(input: { mimeType, sizeBytes, dataUrl, - previewUri: fitted?.uri ?? dataUrl, + previewUri: image.fitted ? image.uri : dataUrl, }); } finally { await removePersistedComposerAttachmentFile(attachment.fileUri); @@ -487,18 +490,15 @@ export async function pickComposerMedia(input: { let image: { base64: string; mimeType: string; name: string; previewUri: string }; try { if (originalMimeType !== null) { - const { File } = await import("expo-file-system"); - const base64 = await new File(asset.uri).base64(); - const fitted = await fitComposerImage({ + const fitted = await fitComposerImageFile({ uri: asset.uri, mimeType: originalMimeType, - base64, }); image = { - base64: fitted?.base64 ?? base64, + base64: fitted.base64, mimeType: originalMimeType, name, - previewUri: fitted?.uri ?? asset.uri, + previewUri: fitted.uri, }; } else { const rendered = await renderPhoto(asset.uri); @@ -585,11 +585,9 @@ export async function convertPastedImagesToAttachments(input: { if (index >= Math.max(0, remainingSlots)) { continue; } - const file = new File(uri); const mimeType = mimeTypeFromUri(uri); - const pastedBase64 = await file.base64(); - const fitted = await fitComposerImage({ uri, mimeType, base64: pastedBase64 }); - const base64 = fitted?.base64 ?? pastedBase64; + const fitted = await fitComposerImageFile({ uri, mimeType }); + const base64 = fitted.base64; const sizeBytes = estimateBase64ByteSize(base64); if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) { continue; @@ -601,7 +599,7 @@ export async function convertPastedImagesToAttachments(input: { mimeType, sizeBytes, dataUrl: `data:${mimeType};base64,${base64}`, - previewUri: ownedTemporaryFile ? `data:${mimeType};base64,${base64}` : (fitted?.uri ?? uri), + previewUri: ownedTemporaryFile ? `data:${mimeType};base64,${base64}` : fitted.uri, }); } catch (error) { console.warn("Failed to read pasted image", uri, error);