feat: Streaming file uploads

packages/kit/src/runtime/app/server/remote/form.js

@@ -4,7 +4,6 @@
 import { get_request_store } from '@sveltejs/kit/internal/server';
 import { DEV } from 'esm-env';
 import {
-	convert_formdata,
 	create_field_proxy,
 	set_nested_value,
 	throw_on_old_property_access,
@@ -104,19 +103,7 @@ export function form(validate_or_fn, maybe_fn) {
 			type: 'form',
 			name: '',
 			id: '',
-			/** @param {FormData} form_data */
-			fn: async (form_data) => {
-				const validate_only = form_data.get('sveltekit:validate_only') === 'true';
-
-				let data = maybe_fn ? convert_formdata(form_data) : undefined;
-
-				if (data && data.id === undefined) {
-					const id = form_data.get('sveltekit:id');
-					if (typeof id === 'string') {
-						data.id = JSON.parse(id);
-					}
-				}
-
+			fn: async (data, meta, form_data) => {
 				// TODO 3.0 remove this warning
 				if (DEV && !data) {
 					const error = () => {
@@ -152,12 +139,12 @@ export function form(validate_or_fn, maybe_fn) {
 				const { event, state } = get_request_store();
 				const validated = await schema?.['~standard'].validate(data);
 
-				if (validate_only) {
+				if (meta.validate_only) {
 					return validated?.issues ?? [];
 				}
 
 				if (validated?.issues !== undefined) {
-					handle_issues(output, validated.issues, event.isRemoteRequest, form_data);
+					handle_issues(output, validated.issues, form_data);
 				} else {
 					if (validated !== undefined) {
 						data = validated.value;
@@ -178,7 +165,7 @@ export function form(validate_or_fn, maybe_fn) {
 						);
 					} catch (e) {
 						if (e instanceof ValidationError) {
-							handle_issues(output, e.issues, event.isRemoteRequest, form_data);
+							handle_issues(output, e.issues, form_data);
 						} else {
 							throw e;
 						}
@@ -297,15 +284,12 @@ export function form(validate_or_fn, maybe_fn) {
 /**
  * @param {{ issues?: InternalRemoteFormIssue[], input?: Record<string, any>, result: any }} output
  * @param {readonly StandardSchemaV1.Issue[]} issues
- * @param {boolean} is_remote_request
- * @param {FormData} form_data
+ * @param {FormData | null} form_data - null if the form is progressively enhanced
  */
-function handle_issues(output, issues, is_remote_request, form_data) {
+function handle_issues(output, issues, form_data) {
 	output.issues = issues.map((issue) => normalize_issue(issue, true));
 
-	// if it was a progressively-enhanced submission, we don't need
-	// to return the input — it's already there
-	if (!is_remote_request) {
+	if (form_data) {
 		output.input = {};
 
 		for (let key of form_data.keys()) {

packages/kit/src/runtime/client/remote-functions/form.svelte.js

@@ -18,7 +18,9 @@ import {
 	set_nested_value,
 	throw_on_old_property_access,
 	build_path_string,
-	normalize_issue
+	normalize_issue,
+	serialize_binary_form,
+	BINARY_FORM_CONTENT_TYPE
 } from '../../form-utils.js';
 
 /**
@@ -182,17 +184,16 @@ export function form(id) {
 				try {
 					await Promise.resolve();
 
-					if (updates.length > 0) {
-						data.set('sveltekit:remote_refreshes', JSON.stringify(updates.map((u) => u._key)));
-					}
-
 					const response = await fetch(`${base}/${app_dir}/remote/${action_id}`, {
 						method: 'POST',
-						body: data,
 						headers: {
-							'x-sveltekit-pathname': location.pathname,
-							'x-sveltekit-search': location.search
-						}
+							'Content-Type': BINARY_FORM_CONTENT_TYPE
+						},
+						body: serialize_binary_form(convert(data), {
+							remote_refreshes: updates.map((u) => u._key),
+							pathname: location.pathname,
+							search: location.search
+						})
 					});
 
 					if (!response.ok) {
@@ -532,7 +533,9 @@ export function form(id) {
 					/** @type {InternalRemoteFormIssue[]} */
 					let array = [];
 
-					const validated = await preflight_schema?.['~standard'].validate(convert(form_data));
+					const data = convert(form_data);
+
+					const validated = await preflight_schema?.['~standard'].validate(data);
 
 					if (validate_id !== id) {
 						return;
@@ -541,11 +544,16 @@ export function form(id) {
 					if (validated?.issues) {
 						array = validated.issues.map((issue) => normalize_issue(issue, false));
 					} else if (!preflightOnly) {
-						form_data.set('sveltekit:validate_only', 'true');
-
 						const response = await fetch(`${base}/${app_dir}/remote/${action_id}`, {
 							method: 'POST',
-							body: form_data
+							headers: {
+								'Content-Type': BINARY_FORM_CONTENT_TYPE
+							},
+							body: serialize_binary_form(data, {
+								validate_only: true,
+								pathname: location.pathname,
+								search: location.search
+							})
 						});
 
 						const result = await response.json();

packages/kit/src/runtime/form-utils.js

@@ -1,8 +1,10 @@
 /** @import { RemoteForm } from '@sveltejs/kit' */
-/** @import { InternalRemoteFormIssue } from 'types' */
+/** @import { BinaryFormMeta, InternalRemoteFormIssue } from 'types' */
 /** @import { StandardSchemaV1 } from '@standard-schema/spec' */
 
 import { DEV } from 'esm-env';
+import * as devalue from 'devalue';
+import { text_decoder } from './utils.js';
 
 /**
  * Sets a value in a nested object using a path string, mutating the original object
@@ -64,6 +66,202 @@
 	return result;
 }
 
+export const BINARY_FORM_CONTENT_TYPE = 'application/x-sveltekit-formdata';
+const BINARY_FORM_VERSION = 0;
+
+/**
+ * The binary format is as follows:
+ * - 1 byte: Format version
+ * - 4 bytes: Length of the header (u32)
+ * - 4 bytes: Number of files (u32)
+ * - header: devalue.stringify([data, meta])
+ * - N files
+ * @param {Record<string, any>} data
+ * @param {BinaryFormMeta} meta
+ * @returns {Blob}
+ */
+export function serialize_binary_form(data, meta) {
+	/** @type {Array<BlobPart>} */
+	const blob_parts = [new Uint8Array([BINARY_FORM_VERSION])];
+
+	/** @type {Array<File>} */
+	const files = [];
+
+	const encoded_header = devalue.stringify([data, meta], {
+		File: (file) => {
+			if (!(file instanceof File)) return;
+			files.push(file);
+			return [file.name, file.type, file.size, file.lastModified, files.length - 1];
+		}
+	});
+	const length_buffer = new Uint8Array(4);
+	const length_view = new DataView(length_buffer.buffer);
+
+	length_view.setUint32(0, encoded_header.length, true);
+	blob_parts.push(length_buffer.slice());
+
+	length_view.setUint32(0, files.length, true);
+	blob_parts.push(length_buffer);
+
+	blob_parts.push(encoded_header);
+
+	blob_parts.push(...files);
+	return new Blob(blob_parts);
+}
+
+/**
+ * @param {Request} request
+ * @returns {Promise<{ data: Record<string, any>; meta: BinaryFormMeta; form_data: FormData | null }>}
+ */
+export async function deserialize_binary_form(request) {
+	if (request.headers.get('content-type') !== BINARY_FORM_CONTENT_TYPE) {
+		const form_data = await request.formData();
+		return { data: convert_formdata(form_data), meta: {}, form_data };
+	}
+	if (!request.body) {
+		throw new Error('Could not deserialize binary form: no body');
+	}
+
+	const reader = request.body.getReader();
+
+	const first_chunk = await reader.read();
+	if (first_chunk.done) {
+		throw new Error('Could not deserialize binary form: empty body');
+	}
+	if (first_chunk.value.byteLength < 1 + 4 + 4) {
+		throw new Error('Could not deserialize binary form: first chunk was too small');
+	}
+	const version = first_chunk.value[0];
+	if (version !== BINARY_FORM_VERSION) {
+		throw new Error(
+			`Could not deserialize binary form: got version ${version}, expected version ${BINARY_FORM_VERSION}`
+		);
+	}
+	const start_view = new DataView(first_chunk.value.buffer);
+	const header_length = start_view.getUint32(1, true);
+	const file_count = start_view.getUint32(5, true);
+
+	// Read the header
+	const header_buffer = new Uint8Array(header_length);
+	header_buffer.set(first_chunk.value.subarray(9, header_length + 9));
+	let received_length = first_chunk.value.byteLength - 9;
+	/** @type {Array<Uint8Array>} */
+	let file_data;
+	if (received_length >= header_length) {
+		file_data = [first_chunk.value.subarray(header_length)];
+	} else {
+		while (true) {
+			const chunk = await reader.read();
+			if (chunk.done) {
+				throw new Error('Could not deserialize binary form: incomplete header');
+			}
+			const header_chunk = chunk.value.subarray(0, header_length - received_length);
+			header_buffer.set(header_chunk, received_length);
+
+			received_length += chunk.value.byteLength;
+
+			if (received_length >= header_length) {
+				file_data = [chunk.value.subarray(header_length)];
+				break;
+			}
+		}
+	}
+
+	/** @type {Array<number>} */
+	const file_sizes = new Array(file_count);
+	/** @type {Array<{start: number, end: number}> | null} */
+	let file_offsets = null;
+	/** @type {Array<Uint8Array<ArrayBuffer>>} */
+	const file_buffers = new Array(file_count);
+
+	const [data, meta] = devalue.parse(text_decoder.decode(header_buffer), {
+		File: ([name, type, size, last_modified, index]) => {
+			file_sizes[index] = size;
+			console.log(file_sizes);
+			return new Proxy(
+				new LazyFile(name, type, size, last_modified, async () => {
+					if (file_buffers[index]) return file_buffers[index];
+					if (file_offsets === null) {
+						file_offsets = new Array(file_count);
+						let start = 0;
+						for (let i = 0; i < file_count; i++) {
+							const end = start + file_sizes[i];
+							file_offsets[i] = { start, end };
+							start = end;
+						}
+					}
+					const { start, end } = file_offsets[index];
+					const buffer = new Uint8Array(end - start);
+					// let offset = 0;
+					// while (offset < buffer.byteLength) {
+					// TODO:
+					// - find the element from `file_data` that contains start + offset
+					// - if it doesn't exist, read from the request body until we get it, and cache results in `file_data`
+					// - copy subarray into `buffer`
+					// }
+					return buffer;
+				}),
+				{
+					getPrototypeOf() {
+						// Trick validators into thinking this is a normal File
+						return File.prototype;
+					}
+				}
+			);
+		}
+	});
+	console.log(data);
+
+	return { data, meta, form_data: null };
+}
+
+class LazyFile {
+	/** @type {() => Promise<ArrayBuffer>} */
+	getter;
+	/**
+	 * @param {string} name
+	 * @param {string} type
+	 * @param {number} size
+	 * @param {number} last_modified
+	 * @param {() => Promise<ArrayBuffer>} getter
+	 */
+	constructor(name, type, size, last_modified, getter) {
+		this.name = name;
+		this.type = type;
+		this.size = size;
+		this.lastModified = last_modified;
+		this.webkitRelativePath = '';
+		this.getter = getter;
+	}
+	arrayBuffer() {
+		return this.getter();
+	}
+	async bytes() {
+		return new Uint8Array(await this.arrayBuffer());
+	}
+	/**
+	 * @param {number=} start
+	 * @param {number=} end
+	 * @param {string=} contentType
+	 */
+	slice(start, end, contentType) {
+		return new LazyFile(this.name, contentType ?? '', this.size, this.lastModified, () =>
+			this.getter().then((buffer) => buffer.slice(start, end))
+		);
+	}
+	stream() {
+		return new ReadableStream({
+			start: async (controller) => {
+				controller.enqueue(await this.arrayBuffer());
+				controller.close();
+			}
+		});
+	}
+	async text() {
+		return text_decoder.decode(await this.arrayBuffer());
+	}
+}
+
 const path_regex = /^[a-zA-Z_$]\w*(\.[a-zA-Z_$]\w*|\[\d+\])*$/;
 
 /**