From 1ab5f83b6b755f1a8925b153e2f7a3e953bf5132 Mon Sep 17 00:00:00 2001
From: Glen Maddern <glen@cloudflare.com>
Date: Sat, 13 Apr 2024 10:42:45 +1000
Subject: [PATCH 01/12] D1: Refactored to remove batching entirely

This was only required for `execute --remote --file`, which will now have a dedicated endpoint.
---
 packages/wrangler/src/d1/constants.ts         |   1 -
 packages/wrangler/src/d1/execute.tsx          | 108 +++++++-----------
 packages/wrangler/src/d1/migrations/apply.tsx |  10 +-
 .../wrangler/src/d1/migrations/helpers.ts     |   4 +-
 4 files changed, 46 insertions(+), 77 deletions(-)

diff --git a/packages/wrangler/src/d1/constants.ts b/packages/wrangler/src/d1/constants.ts
index 90a5459eeb..a2e9834a9d 100644
--- a/packages/wrangler/src/d1/constants.ts
+++ b/packages/wrangler/src/d1/constants.ts
@@ -2,4 +2,3 @@ export const DEFAULT_MIGRATION_PATH = "./migrations";
 export const DEFAULT_MIGRATION_TABLE = "d1_migrations";
 export const LOCATION_CHOICES = ["weur", "eeur", "apac", "oc", "wnam", "enam"];
 // Max number of statements to send in a single /execute call
-export const DEFAULT_BATCH_SIZE = 10_000;
diff --git a/packages/wrangler/src/d1/execute.tsx b/packages/wrangler/src/d1/execute.tsx
index 4ca0793672..ebdd59c375 100644
--- a/packages/wrangler/src/d1/execute.tsx
+++ b/packages/wrangler/src/d1/execute.tsx
@@ -1,4 +1,5 @@
 import assert from "node:assert";
+import fs from "node:fs/promises";
 import path from "node:path";
 import chalk from "chalk";
 import { Static, Text } from "ink";
@@ -16,7 +17,6 @@ import { readFileSync } from "../parse";
 import { readableRelative } from "../paths";
 import { requireAuth } from "../user";
 import { renderToString } from "../utils/render";
-import { DEFAULT_BATCH_SIZE } from "./constants";
 import * as options from "./options";
 import splitSqlQuery from "./splitter";
 import { getDatabaseByNameOrBinding, getDatabaseInfoFromConfig } from "./utils";
@@ -81,7 +81,7 @@ export function Options(yargs: CommonYargsArgv) {
 		.option("batch-size", {
 			describe: "Number of queries to send in a single batch",
 			type: "number",
-			default: DEFAULT_BATCH_SIZE,
+			deprecated: true,
 		});
 }
 
@@ -98,7 +98,6 @@ export const Handler = async (args: HandlerOptions): Promise<void> => {
 		command,
 		json,
 		preview,
-		batchSize,
 	} = args;
 	const existingLogLevel = logger.loggerLevel;
 	if (json) {
@@ -126,7 +125,6 @@ export const Handler = async (args: HandlerOptions): Promise<void> => {
 			command,
 			json,
 			preview,
-			batchSize,
 		});
 
 		// Early exit if prompt rejected
@@ -177,6 +175,10 @@ export const Handler = async (args: HandlerOptions): Promise<void> => {
 	}
 };
 
+type ExecuteInput =
+	| { file: string; command: never }
+	| { file: never; command: string };
+
 export async function executeSql({
 	local,
 	remote,
@@ -188,7 +190,6 @@ export async function executeSql({
 	command,
 	json,
 	preview,
-	batchSize,
 }: {
 	local: boolean | undefined;
 	remote: boolean | undefined;
@@ -200,7 +201,6 @@ export async function executeSql({
 	command: string | undefined;
 	json: boolean | undefined;
 	preview: boolean | undefined;
-	batchSize: number;
 }) {
 	const existingLogLevel = logger.loggerLevel;
 	if (json) {
@@ -208,10 +208,12 @@ export async function executeSql({
 		logger.loggerLevel = "error";
 	}
 
-	const sql = file ? readFileSync(file) : command;
-	if (!sql) {
-		throw new UserError(`Error: must provide --command or --file.`);
-	}
+	const input = file
+		? ({ file } as ExecuteInput)
+		: command
+		? ({ command } as ExecuteInput)
+		: null;
+	if (!input) throw new UserError(`Error: must provide --command or --file.`);
 	if (local && remote) {
 		throw new UserError(
 			`Error: can't use --local and --remote at the same time`
@@ -224,30 +226,22 @@ export async function executeSql({
 		throw new UserError(`Error: can't use --persist-to without --local`);
 	}
 	logger.log(`🌀 Mapping SQL input into an array of statements`);
-	const queries = splitSqlQuery(sql);
 
-	if (file && sql) {
-		if (queries[0].startsWith("SQLite format 3")) {
-			//TODO: update this error to recommend using `wrangler d1 restore` when it exists
-			throw new UserError(
-				"Provided file is a binary SQLite database file instead of an SQL text file.\nThe execute command can only process SQL text files.\nPlease export an SQL file from your SQLite database and try again."
-			);
-		}
-	}
+	if (input.file) await checkForSQLiteBinary(input.file);
+
 	const result =
 		remote || preview
 			? await executeRemotely({
 					config,
 					name,
 					shouldPrompt,
-					batches: batchSplit(queries, batchSize),
-					json,
+					input,
 					preview,
 				})
 			: await executeLocally({
 					config,
 					name,
-					queries,
+					input,
 					persistTo,
 				});
 
@@ -260,12 +254,12 @@ export async function executeSql({
 async function executeLocally({
 	config,
 	name,
-	queries,
+	input,
 	persistTo,
 }: {
 	config: Config;
 	name: string;
-	queries: string[];
+	input: ExecuteInput;
 	persistTo: string | undefined;
 }) {
 	const localDB = getDatabaseInfoFromConfig(config, name);
@@ -296,6 +290,9 @@ async function executeLocally({
 	});
 	const db = await mf.getD1Database("DATABASE");
 
+	const sql = input.file ? readFileSync(input.file) : input.command;
+	const queries = splitSqlQuery(sql);
+
 	let results: D1Result<Record<string, string | number | boolean>>[];
 	try {
 		results = await db.batch(queries.map((query) => db.prepare(query)));
@@ -328,30 +325,21 @@ async function executeRemotely({
 	config,
 	name,
 	shouldPrompt,
-	batches,
-	json,
+	input,
 	preview,
 }: {
 	config: Config;
 	name: string;
 	shouldPrompt: boolean | undefined;
-	batches: string[];
-	json: boolean | undefined;
+	input: ExecuteInput;
 	preview: boolean | undefined;
 }) {
-	const multiple_batches = batches.length > 1;
-	// in JSON mode, we don't want a prompt here
-	if (multiple_batches && !json) {
-		const warning = `⚠️  Too much SQL to send at once, this execution will be sent as ${batches.length} batches.`;
+	if (input.file) {
+		const warning = `ℹ️  This process may take some time, during which your D1 will be unavailable to serve queries.`;
 
 		if (shouldPrompt) {
-			const ok = await confirm(
-				`${warning}\nℹ️  Each batch is sent individually and may leave your DB in an unexpected state if a later batch fails.\n⚠️  Make sure you have a recent backup. Ok to proceed?`
-			);
-			if (!ok) {
-				return null;
-			}
-			logger.log(`🌀 Let's go`);
+			const ok = await confirm(`${warning}\n  Ok to proceed?`);
+			if (!ok) return null;
 		} else {
 			logger.error(warning);
 		}
@@ -376,14 +364,9 @@ async function executeRemotely({
 		"🌀 To execute on your local development database, remove the --remote flag from your wrangler command."
 	);
 
-	const results: QueryResult[] = [];
-	for (const sql of batches) {
-		if (multiple_batches) {
-			logger.log(
-				chalk.gray(`  ${sql.slice(0, 70)}${sql.length > 70 ? "..." : ""}`)
-			);
-		}
-
+	if (input.file) {
+		return null;
+	} else {
 		const result = await fetchResult<QueryResult[]>(
 			`/accounts/${accountId}/d1/database/${dbUuid}/query`,
 			{
@@ -392,13 +375,12 @@ async function executeRemotely({
 					"Content-Type": "application/json",
 					...(db.internal_env ? { "x-d1-internal-env": db.internal_env } : {}),
 				},
-				body: JSON.stringify({ sql }),
+				body: JSON.stringify({ sql: input.command }),
 			}
 		);
 		logResult(result);
-		results.push(...result);
+		return result;
 	}
-	return results;
 }
 
 function logResult(r: QueryResult | QueryResult[]) {
@@ -415,23 +397,19 @@ function logResult(r: QueryResult | QueryResult[]) {
 	);
 }
 
-function batchSplit(queries: string[], batchSize: number) {
-	logger.log(`🌀 Parsing ${queries.length} statements`);
-	const num_batches = Math.ceil(queries.length / batchSize);
-	const batches: string[] = [];
-	for (let i = 0; i < num_batches; i++) {
-		batches.push(queries.slice(i * batchSize, (i + 1) * batchSize).join("; "));
-	}
-	if (num_batches > 1) {
-		logger.log(
-			`🌀 We are sending ${num_batches} batch(es) to D1 (limited to ${batchSize} statements per batch. Use --batch-size to override.)`
-		);
-	}
-	return batches;
-}
-
 function shorten(query: string | undefined, length: number) {
 	return query && query.length > length
 		? query.slice(0, length) + "..."
 		: query;
 }
+
+async function checkForSQLiteBinary(filename: string) {
+	const fd = await fs.open(filename, "r");
+	const buffer = Buffer.alloc(15);
+	await fd.read(buffer, 0, 15);
+	if (buffer.toString("utf8") === "SQLite format 3") {
+		throw new UserError(
+			"Provided file is a binary SQLite database file instead of an SQL text file.\nThe execute command can only process SQL text files.\nPlease export an SQL file from your SQLite database and try again."
+		);
+	}
+}
diff --git a/packages/wrangler/src/d1/migrations/apply.tsx b/packages/wrangler/src/d1/migrations/apply.tsx
index bb98a89d96..95dc81275a 100644
--- a/packages/wrangler/src/d1/migrations/apply.tsx
+++ b/packages/wrangler/src/d1/migrations/apply.tsx
@@ -14,11 +14,7 @@ import { logger } from "../../logger";
 import { requireAuth } from "../../user";
 import { renderToString } from "../../utils/render";
 import { createBackup } from "../backups";
-import {
-	DEFAULT_BATCH_SIZE,
-	DEFAULT_MIGRATION_PATH,
-	DEFAULT_MIGRATION_TABLE,
-} from "../constants";
+import { DEFAULT_MIGRATION_PATH, DEFAULT_MIGRATION_TABLE } from "../constants";
 import { executeSql } from "../execute";
 import { getDatabaseInfoFromConfig, getDatabaseInfoFromId } from "../utils";
 import {
@@ -37,7 +33,7 @@ export function ApplyOptions(yargs: CommonYargsArgv) {
 	return MigrationOptions(yargs).option("batch-size", {
 		describe: "Number of queries to send in a single batch",
 		type: "number",
-		default: DEFAULT_BATCH_SIZE,
+		deprecated: true,
 	});
 }
 
@@ -51,7 +47,6 @@ export const ApplyHandler = withConfig<ApplyHandlerOptions>(
 		remote,
 		persistTo,
 		preview,
-		batchSize,
 	}): Promise<void> => {
 		await printWranglerBanner();
 		const databaseInfo = getDatabaseInfoFromConfig(config, database);
@@ -175,7 +170,6 @@ Your database may not be available to serve requests during the migration, conti
 					file: undefined,
 					json: undefined,
 					preview,
-					batchSize,
 				});
 
 				if (response === null) {
diff --git a/packages/wrangler/src/d1/migrations/helpers.ts b/packages/wrangler/src/d1/migrations/helpers.ts
index 2d98fe91d7..9f07a36fc8 100644
--- a/packages/wrangler/src/d1/migrations/helpers.ts
+++ b/packages/wrangler/src/d1/migrations/helpers.ts
@@ -5,7 +5,7 @@ import { UserError } from "../../errors";
 import { CI } from "../../is-ci";
 import isInteractive from "../../is-interactive";
 import { logger } from "../../logger";
-import { DEFAULT_BATCH_SIZE, DEFAULT_MIGRATION_PATH } from "../constants";
+import { DEFAULT_MIGRATION_PATH } from "../constants";
 import { executeSql } from "../execute";
 import type { ConfigFields, DevConfig, Environment } from "../../config";
 import type { QueryResult } from "../execute";
@@ -118,7 +118,6 @@ const listAppliedMigrations = async ({
 		file: undefined,
 		json: true,
 		preview,
-		batchSize: DEFAULT_BATCH_SIZE,
 	});
 
 	if (!response || response[0].results.length === 0) {
@@ -189,6 +188,5 @@ export const initMigrationsTable = async ({
 		file: undefined,
 		json: true,
 		preview,
-		batchSize: DEFAULT_BATCH_SIZE,
 	});
 };

From 1056b472d8ec7ace8732a29fcca3843b1e5895d1 Mon Sep 17 00:00:00 2001
From: Glen Maddern <glen@cloudflare.com>
Date: Tue, 16 Apr 2024 19:41:26 +1000
Subject: [PATCH 02/12] Adds R2-based SQL import into remote D1 DBs

This hits a new API endpoint 'import', that initially returns a signed R2 PUT url for uploading SQL, then returns polling info as the import completes. This completely bypasses the existing split-chunk-and-execute implementation that often left your DB in an invalid state
---
 packages/wrangler/package.json        |   1 +
 packages/wrangler/src/d1/constants.ts |   1 -
 packages/wrangler/src/d1/execute.tsx  | 200 ++++++++++++++++++++++----
 packages/wrangler/src/d1/types.ts     |  35 +++++
 4 files changed, 209 insertions(+), 28 deletions(-)

diff --git a/packages/wrangler/package.json b/packages/wrangler/package.json
index d512a005f4..62052232ad 100644
--- a/packages/wrangler/package.json
+++ b/packages/wrangler/package.json
@@ -173,6 +173,7 @@
 		"jest": "^29.7.0",
 		"jest-fetch-mock": "^3.0.3",
 		"jest-websocket-mock": "^2.5.0",
+		"md5-file": "^5.0.0",
 		"mime": "^3.0.0",
 		"minimatch": "^5.1.0",
 		"msw": "^0.49.1",
diff --git a/packages/wrangler/src/d1/constants.ts b/packages/wrangler/src/d1/constants.ts
index a2e9834a9d..45e14b379f 100644
--- a/packages/wrangler/src/d1/constants.ts
+++ b/packages/wrangler/src/d1/constants.ts
@@ -1,4 +1,3 @@
 export const DEFAULT_MIGRATION_PATH = "./migrations";
 export const DEFAULT_MIGRATION_TABLE = "d1_migrations";
 export const LOCATION_CHOICES = ["weur", "eeur", "apac", "oc", "wnam", "enam"];
-// Max number of statements to send in a single /execute call
diff --git a/packages/wrangler/src/d1/execute.tsx b/packages/wrangler/src/d1/execute.tsx
index ebdd59c375..a5cbf42ab8 100644
--- a/packages/wrangler/src/d1/execute.tsx
+++ b/packages/wrangler/src/d1/execute.tsx
@@ -1,11 +1,13 @@
+import { createReadStream, promises as fs } from "fs";
 import assert from "node:assert";
-import fs from "node:fs/promises";
 import path from "node:path";
 import chalk from "chalk";
 import { Static, Text } from "ink";
 import Table from "ink-table";
+import md5File from "md5-file";
 import { Miniflare } from "miniflare";
 import React from "react";
+import { fetch } from "undici";
 import { printWranglerBanner } from "../";
 import { fetchResult } from "../cfetch";
 import { readConfig } from "../config";
@@ -13,7 +15,7 @@ import { getLocalPersistencePath } from "../dev/get-local-persistence-path";
 import { confirm } from "../dialogs";
 import { JsonFriendlyFatalError, UserError } from "../errors";
 import { logger } from "../logger";
-import { readFileSync } from "../parse";
+import { APIError, readFileSync } from "../parse";
 import { readableRelative } from "../paths";
 import { requireAuth } from "../user";
 import { renderToString } from "../utils/render";
@@ -25,7 +27,11 @@ import type {
 	CommonYargsArgv,
 	StrictYargsOptionsToInterface,
 } from "../yargs-types";
-import type { Database } from "./types";
+import type {
+	Database,
+	ImportInitResponse,
+	ImportPollingResponse,
+} from "./types";
 import type { D1Result } from "@cloudflare/workers-types/experimental";
 
 export type QueryResult = {
@@ -224,9 +230,7 @@ export async function executeSql({
 	}
 	if (persistTo && !local) {
 		throw new UserError(`Error: can't use --persist-to without --local`);
-	}
-	logger.log(`🌀 Mapping SQL input into an array of statements`);
-
+  }
 	if (input.file) await checkForSQLiteBinary(input.file);
 
 	const result =
@@ -335,13 +339,13 @@ async function executeRemotely({
 	preview: boolean | undefined;
 }) {
 	if (input.file) {
-		const warning = `ℹ️  This process may take some time, during which your D1 will be unavailable to serve queries.`;
+		const warning = `⚠️ This process may take some time, during which your D1 will be unavailable to serve queries.`;
 
 		if (shouldPrompt) {
 			const ok = await confirm(`${warning}\n  Ok to proceed?`);
 			if (!ok) return null;
 		} else {
-			logger.error(warning);
+			logger.log(warning);
 		}
 	}
 
@@ -351,38 +355,180 @@ async function executeRemotely({
 		accountId,
 		name
 	);
-	if (preview && !db.previewDatabaseUuid) {
-		const error = new UserError(
-			"Please define a `preview_database_id` in your wrangler.toml to execute your queries against a preview database"
-		);
-		logger.error(error.message);
-		throw error;
+	if (preview) {
+		if (!db.previewDatabaseUuid) {
+			const error = new UserError(
+				"Please define a `preview_database_id` in your wrangler.toml to execute your queries against a preview database"
+			);
+			logger.error(error.message);
+			throw error;
+		}
+		db.uuid = db.previewDatabaseUuid;
 	}
-	const dbUuid = preview ? db.previewDatabaseUuid : db.uuid;
-	logger.log(`🌀 Executing on remote database ${name} (${dbUuid}):`);
+	logger.log(
+		`🌀 Executing on ${
+			db.previewDatabaseUuid ? "preview" : "remote"
+		} database ${name} (${db.uuid}):`
+	);
 	logger.log(
 		"🌀 To execute on your local development database, remove the --remote flag from your wrangler command."
 	);
 
 	if (input.file) {
+		// TODO: do we need to update hashing code if we upload in parts?
+		const etag = await md5File(input.file);
+
+		logger.log(
+			chalk.gray(
+				`Note: if the execution fails to complete, your DB will return to its original state and you can safely retry.`
+			)
+		);
+
+		const initResponse = await d1ApiPost<
+			| ImportInitResponse
+			| ImportPollingResponse
+			| { success: false; error: string }
+		>(accountId, db, "import", { action: "init", etag });
+
+		// An init response usually returns a {filename, uploadUrl} pair, except if we've detected that file
+		// already exists and is valid, to save people reuploading. In which case `initResponse` has already
+		// kicked the import process off.
+		const uploadRequired = "uploadUrl" in initResponse;
+		if (!uploadRequired) logger.log(`🌀 File already uploaded. Processing.`);
+		const firstPollResponse = uploadRequired
+			? await uploadAndBeginIngestion(
+					accountId,
+					db,
+					input.file,
+					etag,
+					initResponse
+			  )
+			: initResponse;
+
+		const finalResponse = await pollUntilComplete(
+			firstPollResponse,
+			accountId,
+			db
+		);
+
+		if (finalResponse.status !== "complete")
+			throw new APIError({ text: `D1 reset before execute completed!` });
+
+		const {
+			result: { numQueries, finalBookmark, meta },
+		} = finalResponse;
+		logger.log(
+			`🚣 Executed ${numQueries} queries in ${(meta.duration / 1000).toFixed(
+				2
+			)} seconds (${meta.rows_read} rows read, ${
+				meta.rows_written
+			} rows written)\n` +
+				`   Database is currently ${(meta.size_after / 1_000_000).toFixed(
+					2
+				)}MB (at bookmark ${chalk.gray(finalBookmark)}).`
+		);
+
 		return null;
 	} else {
-		const result = await fetchResult<QueryResult[]>(
-			`/accounts/${accountId}/d1/database/${dbUuid}/query`,
-			{
-				method: "POST",
-				headers: {
-					"Content-Type": "application/json",
-					...(db.internal_env ? { "x-d1-internal-env": db.internal_env } : {}),
-				},
-				body: JSON.stringify({ sql: input.command }),
-			}
-		);
+		const result = await d1ApiPost<QueryResult[]>(accountId, db, "query", {
+			sql: input.command,
+		});
 		logResult(result);
 		return result;
 	}
 }
 
+async function uploadAndBeginIngestion(
+	accountId: string,
+	db: Database,
+	file: string,
+	etag: string,
+	initResponse: ImportInitResponse
+) {
+	const { uploadUrl, filename } = initResponse;
+	logger.log(`🌀 Uploading ${filename}...`);
+
+	const { size } = await fs.stat(file);
+
+	const uploadResponse = await fetch(uploadUrl, {
+		method: "PUT",
+		headers: {
+			"Content-length": `${size}`,
+		},
+		body: createReadStream(file),
+		duplex: "half", // required for NodeJS streams over .fetch ?
+	});
+
+	if (uploadResponse.status !== 200) {
+		throw new UserError(
+			`File could not be uploaded. Please retry.\nGot response: ${await uploadResponse.text()}`
+		);
+	}
+
+	const etagResponse = uploadResponse.headers.get("etag");
+	if (!etagResponse) {
+		throw new UserError(`File did not upload successfully. Please retry.`);
+	}
+	if (etag !== etagResponse.replace(/^"|"$/g, "")) {
+		throw new UserError(
+			`File contents did not upload successfully. Please retry.`
+		);
+	}
+	logger.log(`🌀 Uploading complete.`);
+
+	return await d1ApiPost<
+		ImportPollingResponse | { success: false; error: string }
+	>(accountId, db, "import", { action: "ingest", filename, etag });
+}
+
+async function pollUntilComplete(
+	response: ImportPollingResponse | { success: false; error: string },
+	accountId: string,
+	db: Database
+): Promise<ImportPollingResponse> {
+	if (!response.success) throw new Error(response.error);
+
+	response.messages.forEach((line) => {
+		logger.log(`🌀 ${line}`);
+	});
+
+	if (response.status === "complete") {
+		return response;
+	} else if (response.status === "error") {
+		throw new APIError({
+			text: response.errors?.join("\n"),
+			notes: response.messages.map((text) => ({ text })),
+		});
+	} else {
+		const newResponse = await d1ApiPost<
+			ImportPollingResponse | { success: false; error: string }
+		>(accountId, db, "import", {
+			action: "poll",
+			currentBookmark: response.at_bookmark,
+		});
+		return await pollUntilComplete(newResponse, accountId, db);
+	}
+}
+
+async function d1ApiPost<T>(
+	accountId: string,
+	db: Database,
+	action: string,
+	body: unknown
+) {
+	return await fetchResult<T>(
+		`/accounts/${accountId}/d1/database/${db.uuid}/${action}`,
+		{
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+				...(db.internal_env ? { "x-d1-internal-env": db.internal_env } : {}),
+			},
+			body: JSON.stringify(body),
+		}
+	);
+}
+
 function logResult(r: QueryResult | QueryResult[]) {
 	logger.log(
 		`🚣 Executed ${
diff --git a/packages/wrangler/src/d1/types.ts b/packages/wrangler/src/d1/types.ts
index 7d76d3c2a4..bc3c147d3b 100644
--- a/packages/wrangler/src/d1/types.ts
+++ b/packages/wrangler/src/d1/types.ts
@@ -104,3 +104,38 @@ export interface D1QueriesGraphQLResponse {
 		};
 	};
 }
+
+export type ImportInitResponse = {
+	success: true;
+	filename: string;
+	uploadUrl: string;
+};
+export type ImportPollingResponse = {
+	success: true;
+	type: "import";
+	at_bookmark: string;
+	messages: string[];
+	errors: string[];
+} & (
+	| {
+			status: "active" | "error";
+	  }
+	| {
+			status: "complete";
+			result: {
+				success: boolean;
+				finalBookmark: string;
+				numQueries: number;
+				meta: {
+					served_by: string;
+					duration: number;
+					changes: number;
+					last_row_id: number;
+					changed_db: boolean;
+					size_after: number;
+					rows_read: number;
+					rows_written: number;
+				};
+			};
+	  }
+);

From 556f95f37de5244e03f66814acca197b322e118e Mon Sep 17 00:00:00 2001
From: Glen Maddern <glen@cloudflare.com>
Date: Fri, 10 May 2024 10:23:36 +1000
Subject: [PATCH 03/12] D1 export: make --no-schema/--no-data work

Fixes https://github.com/cloudflare/workers-sdk/issues/5445
---
 packages/wrangler/src/d1/export.ts | 89 ++++++++++++++++++------------
 1 file changed, 53 insertions(+), 36 deletions(-)

diff --git a/packages/wrangler/src/d1/export.ts b/packages/wrangler/src/d1/export.ts
index ac4475b305..5c3a53d137 100644
--- a/packages/wrangler/src/d1/export.ts
+++ b/packages/wrangler/src/d1/export.ts
@@ -18,42 +18,56 @@ import type {
 import type { Database } from "./types";
 
 export function Options(yargs: CommonYargsArgv) {
-	return Name(yargs)
-		.option("local", {
-			type: "boolean",
-			describe: "Export from your local DB you use with wrangler dev",
-			conflicts: "remote",
-		})
-		.option("remote", {
-			type: "boolean",
-			describe: "Export from your live D1",
-			conflicts: "local",
-		})
-		.option("no-schema", {
-			type: "boolean",
-			describe: "Only output table contents, not the DB schema",
-			conflicts: "no-data",
-		})
-		.option("no-data", {
-			type: "boolean",
-			describe:
-				"Only output table schema, not the contents of the DBs themselves",
-			conflicts: "no-schema",
-		})
-		.option("table", {
-			type: "string",
-			describe: "Specify which tables to include in export",
-		})
-		.option("output", {
-			type: "string",
-			describe: "Which .sql file to output to",
-			demandOption: true,
-		});
+	return (
+		Name(yargs)
+			.option("local", {
+				type: "boolean",
+				describe: "Export from your local DB you use with wrangler dev",
+				conflicts: "remote",
+			})
+			.option("remote", {
+				type: "boolean",
+				describe: "Export from your live D1",
+				conflicts: "local",
+			})
+			.option("no-schema", {
+				type: "boolean",
+				describe: "Only output table contents, not the DB schema",
+				conflicts: "no-data",
+			})
+			.option("no-data", {
+				type: "boolean",
+				describe:
+					"Only output table schema, not the contents of the DBs themselves",
+				conflicts: "no-schema",
+			})
+			// For --no-schema and --no-data to work, we need their positive versions
+			// to be defined. But keep them hidden as they default to true
+			.option("schema", {
+				type: "boolean",
+				hidden: true,
+				default: true,
+			})
+			.option("data", {
+				type: "boolean",
+				hidden: true,
+				default: true,
+			})
+			.option("table", {
+				type: "string",
+				describe: "Specify which tables to include in export",
+			})
+			.option("output", {
+				type: "string",
+				describe: "Which .sql file to output to",
+				demandOption: true,
+			})
+	);
 }
 
 type HandlerOptions = StrictYargsOptionsToInterface<typeof Options>;
 export const Handler = async (args: HandlerOptions): Promise<void> => {
-	const { local, remote, name, output, noSchema, noData, table } = args;
+	const { local, remote, name, output, schema, data, table } = args;
 	await printWranglerBanner();
 	const config = readConfig(args.config, args);
 
@@ -66,6 +80,9 @@ export const Handler = async (args: HandlerOptions): Promise<void> => {
 		throw new UserError(`You must specify either --local or --remote`);
 	}
 
+	if (!schema && !data)
+		throw new UserError(`You cannot specify both --no-schema and --no-data`);
+
 	// Allow multiple --table x --table y flags or none
 	const tables: string[] = table
 		? Array.isArray(table)
@@ -78,8 +95,8 @@ export const Handler = async (args: HandlerOptions): Promise<void> => {
 		name,
 		output,
 		tables,
-		noSchema,
-		noData
+		!schema,
+		!data
 	);
 	return result;
 };
@@ -105,8 +122,8 @@ async function exportRemotely(
 	name: string,
 	output: string,
 	tables: string[],
-	noSchema?: boolean,
-	noData?: boolean
+	noSchema: boolean,
+	noData: boolean
 ) {
 	const accountId = await requireAuth(config);
 	const db: Database = await getDatabaseByNameOrBinding(

From 12fc12332b216a1e187093541a1526488f4f5a0d Mon Sep 17 00:00:00 2001
From: Glen Maddern <glen@cloudflare.com>
Date: Fri, 10 May 2024 13:25:27 +1000
Subject: [PATCH 04/12] Extracted PollingFailure type

---
 packages/wrangler/src/d1/execute.tsx | 31 ++++++++++++---------
 packages/wrangler/src/d1/export.ts   | 41 +++++++++-------------------
 packages/wrangler/src/d1/types.ts    | 18 ++++++++++++
 3 files changed, 49 insertions(+), 41 deletions(-)

diff --git a/packages/wrangler/src/d1/execute.tsx b/packages/wrangler/src/d1/execute.tsx
index a5cbf42ab8..f8130686ca 100644
--- a/packages/wrangler/src/d1/execute.tsx
+++ b/packages/wrangler/src/d1/execute.tsx
@@ -31,6 +31,7 @@ import type {
 	Database,
 	ImportInitResponse,
 	ImportPollingResponse,
+	PollingFailure,
 } from "./types";
 import type { D1Result } from "@cloudflare/workers-types/experimental";
 
@@ -385,9 +386,7 @@ async function executeRemotely({
 		);
 
 		const initResponse = await d1ApiPost<
-			| ImportInitResponse
-			| ImportPollingResponse
-			| { success: false; error: string }
+			ImportInitResponse | ImportPollingResponse | PollingFailure
 		>(accountId, db, "import", { action: "init", etag });
 
 		// An init response usually returns a {filename, uploadUrl} pair, except if we've detected that file
@@ -476,13 +475,16 @@ async function uploadAndBeginIngestion(
 	}
 	logger.log(`🌀 Uploading complete.`);
 
-	return await d1ApiPost<
-		ImportPollingResponse | { success: false; error: string }
-	>(accountId, db, "import", { action: "ingest", filename, etag });
+	return await d1ApiPost<ImportPollingResponse | PollingFailure>(
+		accountId,
+		db,
+		"import",
+		{ action: "ingest", filename, etag }
+	);
 }
 
 async function pollUntilComplete(
-	response: ImportPollingResponse | { success: false; error: string },
+	response: ImportPollingResponse | PollingFailure,
 	accountId: string,
 	db: Database
 ): Promise<ImportPollingResponse> {
@@ -500,12 +502,15 @@ async function pollUntilComplete(
 			notes: response.messages.map((text) => ({ text })),
 		});
 	} else {
-		const newResponse = await d1ApiPost<
-			ImportPollingResponse | { success: false; error: string }
-		>(accountId, db, "import", {
-			action: "poll",
-			currentBookmark: response.at_bookmark,
-		});
+		const newResponse = await d1ApiPost<ImportPollingResponse | PollingFailure>(
+			accountId,
+			db,
+			"import",
+			{
+				action: "poll",
+				currentBookmark: response.at_bookmark,
+			}
+		);
 		return await pollUntilComplete(newResponse, accountId, db);
 	}
 }
diff --git a/packages/wrangler/src/d1/export.ts b/packages/wrangler/src/d1/export.ts
index 5c3a53d137..577918d0d4 100644
--- a/packages/wrangler/src/d1/export.ts
+++ b/packages/wrangler/src/d1/export.ts
@@ -15,7 +15,7 @@ import type {
 	CommonYargsArgv,
 	StrictYargsOptionsToInterface,
 } from "../yargs-types";
-import type { Database } from "./types";
+import type { Database, ExportPollingResponse, PollingFailure } from "./types";
 
 export function Options(yargs: CommonYargsArgv) {
 	return (
@@ -101,22 +101,6 @@ export const Handler = async (args: HandlerOptions): Promise<void> => {
 	return result;
 };
 
-type PollingResponse = {
-	success: true;
-	type: "export";
-	at_bookmark: string;
-	messages: string[];
-	errors: string[];
-} & (
-	| {
-			status: "active" | "error";
-	  }
-	| {
-			status: "complete";
-			result: { filename: string; signedUrl: string };
-	  }
-);
-
 async function exportRemotely(
 	config: Config,
 	name: string,
@@ -167,17 +151,18 @@ async function pollExport(
 	},
 	currentBookmark: string | undefined,
 	num_parts_uploaded = 0
-): Promise<PollingResponse> {
-	const response = await fetchResult<
-		PollingResponse | { success: false; error: string }
-	>(`/accounts/${accountId}/d1/database/${db.uuid}/export`, {
-		method: "POST",
-		body: JSON.stringify({
-			outputFormat: "polling",
-			dumpOptions,
-			currentBookmark,
-		}),
-	});
+): Promise<ExportPollingResponse> {
+	const response = await fetchResult<ExportPollingResponse | PollingFailure>(
+		`/accounts/${accountId}/d1/database/${db.uuid}/export`,
+		{
+			method: "POST",
+			body: JSON.stringify({
+				outputFormat: "polling",
+				dumpOptions,
+				currentBookmark,
+			}),
+		}
+	);
 
 	if (!response.success) {
 		throw new Error(response.error);
diff --git a/packages/wrangler/src/d1/types.ts b/packages/wrangler/src/d1/types.ts
index bc3c147d3b..064328e02e 100644
--- a/packages/wrangler/src/d1/types.ts
+++ b/packages/wrangler/src/d1/types.ts
@@ -139,3 +139,21 @@ export type ImportPollingResponse = {
 			};
 	  }
 );
+
+export type ExportPollingResponse = {
+	success: true;
+	type: "export";
+	at_bookmark: string;
+	messages: string[];
+	errors: string[];
+} & (
+	| {
+			status: "active" | "error";
+	  }
+	| {
+			status: "complete";
+			result: { filename: string; signedUrl: string };
+	  }
+);
+
+export type PollingFailure = { success: false; error: string };

From 3d8dd6f4af11e659b0ab0173eacefe53d4c54fca Mon Sep 17 00:00:00 2001
From: Glen Maddern <glen@cloudflare.com>
Date: Fri, 10 May 2024 17:57:56 +1000
Subject: [PATCH 05/12] Fixing json mode for remote DB execution

---
 packages/wrangler/src/d1/execute.tsx | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/packages/wrangler/src/d1/execute.tsx b/packages/wrangler/src/d1/execute.tsx
index f8130686ca..42eb68a710 100644
--- a/packages/wrangler/src/d1/execute.tsx
+++ b/packages/wrangler/src/d1/execute.tsx
@@ -126,7 +126,7 @@ export const Handler = async (args: HandlerOptions): Promise<void> => {
 			remote,
 			config,
 			name: database,
-			shouldPrompt: isInteractive && !yes,
+			shouldPrompt: isInteractive && !yes && !json,
 			persistTo,
 			file,
 			command,
@@ -422,12 +422,24 @@ async function executeRemotely({
 			)} seconds (${meta.rows_read} rows read, ${
 				meta.rows_written
 			} rows written)\n` +
-				`   Database is currently ${(meta.size_after / 1_000_000).toFixed(
-					2
-				)}MB (at bookmark ${chalk.gray(finalBookmark)}).`
+				chalk.gray(`   Database is currently at bookmark ${finalBookmark}.`)
 		);
 
-		return null;
+		return [
+			{
+				results: [
+					{
+						"Total queries executed": numQueries,
+						"Rows read": meta.rows_read,
+						"Rows written": meta.rows_written,
+						"Databas size (MB)": (meta.size_after / 1_000_000).toFixed(2),
+					},
+				],
+				success: true,
+				finalBookmark,
+				meta,
+			},
+		];
 	} else {
 		const result = await d1ApiPost<QueryResult[]>(accountId, db, "query", {
 			sql: input.command,

From 575409f9f0d19524a28fbd3a7a1db551e544cf6d Mon Sep 17 00:00:00 2001
From: Glen Maddern <glen83@gmail.com>
Date: Sat, 11 May 2024 06:40:11 +0100
Subject: [PATCH 06/12] Updates based on PR feedback

---
 packages/wrangler/src/d1/execute.tsx | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/wrangler/src/d1/execute.tsx b/packages/wrangler/src/d1/execute.tsx
index 42eb68a710..67926647f9 100644
--- a/packages/wrangler/src/d1/execute.tsx
+++ b/packages/wrangler/src/d1/execute.tsx
@@ -231,7 +231,7 @@ export async function executeSql({
 	}
 	if (persistTo && !local) {
 		throw new UserError(`Error: can't use --persist-to without --local`);
-  }
+	}
 	if (input.file) await checkForSQLiteBinary(input.file);
 
 	const result =
@@ -242,13 +242,13 @@ export async function executeSql({
 					shouldPrompt,
 					input,
 					preview,
-				})
+			  })
 			: await executeLocally({
 					config,
 					name,
 					input,
 					persistTo,
-				});
+			  });
 
 	if (json) {
 		logger.loggerLevel = existingLogLevel;

From a16f5fc610f49e7423039df23a38224b61a3116f Mon Sep 17 00:00:00 2001
From: Glen Maddern <glen83@gmail.com>
Date: Sat, 11 May 2024 06:40:11 +0100
Subject: [PATCH 07/12] Updates based on PR feedback

---
 packages/wrangler/package.json       |  2 +-
 packages/wrangler/src/d1/execute.tsx | 15 +++++++++------
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/packages/wrangler/package.json b/packages/wrangler/package.json
index 62052232ad..2f112cb62b 100644
--- a/packages/wrangler/package.json
+++ b/packages/wrangler/package.json
@@ -173,7 +173,7 @@
 		"jest": "^29.7.0",
 		"jest-fetch-mock": "^3.0.3",
 		"jest-websocket-mock": "^2.5.0",
-		"md5-file": "^5.0.0",
+		"md5-file": "5.0.0",
 		"mime": "^3.0.0",
 		"minimatch": "^5.1.0",
 		"msw": "^0.49.1",
diff --git a/packages/wrangler/src/d1/execute.tsx b/packages/wrangler/src/d1/execute.tsx
index 67926647f9..8a48bab373 100644
--- a/packages/wrangler/src/d1/execute.tsx
+++ b/packages/wrangler/src/d1/execute.tsx
@@ -89,6 +89,7 @@ export function Options(yargs: CommonYargsArgv) {
 			describe: "Number of queries to send in a single batch",
 			type: "number",
 			deprecated: true,
+			hidden: true
 		});
 }
 
@@ -340,13 +341,13 @@ async function executeRemotely({
 	preview: boolean | undefined;
 }) {
 	if (input.file) {
-		const warning = `⚠️ This process may take some time, during which your D1 will be unavailable to serve queries.`;
+		const warning = `⚠️ This process may take some time, during which your D1 database will be unavailable to serve queries.`;
 
 		if (shouldPrompt) {
 			const ok = await confirm(`${warning}\n  Ok to proceed?`);
 			if (!ok) return null;
 		} else {
-			logger.log(warning);
+			logger.warn(warning);
 		}
 	}
 
@@ -358,11 +359,9 @@ async function executeRemotely({
 	);
 	if (preview) {
 		if (!db.previewDatabaseUuid) {
-			const error = new UserError(
+			throw new UserError(
 				"Please define a `preview_database_id` in your wrangler.toml to execute your queries against a preview database"
 			);
-			logger.error(error.message);
-			throw error;
 		}
 		db.uuid = db.previewDatabaseUuid;
 	}
@@ -395,7 +394,9 @@ async function executeRemotely({
 		const uploadRequired = "uploadUrl" in initResponse;
 		if (!uploadRequired) logger.log(`🌀 File already uploaded. Processing.`);
 		const firstPollResponse = uploadRequired
-			? await uploadAndBeginIngestion(
+			? // Upload the file to R2, then inform D1 to start processing it. The server delays before responding
+			  // in case the file is quite small and can be processed without a second round-trip.
+			  await uploadAndBeginIngestion(
 					accountId,
 					db,
 					input.file,
@@ -404,6 +405,8 @@ async function executeRemotely({
 			  )
 			: initResponse;
 
+		// If the file takes longer than the specified delay (~1s) to import, we'll need to continue polling
+		// until it's complete. If it's already finished, this call will early-exit.
 		const finalResponse = await pollUntilComplete(
 			firstPollResponse,
 			accountId,

From 9dae146088bc4b8184e9b2ad3ced8b8d11d2f042 Mon Sep 17 00:00:00 2001
From: Glen Maddern <glen@cloudflare.com>
Date: Tue, 14 May 2024 20:28:42 +1000
Subject: [PATCH 08/12] Added test to cover binary sqlite3 file check

---
 .../wrangler/src/__tests__/d1/execute.test.ts |  31 ++++++++++++++++++
 .../src/__tests__/d1/fixtures/db.sqlite3      | Bin 0 -> 4096 bytes
 packages/wrangler/src/d1/execute.tsx          |   4 +--
 3 files changed, 33 insertions(+), 2 deletions(-)
 create mode 100644 packages/wrangler/src/__tests__/d1/fixtures/db.sqlite3

diff --git a/packages/wrangler/src/__tests__/d1/execute.test.ts b/packages/wrangler/src/__tests__/d1/execute.test.ts
index 4ab325389c..d6941e383e 100644
--- a/packages/wrangler/src/__tests__/d1/execute.test.ts
+++ b/packages/wrangler/src/__tests__/d1/execute.test.ts
@@ -1,5 +1,10 @@
+import fs from "node:fs";
+import { join } from "path";
+import { rest } from "msw";
 import { mockConsoleMethods } from "../helpers/mock-console";
 import { useMockIsTTY } from "../helpers/mock-istty";
+import { mockGetMemberships, mockOAuthFlow } from "../helpers/mock-oauth-flow";
+import { msw } from "../helpers/msw";
 import { runInTempDir } from "../helpers/run-in-tmp";
 import { runWrangler } from "../helpers/run-wrangler";
 import writeWranglerToml from "../helpers/write-wrangler-toml";
@@ -7,6 +12,7 @@ import writeWranglerToml from "../helpers/write-wrangler-toml";
 describe("execute", () => {
 	mockConsoleMethods();
 	runInTempDir();
+	const { mockOAuthServerCallback } = mockOAuthFlow();
 	const { setIsTTY } = useMockIsTTY();
 
 	it("should require login when running against prod", async () => {
@@ -87,4 +93,29 @@ describe("execute", () => {
 			)
 		);
 	});
+
+	it("should reject a binary SQLite DB", async () => {
+		setIsTTY(false);
+		writeWranglerToml({
+			d1_databases: [
+				{ binding: "DATABASE", database_name: "db", database_id: "xxxx" },
+			],
+		});
+		const path = join(__dirname, "fixtures", "db.sqlite3");
+		fs.copyFileSync(path, "db.sqlite3");
+
+		await expect(
+			runWrangler(`d1 execute db --file db.sqlite3 --local --json`)
+		).rejects.toThrowError(
+			JSON.stringify(
+				{
+					error: {
+						text: "Provided file is a binary SQLite database file instead of an SQL text file. The execute command can only process SQL text files. Please export an SQL file from your SQLite database and try again.",
+					},
+				},
+				null,
+				2
+			)
+		);
+	});
 });
diff --git a/packages/wrangler/src/__tests__/d1/fixtures/db.sqlite3 b/packages/wrangler/src/__tests__/d1/fixtures/db.sqlite3
new file mode 100644
index 0000000000000000000000000000000000000000..2f7292d0901b002071ec05a1b6f78ffec474647f
GIT binary patch
literal 4096
zcmWFz^vNtqRY=P(%1ta$FlG>7U}9o$P*7lCU|@t|AVoG{WY8<?<^^dNAlr;ljiVtj
n8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*O6ovo*c9sVc

literal 0
HcmV?d00001

diff --git a/packages/wrangler/src/d1/execute.tsx b/packages/wrangler/src/d1/execute.tsx
index 8a48bab373..b55cf81f20 100644
--- a/packages/wrangler/src/d1/execute.tsx
+++ b/packages/wrangler/src/d1/execute.tsx
@@ -89,7 +89,7 @@ export function Options(yargs: CommonYargsArgv) {
 			describe: "Number of queries to send in a single batch",
 			type: "number",
 			deprecated: true,
-			hidden: true
+			hidden: true,
 		});
 }
 
@@ -575,7 +575,7 @@ async function checkForSQLiteBinary(filename: string) {
 	await fd.read(buffer, 0, 15);
 	if (buffer.toString("utf8") === "SQLite format 3") {
 		throw new UserError(
-			"Provided file is a binary SQLite database file instead of an SQL text file.\nThe execute command can only process SQL text files.\nPlease export an SQL file from your SQLite database and try again."
+			"Provided file is a binary SQLite database file instead of an SQL text file. The execute command can only process SQL text files. Please export an SQL file from your SQLite database and try again."
 		);
 	}
 }

From f6d868552c8f97d8e80cf810cab5a5493178b87d Mon Sep 17 00:00:00 2001
From: Glen Maddern <glen@cloudflare.com>
Date: Tue, 14 May 2024 20:31:26 +1000
Subject: [PATCH 09/12] Added changeset

---
 .changeset/fluffy-files-brush.md | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 .changeset/fluffy-files-brush.md

diff --git a/.changeset/fluffy-files-brush.md b/.changeset/fluffy-files-brush.md
new file mode 100644
index 0000000000..643ea0205a
--- /dev/null
+++ b/.changeset/fluffy-files-brush.md
@@ -0,0 +1,5 @@
+---
+"wrangler": minor
+---
+
+Improved `d1 execute --file --remote` performance & added support for much larger SQL files within a single transaction.

From 23fbf2f09aed0a48ab65da061ddb29d3bc596ccc Mon Sep 17 00:00:00 2001
From: Glen Maddern <glen@cloudflare.com>
Date: Thu, 16 May 2024 07:27:59 +1000
Subject: [PATCH 10/12] Added spinners to the output during async operations

---
 .../wrangler/src/__tests__/d1/execute.test.ts |  4 ---
 packages/wrangler/src/d1/execute.tsx          | 30 +++++++++++--------
 pnpm-lock.yaml                                | 10 +++++++
 3 files changed, 28 insertions(+), 16 deletions(-)

diff --git a/packages/wrangler/src/__tests__/d1/execute.test.ts b/packages/wrangler/src/__tests__/d1/execute.test.ts
index d6941e383e..757fdb6853 100644
--- a/packages/wrangler/src/__tests__/d1/execute.test.ts
+++ b/packages/wrangler/src/__tests__/d1/execute.test.ts
@@ -1,10 +1,7 @@
 import fs from "node:fs";
 import { join } from "path";
-import { rest } from "msw";
 import { mockConsoleMethods } from "../helpers/mock-console";
 import { useMockIsTTY } from "../helpers/mock-istty";
-import { mockGetMemberships, mockOAuthFlow } from "../helpers/mock-oauth-flow";
-import { msw } from "../helpers/msw";
 import { runInTempDir } from "../helpers/run-in-tmp";
 import { runWrangler } from "../helpers/run-wrangler";
 import writeWranglerToml from "../helpers/write-wrangler-toml";
@@ -12,7 +9,6 @@ import writeWranglerToml from "../helpers/write-wrangler-toml";
 describe("execute", () => {
 	mockConsoleMethods();
 	runInTempDir();
-	const { mockOAuthServerCallback } = mockOAuthFlow();
 	const { setIsTTY } = useMockIsTTY();
 
 	it("should require login when running against prod", async () => {
diff --git a/packages/wrangler/src/d1/execute.tsx b/packages/wrangler/src/d1/execute.tsx
index b55cf81f20..af6bac6b69 100644
--- a/packages/wrangler/src/d1/execute.tsx
+++ b/packages/wrangler/src/d1/execute.tsx
@@ -1,6 +1,7 @@
 import { createReadStream, promises as fs } from "fs";
 import assert from "node:assert";
 import path from "node:path";
+import { spinnerWhile } from "@cloudflare/cli/interactive";
 import chalk from "chalk";
 import { Static, Text } from "ink";
 import Table from "ink-table";
@@ -384,9 +385,12 @@ async function executeRemotely({
 			)
 		);
 
-		const initResponse = await d1ApiPost<
-			ImportInitResponse | ImportPollingResponse | PollingFailure
-		>(accountId, db, "import", { action: "init", etag });
+		const initResponse = await spinnerWhile({
+			promise: d1ApiPost<
+				ImportInitResponse | ImportPollingResponse | PollingFailure
+			>(accountId, db, "import", { action: "init", etag }),
+			startMessage: "Checking if file needs uploading",
+		});
 
 		// An init response usually returns a {filename, uploadUrl} pair, except if we've detected that file
 		// already exists and is valid, to save people reuploading. In which case `initResponse` has already
@@ -460,17 +464,20 @@ async function uploadAndBeginIngestion(
 	initResponse: ImportInitResponse
 ) {
 	const { uploadUrl, filename } = initResponse;
-	logger.log(`🌀 Uploading ${filename}...`);
 
 	const { size } = await fs.stat(file);
 
-	const uploadResponse = await fetch(uploadUrl, {
-		method: "PUT",
-		headers: {
-			"Content-length": `${size}`,
-		},
-		body: createReadStream(file),
-		duplex: "half", // required for NodeJS streams over .fetch ?
+	const uploadResponse = await spinnerWhile({
+		promise: fetch(uploadUrl, {
+			method: "PUT",
+			headers: {
+				"Content-length": `${size}`,
+			},
+			body: createReadStream(file),
+			duplex: "half", // required for NodeJS streams over .fetch ?
+		}),
+		startMessage: `🌀 Uploading ${filename}`,
+		endMessage: `🌀 Uploading complete.`,
 	});
 
 	if (uploadResponse.status !== 200) {
@@ -488,7 +495,6 @@ async function uploadAndBeginIngestion(
 			`File contents did not upload successfully. Please retry.`
 		);
 	}
-	logger.log(`🌀 Uploading complete.`);
 
 	return await d1ApiPost<ImportPollingResponse | PollingFailure>(
 		accountId,
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 7f2c90eefd..779cc7df98 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1745,6 +1745,9 @@ importers:
       jest-websocket-mock:
         specifier: ^2.5.0
         version: 2.5.0
+      md5-file:
+        specifier: 5.0.0
+        version: 5.0.0
       mime:
         specifier: ^3.0.0
         version: 3.0.0
@@ -8673,6 +8676,11 @@ packages:
     resolution: {integrity: sha512-s2EMBOWtXFc8dgqvoAzKJXxNHibcdJMV0gwqKUaw9E2JBJuGUK7DrNKrA6g/i+v72TT16+6sVm5mS3thaMLQUw==}
     engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
 
+  md5-file@5.0.0:
+    resolution: {integrity: sha512-xbEFXCYVWrSx/gEKS1VPlg84h/4L20znVIulKw6kMfmBUAZNAnF00eczz9ICMl+/hjQGo5KSXRxbL/47X3rmMw==}
+    engines: {node: '>=10.13.0'}
+    hasBin: true
+
   md5-hex@3.0.1:
     resolution: {integrity: sha512-BUiRtTtV39LIJwinWBjqVsU9xhdnz7/i889V859IBFpuqGAj6LuOvHv5XLbgZ2R7ptJoJaEcxkv88/h25T7Ciw==}
     engines: {node: '>=8'}
@@ -20594,6 +20602,8 @@ snapshots:
     dependencies:
       escape-string-regexp: 5.0.0
 
+  md5-file@5.0.0: {}
+
   md5-hex@3.0.1:
     dependencies:
       blueimp-md5: 2.19.0

From 3b0c941ea6f79642f94731429b9284da5c527da0 Mon Sep 17 00:00:00 2001
From: Glen Maddern <glen@cloudflare.com>
Date: Thu, 16 May 2024 07:38:04 +1000
Subject: [PATCH 11/12] Linting

---
 packages/wrangler/src/d1/execute.tsx | 37 ++++++++++++++++++----------
 packages/wrangler/src/d1/export.ts   |  3 ++-
 2 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/packages/wrangler/src/d1/execute.tsx b/packages/wrangler/src/d1/execute.tsx
index af6bac6b69..1dee76964b 100644
--- a/packages/wrangler/src/d1/execute.tsx
+++ b/packages/wrangler/src/d1/execute.tsx
@@ -220,9 +220,11 @@ export async function executeSql({
 	const input = file
 		? ({ file } as ExecuteInput)
 		: command
-		? ({ command } as ExecuteInput)
-		: null;
-	if (!input) throw new UserError(`Error: must provide --command or --file.`);
+			? ({ command } as ExecuteInput)
+			: null;
+	if (!input) {
+		throw new UserError(`Error: must provide --command or --file.`);
+	}
 	if (local && remote) {
 		throw new UserError(
 			`Error: can't use --local and --remote at the same time`
@@ -234,7 +236,9 @@ export async function executeSql({
 	if (persistTo && !local) {
 		throw new UserError(`Error: can't use --persist-to without --local`);
 	}
-	if (input.file) await checkForSQLiteBinary(input.file);
+	if (input.file) {
+		await checkForSQLiteBinary(input.file);
+	}
 
 	const result =
 		remote || preview
@@ -244,13 +248,13 @@ export async function executeSql({
 					shouldPrompt,
 					input,
 					preview,
-			  })
+				})
 			: await executeLocally({
 					config,
 					name,
 					input,
 					persistTo,
-			  });
+				});
 
 	if (json) {
 		logger.loggerLevel = existingLogLevel;
@@ -346,7 +350,9 @@ async function executeRemotely({
 
 		if (shouldPrompt) {
 			const ok = await confirm(`${warning}\n  Ok to proceed?`);
-			if (!ok) return null;
+			if (!ok) {
+				return null;
+			}
 		} else {
 			logger.warn(warning);
 		}
@@ -396,17 +402,19 @@ async function executeRemotely({
 		// already exists and is valid, to save people reuploading. In which case `initResponse` has already
 		// kicked the import process off.
 		const uploadRequired = "uploadUrl" in initResponse;
-		if (!uploadRequired) logger.log(`🌀 File already uploaded. Processing.`);
+		if (!uploadRequired) {
+			logger.log(`🌀 File already uploaded. Processing.`);
+		}
 		const firstPollResponse = uploadRequired
 			? // Upload the file to R2, then inform D1 to start processing it. The server delays before responding
-			  // in case the file is quite small and can be processed without a second round-trip.
-			  await uploadAndBeginIngestion(
+				// in case the file is quite small and can be processed without a second round-trip.
+				await uploadAndBeginIngestion(
 					accountId,
 					db,
 					input.file,
 					etag,
 					initResponse
-			  )
+				)
 			: initResponse;
 
 		// If the file takes longer than the specified delay (~1s) to import, we'll need to continue polling
@@ -417,8 +425,9 @@ async function executeRemotely({
 			db
 		);
 
-		if (finalResponse.status !== "complete")
+		if (finalResponse.status !== "complete") {
 			throw new APIError({ text: `D1 reset before execute completed!` });
+		}
 
 		const {
 			result: { numQueries, finalBookmark, meta },
@@ -509,7 +518,9 @@ async function pollUntilComplete(
 	accountId: string,
 	db: Database
 ): Promise<ImportPollingResponse> {
-	if (!response.success) throw new Error(response.error);
+	if (!response.success) {
+		throw new Error(response.error);
+	}
 
 	response.messages.forEach((line) => {
 		logger.log(`🌀 ${line}`);
diff --git a/packages/wrangler/src/d1/export.ts b/packages/wrangler/src/d1/export.ts
index 577918d0d4..ac3dc9f4d1 100644
--- a/packages/wrangler/src/d1/export.ts
+++ b/packages/wrangler/src/d1/export.ts
@@ -80,8 +80,9 @@ export const Handler = async (args: HandlerOptions): Promise<void> => {
 		throw new UserError(`You must specify either --local or --remote`);
 	}
 
-	if (!schema && !data)
+	if (!schema && !data) {
 		throw new UserError(`You cannot specify both --no-schema and --no-data`);
+	}
 
 	// Allow multiple --table x --table y flags or none
 	const tables: string[] = table

From cf78a4078ddbe233292d6bbc49112364588d469e Mon Sep 17 00:00:00 2001
From: Rahul Sethi <5822355+RamIdeas@users.noreply.github.com>
Date: Thu, 16 May 2024 14:00:26 +0100
Subject: [PATCH 12/12] Update changeset

---
 .changeset/fluffy-files-brush.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.changeset/fluffy-files-brush.md b/.changeset/fluffy-files-brush.md
index 643ea0205a..29221c524d 100644
--- a/.changeset/fluffy-files-brush.md
+++ b/.changeset/fluffy-files-brush.md
@@ -2,4 +2,4 @@
 "wrangler": minor
 ---
 
-Improved `d1 execute --file --remote` performance & added support for much larger SQL files within a single transaction.
+feature: Improved `d1 execute --file --remote` performance & added support for much larger SQL files within a single transaction.