Python support #4901

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged

penalosa merged 8 commits into main from python-no-bundle

Feb 6, 2024

.changeset/smooth-goats-watch.md

-Original file line number
+Diff line change
@@ -0,0 +1,7 @@
+    ---
+    "wrangler": minor
+    ---
+    feature: implemented Python support in Wrangler
+    Python Workers are now supported by `wrangler deploy` and `wrangler dev`.

fixtures/python-worker/requirements.txt

Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		bcrypt==4.0.1

fixtures/python-worker/src/arith.py

Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		def mul(a,b):
		return a*b

fixtures/python-worker/src/index.py

-Original file line number
+Diff line change
@@ -0,0 +1,8 @@
+    from js import Response
+    from other import add
+    from arith import mul
+    import bcrypt
+    def fetch(request):
+      password = b"super secret password"
+      hashed = bcrypt.hashpw(password, bcrypt.gensalt(14))
+      return Response.new(f"Hi world {add(1,2)} {mul(2,3)} {hashed}")

fixtures/python-worker/src/other.py

Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		def add(a, b):
		return a + b

fixtures/python-worker/wrangler.toml

-Original file line number
+Diff line change
@@ -0,0 +1,4 @@
+    name = "dep-python-worker"
+    main = "src/index.py"
+    compatibility_flags = ["experimental"]
+    compatibility_date = "2024-01-29"

packages/wrangler/e2e/dev.test.ts

-Original file line number
+Diff line change
@@ Expand Up / @@ -268,6 +268,62 @@ describe("basic dev tests", () => { @@
     	});
     });
+    describe("basic dev python tests", () => {
+    	let worker: DevWorker;
+    	beforeEach(async () => {
+    		worker = await makeWorker();
+    		await worker.seed((workerName) => ({
+    			"wrangler.toml": dedent`
+    					name = "${workerName}"
+    					main = "index.py"
+    					compatibility_date = "2023-01-01"
+    					compatibility_flags = ["experimental"]
+    			`,
+    			"index.py": dedent`
+    				from js import Response
+    				def fetch(request):
+    					return Response.new('py hello world')`,
+    			"package.json": dedent`
+    					{
+    						"name": "${workerName}",
+    						"version": "0.0.0",
+    						"private": true
+    					}
+    					`,
+    		}));
+    	});
+    	it("can run and modify python worker during dev session (local)", async () => {
+    		await worker.runDevSession("", async (session) => {
+    			const { text } = await retry(
+    				(s) => s.status !== 200,
+    				async () => {
+    					const r = await fetch(`http://127.0.0.1:${session.port}`);
+    					return { text: await r.text(), status: r.status };
+    				}
+    			);
+    			expect(text).toMatchInlineSnapshot('"py hello world"');
+    			await worker.seed({
+    				"index.py": dedent`
+    					from js import Response
+    					def fetch(request):
+    						return Response.new('Updated Python Worker value')`,
+    			});
+    			const { text: text2 } = await retry(
+    				(s) => s.status !== 200 || s.text === "py hello world",
+    				async () => {
+    					const r = await fetch(`http://127.0.0.1:${session.port}`);
+    					return { text: await r.text(), status: r.status };
+    				}
+    			);
+    			expect(text2).toMatchInlineSnapshot('"Updated Python Worker value"');
+    		});
+    	});
+    });
     describe("dev registry", () => {
     	let a: DevWorker;
     	let b: DevWorker;
@@ Expand Down @@

packages/wrangler/src/__tests__/deploy.test.ts

-Original file line number
+Diff line change
@@ Expand Up / @@ -8775,6 +8775,72 @@ export default{ @@
     		});
     	});
+    	describe("python", () => {
+    		it("should upload python module defined in wrangler.toml", async () => {
+    			writeWranglerToml({
+    				main: "index.py",
+    			});
+    			await fs.promises.writeFile(
+    				"index.py",
+    				"from js import Response;\ndef fetch(request):\n return Response.new('hello')"
+    			);
+    			mockSubDomainRequest();
+    			mockUploadWorkerRequest({
+    				expectedMainModule: "index",
+    			});
+    			await runWrangler("deploy");
+    			expect(
+    				std.out.replace(
+    					/.wrangler\/tmp\/deploy-(.+)\/index.py/,
+    					".wrangler/tmp/deploy/index.py"
+    				)
+    			).toMatchInlineSnapshot(`
+    			"┌──────────────────────────────────────┬────────┬──────────┐
+    			│ Name                                 │ Type   │ Size     │
+    			├──────────────────────────────────────┼────────┼──────────┤
+    			│ .wrangler/tmp/deploy/index.py │ python │ xx KiB │
+    			└──────────────────────────────────────┴────────┴──────────┘
+    			Total Upload: xx KiB / gzip: xx KiB
+    			Uploaded test-name (TIMINGS)
+    			Published test-name (TIMINGS)
+    			  https://test-name.test-sub-domain.workers.dev
+    			Current Deployment ID: Galaxy-Class"
+    		`);
+    		});
+    		it("should upload python module specified in CLI args", async () => {
+    			writeWranglerToml();
+    			await fs.promises.writeFile(
+    				"index.py",
+    				"from js import Response;\ndef fetch(request):\n return Response.new('hello')"
+    			);
+    			mockSubDomainRequest();
+    			mockUploadWorkerRequest({
+    				expectedMainModule: "index",
+    			});
+    			await runWrangler("deploy index.py");
+    			expect(
+    				std.out.replace(
+    					/.wrangler\/tmp\/deploy-(.+)\/index.py/,
+    					".wrangler/tmp/deploy/index.py"
+    				)
+    			).toMatchInlineSnapshot(`
+    			"┌──────────────────────────────────────┬────────┬──────────┐
+    			│ Name                                 │ Type   │ Size     │
+    			├──────────────────────────────────────┼────────┼──────────┤
+    			│ .wrangler/tmp/deploy/index.py │ python │ xx KiB │
+    			└──────────────────────────────────────┴────────┴──────────┘
+    			Total Upload: xx KiB / gzip: xx KiB
+    			Uploaded test-name (TIMINGS)
+    			Published test-name (TIMINGS)
+    			  https://test-name.test-sub-domain.workers.dev
+    			Current Deployment ID: Galaxy-Class"
+    		`);
+    		});
+    	});
     	describe("hyperdrive", () => {
     		it("should upload hyperdrive bindings", async () => {
     			writeWranglerToml({
@@ Expand Down @@

packages/wrangler/src/config/environment.ts

-Original file line number
+Diff line change
@@ Expand Up / @@ -748,7 +748,9 @@ export type ConfigModuleRuleType = @@
     	| "CommonJS"
     	| "CompiledWasm"
     	| "Text"
-    	| "Data";
+    	| "Data"
+    	| "PythonModule"
+    	| "PythonRequirement";
     export type TailConsumer = {
     	/** The name of the service tail events will be forwarded to. */
@@ Expand Down @@

packages/wrangler/src/config/index.ts

-Original file line number
+Diff line change
@@ Expand Up / @@ -57,6 +57,17 @@ export function readConfig<CommandArgs>( @@
     		throw new UserError(diagnostics.renderErrors());
     	}
+    	const mainModule = "script" in args ? args.script : config.main;
+    	if (typeof mainModule === "string" && mainModule.endsWith(".py")) {
+    		// Workers with a python entrypoint should have bundling turned off, since all of Wrangler's bundling is JS/TS specific
+    		config.no_bundle = true;
+    		// Workers with a python entrypoint need module rules for "*.py". Add one automatically as a DX nicety
+    		if (!config.rules.some((rule) => rule.type === "PythonModule")) {
+    			config.rules.push({ type: "PythonModule", globs: ["**/*.py"] });
+    		}
+    	}
     	return config;
     }
@@ Expand Down @@

packages/wrangler/src/deploy/deploy.ts

-Original file line number
+Diff line change
@@ Expand Up / @@ -46,7 +46,11 @@ import type { @@
     	ZoneNameRoute,
     } from "../config/environment";
     import type { Entry } from "../deployment-bundle/entry";
-    import type { CfPlacement, CfWorkerInit } from "../deployment-bundle/worker";
+    import type {
+    	CfModuleType,
+    	CfPlacement,
+    	CfWorkerInit,
+    } from "../deployment-bundle/worker";
     import type { PutConsumerBody } from "../queues/client";
     import type { AssetPaths } from "../sites";
     import type { RetrieveSourceMapFunction } from "../sourcemap";
@@ Expand Down Expand Up @@
     		const worker: CfWorkerInit = {
     			name: scriptName,
     			main: {
-    				name: entryPointName,
+    				name: stripPySuffix(entryPointName, bundleType),
     				filePath: resolvedEntryPointPath,
     				content: content,
     				type: bundleType,
@@ Expand Down Expand Up @@
     	});
     }
+    // TODO(soon): workerd requires python modules to be named without a file extension
+    // We should remove this restriction
+    function stripPySuffix(modulePath: string, type?: CfModuleType) {
+    	if (type === "python" && modulePath.endsWith(".py")) {
+    		return modulePath.slice(0, -3);
+    	}
+    	return modulePath;
+    }
     async function noBundleWorker(
     	entry: Entry,
     	rules: Rule[],
@@ Expand All / @@ -1161,10 +1174,14 @@ async function noBundleWorker( @@
     		await writeAdditionalModules(modules, outDir);
     	}
+    	const bundleType = getBundleType(entry.format, entry.file);
     	return {
-    		modules,
+    		modules: modules.map((m) => ({
+    			...m,
+    			name: stripPySuffix(m.name, m.type),
+    		})),
     		dependencies: {} as { [path: string]: { bytesInOutput: number } },
     		resolvedEntryPointPath: entry.file,
-    		bundleType: getBundleType(entry.format),
+    		bundleType,
     	};
     }

packages/wrangler/src/deployment-bundle/bundle-type.ts

-Original file line number
+Diff line change
@@ -1,8 +1,14 @@
     import type { CfModuleType, CfScriptFormat } from "./worker";
     /**
-     * Compute the entry-point type from the bundle format.
+     * Compute the entry-point module type from the bundle format.
      */
-    export function getBundleType(format: CfScriptFormat): CfModuleType {
+    export function getBundleType(
+    	format: CfScriptFormat,
+    	file?: string
+    ): CfModuleType {
+    	if (file && file.endsWith(".py")) {
+    		return "python";
+    	}
     	return format === "modules" ? "esm" : "commonjs";
     }

packages/wrangler/src/deployment-bundle/create-worker-upload-form.ts

-Original file line number
+Diff line change
@@ Expand Up / @@ -25,6 +25,10 @@ export function toMimeType(type: CfModuleType): string { @@
     			return "application/octet-stream";
     		case "text":
     			return "text/plain";
+    		case "python":
+    			return "text/x-python";
+    		case "python-requirement":
+    			return "text/x-python-requirement";
     		default:
     			throw new TypeError("Unsupported module: " + type);
     	}
@@ Expand Down @@

packages/wrangler/src/deployment-bundle/find-additional-modules.ts

-Original file line number
+Diff line change
@@ Expand Up / @@ -4,6 +4,7 @@ import chalk from "chalk"; @@
     import globToRegExp from "glob-to-regexp";
     import { UserError } from "../errors";
     import { logger } from "../logger";
+    import { getBundleType } from "./bundle-type";
     import { RuleTypeToModuleType } from "./module-collection";
     import { parseRules } from "./rules";
     import type { Rule } from "../config/environment";
@@ Expand Down Expand Up / @@ -49,14 +50,47 @@ export async function findAdditionalModules( @@
     			name: m.name,
     		}));
+    	// Try to find a requirements.txt file
+    	const isPythonEntrypoint =
+    		getBundleType(entry.format, entry.file) === "python";
+    	if (isPythonEntrypoint) {
+    		try {
+    			const pythonRequirements = await readFile(
+    				path.resolve(entry.directory, "requirements.txt"),
+    				"utf-8"
+    			);
+    			// This is incredibly naive. However, it supports common syntax for requirements.txt
+    			for (const requirement of pythonRequirements.split("\n")) {
+    				const packageName = requirement.match(/^[^\d\W]\w*/);
+    				if (typeof packageName?.[0] === "string") {
+    					modules.push({
+    						type: "python-requirement",
+    						name: packageName?.[0],
+    						content: "",
+    						filePath: undefined,
+    					});
+    				}
+    			}
+    			// We don't care if a requirements.txt isn't found
+    		} catch (e) {
+    			logger.debug(
+    				"Python entrypoint detected, but no requirements.txt file found."
+    			);
+    		}
+    	}
     	if (modules.length > 0) {
     		logger.info(`Attaching additional modules:`);
     		logger.table(
     			modules.map(({ name, type, content }) => {
     				return {
     					Name: name,
     					Type: type ?? "",
-    					Size: `${(content.length / 1024).toFixed(2)} KiB`,
+    					Size:
+    						type === "python-requirement"
+    							? ""
+    							: `${(content.length / 1024).toFixed(2)} KiB`,
     				};
     			})
     		);
@@ Expand Down @@

packages/wrangler/src/deployment-bundle/guess-worker-format.ts

-Original file line number
+Diff line change
@@ Expand Up / @@ -20,6 +20,17 @@ export default async function guessWorkerFormat( @@
     	hint: CfScriptFormat | undefined,
     	tsconfig?: string | undefined
     ): Promise<CfScriptFormat> {
+    	const parsedEntryPath = path.parse(entryFile);
+    	if (parsedEntryPath.ext == ".py") {
+    		logger.warn(
+    			`The entrypoint ${path.relative(
+    				process.cwd(),
+    				entryFile
+    			)} defines a Python worker, support for Python workers is currently experimental.`
+    		);
+    		return "modules";
+    	}
     	const result = await esbuild.build({
     		...COMMON_ESBUILD_OPTIONS,
     		entryPoints: [entryFile],
@@ Expand Down @@

packages/wrangler/src/deployment-bundle/module-collection.ts

-Original file line number
+Diff line change
@@ Expand Up @@
     		CompiledWasm: "compiled-wasm",
     		Data: "buffer",
     		Text: "text",
+    		PythonModule: "python",
+    		PythonRequirement: "python-requirement",
     	};
     export const ModuleTypeToRuleType = flipObject(RuleTypeToModuleType);
@@ Expand Down @@

packages/wrangler/src/deployment-bundle/worker.ts

-Original file line number
+Diff line change
@@ Expand Up / @@ -14,7 +14,9 @@ export type CfModuleType = @@
     	| "commonjs"
     	| "compiled-wasm"
     	| "text"
-    	| "buffer";
+    	| "buffer"
+    	| "python"
+    	| "python-requirement";
     /**
      * An imported module.
@@ Expand Down @@

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Python support #4901

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!