diff --git a/.changeset/smooth-goats-watch.md b/.changeset/smooth-goats-watch.md new file mode 100644 index 000000000000..e430e76c641a --- /dev/null +++ b/.changeset/smooth-goats-watch.md @@ -0,0 +1,7 @@ +--- +"wrangler": minor +--- + +feature: implemented Python support in Wrangler + +Python Workers are now supported by `wrangler deploy` and `wrangler dev`. diff --git a/fixtures/python-worker/requirements.txt b/fixtures/python-worker/requirements.txt new file mode 100644 index 000000000000..6b9ac5866230 --- /dev/null +++ b/fixtures/python-worker/requirements.txt @@ -0,0 +1 @@ +bcrypt==4.0.1 \ No newline at end of file diff --git a/fixtures/python-worker/src/arith.py b/fixtures/python-worker/src/arith.py new file mode 100644 index 000000000000..0bcf862adf20 --- /dev/null +++ b/fixtures/python-worker/src/arith.py @@ -0,0 +1,2 @@ +def mul(a,b): + return a*b diff --git a/fixtures/python-worker/src/index.py b/fixtures/python-worker/src/index.py new file mode 100644 index 000000000000..9177bde348dc --- /dev/null +++ b/fixtures/python-worker/src/index.py @@ -0,0 +1,8 @@ +from js import Response +from other import add +from arith import mul +import bcrypt +def fetch(request): + password = b"super secret password" + hashed = bcrypt.hashpw(password, bcrypt.gensalt(14)) + return Response.new(f"Hi world {add(1,2)} {mul(2,3)} {hashed}") diff --git a/fixtures/python-worker/src/other.py b/fixtures/python-worker/src/other.py new file mode 100644 index 000000000000..2a99cdfa95f0 --- /dev/null +++ b/fixtures/python-worker/src/other.py @@ -0,0 +1,2 @@ +def add(a, b): + return a + b diff --git a/fixtures/python-worker/wrangler.toml b/fixtures/python-worker/wrangler.toml new file mode 100644 index 000000000000..882e41bd6089 --- /dev/null +++ b/fixtures/python-worker/wrangler.toml @@ -0,0 +1,4 @@ +name = "dep-python-worker" +main = "src/index.py" +compatibility_flags = ["experimental"] +compatibility_date = "2024-01-29" \ No newline at end of file diff --git a/packages/wrangler/e2e/dev.test.ts b/packages/wrangler/e2e/dev.test.ts index faeb94c0e6a2..9b3b337df66b 100644 --- a/packages/wrangler/e2e/dev.test.ts +++ b/packages/wrangler/e2e/dev.test.ts @@ -268,6 +268,62 @@ describe("basic dev tests", () => { }); }); +describe("basic dev python tests", () => { + let worker: DevWorker; + + beforeEach(async () => { + worker = await makeWorker(); + await worker.seed((workerName) => ({ + "wrangler.toml": dedent` + name = "${workerName}" + main = "index.py" + compatibility_date = "2023-01-01" + compatibility_flags = ["experimental"] + `, + "index.py": dedent` + from js import Response + def fetch(request): + return Response.new('py hello world')`, + "package.json": dedent` + { + "name": "${workerName}", + "version": "0.0.0", + "private": true + } + `, + })); + }); + + it("can run and modify python worker during dev session (local)", async () => { + await worker.runDevSession("", async (session) => { + const { text } = await retry( + (s) => s.status !== 200, + async () => { + const r = await fetch(`http://127.0.0.1:${session.port}`); + return { text: await r.text(), status: r.status }; + } + ); + expect(text).toMatchInlineSnapshot('"py hello world"'); + + await worker.seed({ + "index.py": dedent` + from js import Response + def fetch(request): + return Response.new('Updated Python Worker value')`, + }); + + const { text: text2 } = await retry( + (s) => s.status !== 200 || s.text === "py hello world", + async () => { + const r = await fetch(`http://127.0.0.1:${session.port}`); + return { text: await r.text(), status: r.status }; + } + ); + expect(text2).toMatchInlineSnapshot('"Updated Python Worker value"'); + }); + }); +}); + describe("dev registry", () => { let a: DevWorker; let b: DevWorker; diff --git a/packages/wrangler/src/__tests__/deploy.test.ts b/packages/wrangler/src/__tests__/deploy.test.ts index 9729c4813960..f7a073c3755b 100644 --- a/packages/wrangler/src/__tests__/deploy.test.ts +++ b/packages/wrangler/src/__tests__/deploy.test.ts @@ -8775,6 +8775,72 @@ export default{ }); }); + describe("python", () => { + it("should upload python module defined in wrangler.toml", async () => { + writeWranglerToml({ + main: "index.py", + }); + await fs.promises.writeFile( + "index.py", + "from js import Response;\ndef fetch(request):\n return Response.new('hello')" + ); + mockSubDomainRequest(); + mockUploadWorkerRequest({ + expectedMainModule: "index", + }); + + await runWrangler("deploy"); + expect( + std.out.replace( + /.wrangler\/tmp\/deploy-(.+)\/index.py/, + ".wrangler/tmp/deploy/index.py" + ) + ).toMatchInlineSnapshot(` + "┌──────────────────────────────────────┬────────┬──────────┐ + │ Name │ Type │ Size │ + ├──────────────────────────────────────┼────────┼──────────┤ + │ .wrangler/tmp/deploy/index.py │ python │ xx KiB │ + └──────────────────────────────────────┴────────┴──────────┘ + Total Upload: xx KiB / gzip: xx KiB + Uploaded test-name (TIMINGS) + Published test-name (TIMINGS) + https://test-name.test-sub-domain.workers.dev + Current Deployment ID: Galaxy-Class" + `); + }); + + it("should upload python module specified in CLI args", async () => { + writeWranglerToml(); + await fs.promises.writeFile( + "index.py", + "from js import Response;\ndef fetch(request):\n return Response.new('hello')" + ); + mockSubDomainRequest(); + mockUploadWorkerRequest({ + expectedMainModule: "index", + }); + + await runWrangler("deploy index.py"); + expect( + std.out.replace( + /.wrangler\/tmp\/deploy-(.+)\/index.py/, + ".wrangler/tmp/deploy/index.py" + ) + ).toMatchInlineSnapshot(` + "┌──────────────────────────────────────┬────────┬──────────┐ + │ Name │ Type │ Size │ + ├──────────────────────────────────────┼────────┼──────────┤ + │ .wrangler/tmp/deploy/index.py │ python │ xx KiB │ + └──────────────────────────────────────┴────────┴──────────┘ + Total Upload: xx KiB / gzip: xx KiB + Uploaded test-name (TIMINGS) + Published test-name (TIMINGS) + https://test-name.test-sub-domain.workers.dev + Current Deployment ID: Galaxy-Class" + `); + }); + }); + describe("hyperdrive", () => { it("should upload hyperdrive bindings", async () => { writeWranglerToml({ diff --git a/packages/wrangler/src/config/environment.ts b/packages/wrangler/src/config/environment.ts index f614072b2625..b7057b43cdc5 100644 --- a/packages/wrangler/src/config/environment.ts +++ b/packages/wrangler/src/config/environment.ts @@ -748,7 +748,9 @@ export type ConfigModuleRuleType = | "CommonJS" | "CompiledWasm" | "Text" - | "Data"; + | "Data" + | "PythonModule" + | "PythonRequirement"; export type TailConsumer = { /** The name of the service tail events will be forwarded to. */ diff --git a/packages/wrangler/src/config/index.ts b/packages/wrangler/src/config/index.ts index 2a016ca1ba66..99219c330d48 100644 --- a/packages/wrangler/src/config/index.ts +++ b/packages/wrangler/src/config/index.ts @@ -57,6 +57,17 @@ export function readConfig( throw new UserError(diagnostics.renderErrors()); } + const mainModule = "script" in args ? args.script : config.main; + if (typeof mainModule === "string" && mainModule.endsWith(".py")) { + // Workers with a python entrypoint should have bundling turned off, since all of Wrangler's bundling is JS/TS specific + config.no_bundle = true; + + // Workers with a python entrypoint need module rules for "*.py". Add one automatically as a DX nicety + if (!config.rules.some((rule) => rule.type === "PythonModule")) { + config.rules.push({ type: "PythonModule", globs: ["**/*.py"] }); + } + } + return config; } diff --git a/packages/wrangler/src/deploy/deploy.ts b/packages/wrangler/src/deploy/deploy.ts index 5eab7f4b4cf3..602a37083233 100644 --- a/packages/wrangler/src/deploy/deploy.ts +++ b/packages/wrangler/src/deploy/deploy.ts @@ -46,7 +46,11 @@ import type { ZoneNameRoute, } from "../config/environment"; import type { Entry } from "../deployment-bundle/entry"; -import type { CfPlacement, CfWorkerInit } from "../deployment-bundle/worker"; +import type { + CfModuleType, + CfPlacement, + CfWorkerInit, +} from "../deployment-bundle/worker"; import type { PutConsumerBody } from "../queues/client"; import type { AssetPaths } from "../sites"; import type { RetrieveSourceMapFunction } from "../sourcemap"; @@ -616,7 +620,7 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m const worker: CfWorkerInit = { name: scriptName, main: { - name: entryPointName, + name: stripPySuffix(entryPointName, bundleType), filePath: resolvedEntryPointPath, content: content, type: bundleType, @@ -1151,6 +1155,15 @@ function updateQueueConsumers(config: Config): Promise[] { }); } +// TODO(soon): workerd requires python modules to be named without a file extension +// We should remove this restriction +function stripPySuffix(modulePath: string, type?: CfModuleType) { + if (type === "python" && modulePath.endsWith(".py")) { + return modulePath.slice(0, -3); + } + return modulePath; +} + async function noBundleWorker( entry: Entry, rules: Rule[], @@ -1161,10 +1174,14 @@ async function noBundleWorker( await writeAdditionalModules(modules, outDir); } + const bundleType = getBundleType(entry.format, entry.file); return { - modules, + modules: modules.map((m) => ({ + ...m, + name: stripPySuffix(m.name, m.type), + })), dependencies: {} as { [path: string]: { bytesInOutput: number } }, resolvedEntryPointPath: entry.file, - bundleType: getBundleType(entry.format), + bundleType, }; } diff --git a/packages/wrangler/src/deployment-bundle/bundle-type.ts b/packages/wrangler/src/deployment-bundle/bundle-type.ts index c261debc017c..f9923d08c0c0 100644 --- a/packages/wrangler/src/deployment-bundle/bundle-type.ts +++ b/packages/wrangler/src/deployment-bundle/bundle-type.ts @@ -1,8 +1,14 @@ import type { CfModuleType, CfScriptFormat } from "./worker"; /** - * Compute the entry-point type from the bundle format. + * Compute the entry-point module type from the bundle format. */ -export function getBundleType(format: CfScriptFormat): CfModuleType { +export function getBundleType( + format: CfScriptFormat, + file?: string +): CfModuleType { + if (file && file.endsWith(".py")) { + return "python"; + } return format === "modules" ? "esm" : "commonjs"; } diff --git a/packages/wrangler/src/deployment-bundle/create-worker-upload-form.ts b/packages/wrangler/src/deployment-bundle/create-worker-upload-form.ts index 554b82dc06e8..ee5dc6c5a70b 100644 --- a/packages/wrangler/src/deployment-bundle/create-worker-upload-form.ts +++ b/packages/wrangler/src/deployment-bundle/create-worker-upload-form.ts @@ -25,6 +25,10 @@ export function toMimeType(type: CfModuleType): string { return "application/octet-stream"; case "text": return "text/plain"; + case "python": + return "text/x-python"; + case "python-requirement": + return "text/x-python-requirement"; default: throw new TypeError("Unsupported module: " + type); } diff --git a/packages/wrangler/src/deployment-bundle/find-additional-modules.ts b/packages/wrangler/src/deployment-bundle/find-additional-modules.ts index 5259bbff4cdd..a00e69903668 100644 --- a/packages/wrangler/src/deployment-bundle/find-additional-modules.ts +++ b/packages/wrangler/src/deployment-bundle/find-additional-modules.ts @@ -4,6 +4,7 @@ import chalk from "chalk"; import globToRegExp from "glob-to-regexp"; import { UserError } from "../errors"; import { logger } from "../logger"; +import { getBundleType } from "./bundle-type"; import { RuleTypeToModuleType } from "./module-collection"; import { parseRules } from "./rules"; import type { Rule } from "../config/environment"; @@ -49,6 +50,36 @@ export async function findAdditionalModules( name: m.name, })); + // Try to find a requirements.txt file + const isPythonEntrypoint = + getBundleType(entry.format, entry.file) === "python"; + + if (isPythonEntrypoint) { + try { + const pythonRequirements = await readFile( + path.resolve(entry.directory, "requirements.txt"), + "utf-8" + ); + + // This is incredibly naive. However, it supports common syntax for requirements.txt + for (const requirement of pythonRequirements.split("\n")) { + const packageName = requirement.match(/^[^\d\W]\w*/); + if (typeof packageName?.[0] === "string") { + modules.push({ + type: "python-requirement", + name: packageName?.[0], + content: "", + filePath: undefined, + }); + } + } + // We don't care if a requirements.txt isn't found + } catch (e) { + logger.debug( + "Python entrypoint detected, but no requirements.txt file found." + ); + } + } if (modules.length > 0) { logger.info(`Attaching additional modules:`); logger.table( @@ -56,7 +87,10 @@ export async function findAdditionalModules( return { Name: name, Type: type ?? "", - Size: `${(content.length / 1024).toFixed(2)} KiB`, + Size: + type === "python-requirement" + ? "" + : `${(content.length / 1024).toFixed(2)} KiB`, }; }) ); diff --git a/packages/wrangler/src/deployment-bundle/guess-worker-format.ts b/packages/wrangler/src/deployment-bundle/guess-worker-format.ts index 3efc0df4ac42..a2f81419d319 100644 --- a/packages/wrangler/src/deployment-bundle/guess-worker-format.ts +++ b/packages/wrangler/src/deployment-bundle/guess-worker-format.ts @@ -20,6 +20,17 @@ export default async function guessWorkerFormat( hint: CfScriptFormat | undefined, tsconfig?: string | undefined ): Promise { + const parsedEntryPath = path.parse(entryFile); + if (parsedEntryPath.ext == ".py") { + logger.warn( + `The entrypoint ${path.relative( + process.cwd(), + entryFile + )} defines a Python worker, support for Python workers is currently experimental.` + ); + return "modules"; + } + const result = await esbuild.build({ ...COMMON_ESBUILD_OPTIONS, entryPoints: [entryFile], diff --git a/packages/wrangler/src/deployment-bundle/module-collection.ts b/packages/wrangler/src/deployment-bundle/module-collection.ts index c8ee2905b8f2..97218dee50d2 100644 --- a/packages/wrangler/src/deployment-bundle/module-collection.ts +++ b/packages/wrangler/src/deployment-bundle/module-collection.ts @@ -32,6 +32,8 @@ export const RuleTypeToModuleType: Record = CompiledWasm: "compiled-wasm", Data: "buffer", Text: "text", + PythonModule: "python", + PythonRequirement: "python-requirement", }; export const ModuleTypeToRuleType = flipObject(RuleTypeToModuleType); diff --git a/packages/wrangler/src/deployment-bundle/worker.ts b/packages/wrangler/src/deployment-bundle/worker.ts index 3c10dfa08729..2cc6b7b506ce 100644 --- a/packages/wrangler/src/deployment-bundle/worker.ts +++ b/packages/wrangler/src/deployment-bundle/worker.ts @@ -14,7 +14,9 @@ export type CfModuleType = | "commonjs" | "compiled-wasm" | "text" - | "buffer"; + | "buffer" + | "python" + | "python-requirement"; /** * An imported module. diff --git a/packages/wrangler/src/dev/miniflare.ts b/packages/wrangler/src/dev/miniflare.ts index 3fb8f577e834..54dd9c08b37d 100644 --- a/packages/wrangler/src/dev/miniflare.ts +++ b/packages/wrangler/src/dev/miniflare.ts @@ -1,6 +1,6 @@ import assert from "node:assert"; import { randomUUID } from "node:crypto"; -import { realpathSync } from "node:fs"; +import { readFileSync, realpathSync } from "node:fs"; import path from "node:path"; import { Log, LogLevel, Miniflare, Mutex, TypedEventTarget } from "miniflare"; import { AIFetcher } from "../ai/fetcher"; @@ -16,6 +16,7 @@ import type { CfDurableObject, CfHyperdrive, CfKvNamespace, + CfModuleType, CfQueue, CfR2Bucket, CfScriptFormat, @@ -173,29 +174,46 @@ function buildLog(): Log { return new WranglerLog(level, { prefix: "wrangler-UserWorker" }); } +// TODO(soon): workerd requires python modules to be named without a file extension +// We should remove this restriction +function stripPySuffix(modulePath: string, type?: CfModuleType) { + if (type === "python" && modulePath.endsWith(".py")) { + return modulePath.slice(0, -3); + } + return modulePath; +} + async function buildSourceOptions( config: ConfigBundle ): Promise { const scriptPath = realpathSync(config.bundle.path); if (config.format === "modules") { const modulesRoot = path.dirname(scriptPath); - const { entrypointSource, modules } = withSourceURLs( - scriptPath, - config.bundle.modules - ); + const { entrypointSource, modules } = + config.bundle.type === "python" + ? { + entrypointSource: readFileSync(scriptPath, "utf8"), + modules: config.bundle.modules, + } + : withSourceURLs(scriptPath, config.bundle.modules); + return { modulesRoot, + modules: [ // Entrypoint { - type: "ESModule", - path: scriptPath, + type: ModuleTypeToRuleType[config.bundle.type], + path: stripPySuffix(scriptPath, config.bundle.type), contents: entrypointSource, }, // Misc (WebAssembly, etc, ...) ...modules.map((module) => ({ type: ModuleTypeToRuleType[module.type ?? "esm"], - path: path.resolve(modulesRoot, module.name), + path: stripPySuffix( + path.resolve(modulesRoot, module.name), + module.type + ), contents: module.content, })), ], diff --git a/packages/wrangler/src/dev/use-esbuild.ts b/packages/wrangler/src/dev/use-esbuild.ts index 66aecb26bbdf..0c3af4824a15 100644 --- a/packages/wrangler/src/dev/use-esbuild.ts +++ b/packages/wrangler/src/dev/use-esbuild.ts @@ -213,7 +213,8 @@ export function useEsbuild({ id: 0, entry, path: bundleResult?.resolvedEntryPointPath ?? entry.file, - type: bundleResult?.bundleType ?? getBundleType(entry.format), + type: + bundleResult?.bundleType ?? getBundleType(entry.format, entry.file), modules: bundleResult ? bundleResult.modules : newAdditionalModules, dependencies: bundleResult?.dependencies ?? {}, sourceMapPath: bundleResult?.sourceMapPath, diff --git a/packages/wrangler/src/type-generation.ts b/packages/wrangler/src/type-generation.ts index a7f958043d44..876a3d53fe69 100644 --- a/packages/wrangler/src/type-generation.ts +++ b/packages/wrangler/src/type-generation.ts @@ -4,6 +4,7 @@ import { getEntry } from "./deployment-bundle/entry"; import { UserError } from "./errors"; import { logger } from "./logger"; import type { Config } from "./config"; +import type { CfScriptFormat } from "./deployment-bundle/worker"; // Currently includes bindings & rules for declaring modules @@ -147,7 +148,7 @@ function writeDTSFile({ }: { envTypeStructure: string[]; modulesTypeStructure: string[]; - formatType: "modules" | "service-worker"; + formatType: CfScriptFormat; }) { const wranglerOverrideDTSPath = findUpSync("worker-configuration.d.ts"); try {