From 0a0b0a3f6f7252c4060ea230860a173f67db29a6 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:49:20 +0000 Subject: [PATCH 1/3] upgrade-webkit: list the upstream commits that touch Bun's copies of WebCore files A WebKit upgrade merges upstream into the fork. It does not change the files under src/jsc/bindings that were copied from Source/WebCore, and the checklist had no step that looks at what upstream changed in them. scripts/webcore-copies.ts lists the upstream commits in a range that touch the upstream file of a copy, or the .idl of a JS.cpp binding. The checklist runs it and says what to do with the list. --- .claude/commands/upgrade-webkit.md | 7 + scripts/webcore-copies.ts | 332 +++++++++++++++ test/internal/webcore-copies.test.ts | 591 +++++++++++++++++++++++++++ 3 files changed, 930 insertions(+) create mode 100644 scripts/webcore-copies.ts create mode 100644 test/internal/webcore-copies.test.ts diff --git a/.claude/commands/upgrade-webkit.md b/.claude/commands/upgrade-webkit.md index 42bdc4b8261d..d1f755a20b1f 100644 --- a/.claude/commands/upgrade-webkit.md +++ b/.claude/commands/upgrade-webkit.md @@ -19,6 +19,13 @@ To do that: - Fix the merge conflicts (preserve the fork's Bun-specific changes) - bun run jsc:build:debug — from the bun repo root, builds just JSC (and its `jsc` shell), in bun's own build graph. File lists come from the checkout, but `scripts/build/deps/webkit.ts` restates WebKit's generator commands, compiler flags and option values by hand, and nothing checks them. Read `git diff $OLD_BASE upstream/main -- Source/cmake Source/JavaScriptCore/CMakeLists.txt Source/WTF/wtf/CMakeLists.txt Source/bmalloc/CMakeLists.txt` for changed `add_custom_command`s (→ `gen()`), compiler flags (→ `webkitFlags()`) and `WEBKIT_OPTION_*` / `SET_AND_EXPOSE_TO_BUILD` (→ `rows`), and carry them over in the bun PR - While it compiles, in another task review the JSC commits between $OLD_BASE and upstream/main (Source/JavaScriptCore, Source/WTF, Source/bmalloc). Write up a summary in a file called "webkit-changes.md" +- In the same task, run `bun scripts/webcore-copies.ts $OLD_BASE upstream/main` from the bun repo root. Pass the commit itself if the task's shell does not have `$OLD_BASE`. The script lists the upstream commits in that range that touch a file Bun has its own copy of. Bun does not build Source/WebCore: the files under src/jsc/bindings that come from it (most are in webcore/ and webcrypto/) were copied and edited by hand, and the merge does not change them. An upstream fix reaches a copy only when it is ported. The build is using the bun checkout: this task writes webkit-changes.md and, where a line below says so, `upstreamOf` in the script, and no other file + - For each commit, read `git -C vendor/WebKit show -- ` beside Bun's copy. A behavior change (a bug fix, a spec update) applies when the copy still has the code that it changes, even where the lines no longer match. If the title names a behavior, try it with the installed `bun`. A refactor that changes no behavior needs no port + - "% same" is how much of the copy's code its upstream file also has (all of its upstream files together for a copy in `upstreamOf`; a copy too short to measure has no number). It tells how well the diff lines up with the copy. It falls when either side changes, so a low value is no reason to pass over a commit + - "(binding)" marks Bun's `JS.cpp` for a changed `.idl`: compare the interface + - `upstreamOf` in the script names the upstream files of a copy that its file name does not lead to. When a commit moves code of a copy into another upstream file (it splits the file, or the list marks the file "(gone)" and does not list the copy under a new path), name both files for the copy there, the old one and the new one, and run the script again + - Not listed: changes to upstream's implementation of streams (src/jsc/bindings/webcore/streams is Bun's own; a changed stream `.idl` is listed as a binding), and WebCore code that Bun keeps in a file of another name unless `upstreamOf` names it + - In webkit-changes.md, write how many commits the script listed. For each commit that applies, write the number of the issue or PR that already covers it (search first), of the PR that ports it ("this PR" for a port in the upgrade PR), or of a new issue. Write "already in Bun" for a fix that Bun has in another form. Port with a test, after the build below has finished or in a PR of its own - bun run build:local — full Bun build with the local WebKit (same build directory and objects as the JSC build above) - After it compiles, run some code to make sure things work: `bun run build:local -p '42'` - Publish the new WebKit: diff --git a/scripts/webcore-copies.ts b/scripts/webcore-copies.ts new file mode 100644 index 000000000000..f5a9580c8cb5 --- /dev/null +++ b/scripts/webcore-copies.ts @@ -0,0 +1,332 @@ +#!/usr/bin/env bun +/** + * Lists the upstream WebKit commits that touch a file Bun keeps its own copy + * of. + * + * bun scripts/webcore-copies.ts [] + * + * Both are revisions in the WebKit clone (vendor/WebKit, or $BUN_WEBKIT_PATH): + * the upstream commit the fork was merged with before, and the one it is + * merged with now (`upstream/main` when left out). The clone needs the history + * between the two. One made with `--filter=blob:none` fetches each file that + * is compared. + * + * Bun does not build Source/WebCore. The files under src/jsc/bindings that + * come from it (most of them in webcore/ and webcrypto/) were copied by hand + * and edited to run outside a browser. Merging upstream into the fork does not + * change them, so an upstream fix reaches a copy only when someone ports it. + * .claude/commands/upgrade-webkit.md says what to do with the list. + * + * Changes to upstream's implementation of streams are not listed: + * src/jsc/bindings/webcore/streams is Bun's own. A changed .idl of a stream + * class is listed, like every .idl that Bun has a binding for. + */ + +import { existsSync, readFileSync } from "node:fs"; +import { basename, dirname, join } from "node:path"; + +const copiesRoot = "src/jsc/bindings"; +const upstreamRoot = "Source/WebCore"; + +/** + * The copies that a file name does not find: Bun's file, then every upstream + * file its code comes from. An entry replaces the match by name, so it also + * has the upstream file of the same name when there is one. + * + * Add an upstream file to an entry, beside the one the code came from, when + * upstream renames or splits a file that has a copy or moves code out of it. + * Add an entry when WebCore code goes into a Bun file of another name. The + * script fails when a file that is named here is not there, so that an entry + * cannot stop matching without notice. + */ +export const upstreamOf: Record = { + // Upstream moved much of structured clone into JavaScriptCore. Bun's copy still has it in one file. + "src/jsc/bindings/webcore/SerializedScriptValue.cpp": [ + "Source/WebCore/bindings/js/SerializedScriptValue.cpp", + "Source/JavaScriptCore/runtime/CloneBase.h", + "Source/JavaScriptCore/runtime/CloneSerializerBase.h", + "Source/JavaScriptCore/runtime/CloneDeserializerBase.h", + "Source/JavaScriptCore/runtime/StructuredCloneTags.h", + "Source/JavaScriptCore/runtime/StructuredCloneTags.cpp", + ], + // Upstream split the inline functions off. + "src/jsc/bindings/webcore/EventTarget.h": [ + "Source/WebCore/dom/EventTarget.h", + "Source/WebCore/dom/EventTargetInlines.h", + ], + "src/jsc/bindings/webcore/JSValueInWrappedObject.h": [ + "Source/WebCore/bindings/js/JSValueInWrappedObject.h", + "Source/WebCore/bindings/js/JSValueInWrappedObjectInlines.h", + ], + // The OpenSSL half is in a file of its own upstream. Bun wrote its half into the algorithm's file. + "src/jsc/bindings/webcrypto/CryptoAlgorithmEd25519.cpp": [ + "Source/WebCore/crypto/algorithms/CryptoAlgorithmEd25519.cpp", + "Source/WebCore/crypto/openssl/CryptoAlgorithmEd25519OpenSSL.cpp", + ], + "src/jsc/bindings/webcrypto/CryptoAlgorithmX25519.cpp": [ + "Source/WebCore/crypto/algorithms/CryptoAlgorithmX25519.cpp", + "Source/WebCore/crypto/openssl/CryptoAlgorithmX25519OpenSSL.cpp", + ], + // Made from the Cocoa file, before upstream had an OpenSSL one. + "src/jsc/bindings/webcrypto/CryptoKeyOKPOpenSSL.cpp": [ + "Source/WebCore/crypto/openssl/CryptoKeyOKPOpenSSL.cpp", + "Source/WebCore/crypto/cocoa/CryptoKeyOKPCocoa.cpp", + ], + // WebCore code in a file with a name of Bun's. + "src/jsc/bindings/webcrypto/CryptoDigest.cpp": ["Source/WebCore/PAL/pal/crypto/openssl/CryptoDigestOpenSSL.cpp"], + "src/jsc/bindings/BunClientData.h": ["Source/WebCore/bindings/js/WebCoreJSClientData.h"], + "src/jsc/bindings/BunClientData.cpp": ["Source/WebCore/bindings/js/WebCoreJSClientData.cpp"], + "src/jsc/bindings/BunGCOutputConstraint.h": ["Source/WebCore/bindings/js/DOMGCOutputConstraint.h"], + "src/jsc/bindings/BunGCOutputConstraint.cpp": ["Source/WebCore/bindings/js/DOMGCOutputConstraint.cpp"], + "src/jsc/bindings/BunInjectedScriptHost.h": ["Source/WebCore/inspector/WebInjectedScriptHost.h"], + "src/jsc/bindings/BunInjectedScriptHost.cpp": ["Source/WebCore/inspector/WebInjectedScriptHost.cpp"], + "src/jsc/bindings/DOMWrapperWorld-class.h": ["Source/WebCore/bindings/js/DOMWrapperWorld.h"], + "src/jsc/bindings/ZigGlobalObject.h": ["Source/WebCore/bindings/js/JSDOMGlobalObject.h"], + "src/jsc/bindings/ZigGlobalObject.cpp": ["Source/WebCore/bindings/js/JSDOMGlobalObject.cpp"], + // node:events on the listener code of EventTarget. + "src/jsc/bindings/webcore/EventEmitter.cpp": ["Source/WebCore/dom/EventTarget.cpp"], + "src/jsc/bindings/webcore/JSEventEmitterCustom.h": ["Source/WebCore/bindings/js/JSEventTargetCustom.h"], + "src/jsc/bindings/webcore/IdentifierEventListenerMap.cpp": ["Source/WebCore/dom/EventListenerMap.cpp"], + "src/jsc/bindings/webcore/IdentifierEventListenerMap.h": [ + "Source/WebCore/dom/EventListenerMap.h", + "Source/WebCore/dom/RegisteredEventListener.h", + ], +}; + +export interface ChangedFile { + /** Path in WebKit. */ + upstream: string; + /** The new base does not have it: the range deleted it, or moved it. */ + gone: boolean; + /** Bun's copies of it, with `compareToUpstream`'s share. */ + copies: { path: string; share: number | undefined }[]; + /** Bun's `JS.cpp`, when `upstream` is `.idl`. */ + binding: string | undefined; +} + +export interface UpstreamCommit { + sha: string; + date: string; + title: string; + files: ChangedFile[]; +} + +export interface CopyChanges { + /** Newest first. */ + commits: UpstreamCommit[]; + /** How many commits the range has, listed or not. */ + total: number; +} + +/** What the command prints without a stack trace. */ +class Failure extends Error {} + +function git(cwd: string, ...args: string[]): string { + const { exitCode, stdout, stderr } = Bun.spawnSync({ cmd: ["git", ...args], cwd, stdout: "pipe", stderr: "pipe" }); + if (exitCode !== 0) throw new Failure(`\`git ${args.join(" ")}\` failed in ${cwd}:\n${stderr}`); + return stdout.toString(); +} + +const trimmedLines = (text: string) => + text + .split("\n") + .map(line => line.trim()) + .filter(line => line !== ""); + +/** + * Whether `copy` still has text of `upstream`, and how much. + * + * Two files with the same name are not always related: Bun's Path.cpp is + * node:path, WebCore's draws curves. `isCopy` asks for one shared line of 40 + * characters or more. The license header of a real copy has several, and + * `#pragma once`, `namespace WebCore {` or `#include "config.h"` are shorter. + * + * `share` is the part of the copy's code that the upstream text has too, where + * code is every line of 16 characters or more outside a comment (so not a + * brace, `return true;` or the license). It says how well an upstream diff + * lines up with the copy. It falls when Bun edits the copy and also when + * upstream edits, splits or moves its file, so a low share does not mean that + * a fix has nothing to apply to. It is undefined for a copy with fewer than 8 + * lines of code, where one edited line decides it. + */ +export function compareToUpstream(copy: string, upstream: string): { isCopy: boolean; share: number | undefined } { + const theirs = new Set(trimmedLines(upstream)); + const code = trimmedLines(copy.replace(/\/\*[\s\S]*?\*\//g, "")).filter( + line => line.length >= 16 && !line.startsWith("//"), + ); + return { + isCopy: trimmedLines(copy).some(line => line.length >= 40 && theirs.has(line)), + share: code.length >= 8 ? code.filter(line => theirs.has(line)).length / code.length : undefined, + }; +} + +/** + * A copy can have a name that differs a little from upstream's: + * CryptoAlgorithmAES_KW.cpp is CryptoAlgorithmAESKW.cpp there, and + * CustomEventCustom.cpp is JSCustomEventCustom.cpp. A file whose name differs + * in that way must share a quarter of its code to count. + */ +const withoutUnderscores = (path: string) => basename(path).replaceAll("_", ""); +const renamedCopyShare = 0.25; + +/** + * The commits in `oldBase..newBase` of the WebKit clone at `webkitRoot` that + * touch an upstream file with a counterpart among the tracked files under + * src/jsc/bindings of the bun checkout at `bunRoot`. A counterpart is: + * - a copy that `table` names for the file, + * - a copy found by name: a file of the same name, or of nearly the same name, + * that passes `compareToUpstream`, + * - for `.idl`, the binding `JS.cpp`. WebCore generates that file + * from the IDL, and Bun's is kept by hand. + */ +export function listCopyChanges( + bunRoot: string, + webkitRoot: string, + oldBase: string, + newBase: string, + table = upstreamOf, +): CopyChanges { + const range = `${oldBase}..${newBase}`; + const total = Number(git(webkitRoot, "rev-list", "--count", range)); + if (total === 0) throw new Failure(`No commit is in ${range}. Is the old base set, and is it the older of the two?`); + + const tracked = new Set( + trimmedLines(git(bunRoot, "ls-files", "--", copiesRoot)).filter(path => existsSync(join(bunRoot, path))), + ); + const ours = (path: string) => readFileSync(join(bunRoot, path), "utf8"); + const byName = new Map([...tracked].map(path => [basename(path), path])); + const byLooseName = Map.groupBy(tracked, withoutUnderscores); + + // `table` the other way round. + const namedCopies = new Map(); + for (const [copy, upstream] of Object.entries(table)) { + for (const path of upstream) namedCopies.set(path, [...(namedCopies.get(path) ?? []), copy]); + } + const outsideRoot = [...namedCopies.keys()].filter(path => !path.startsWith(`${upstreamRoot}/`)); + + // Which upstream files each end has. A failed read of one of them is then an error, not a file that is not there. + const filesAt = (rev: string) => + new Set(trimmedLines(git(webkitRoot, "ls-tree", "-r", "--name-only", rev, "--", upstreamRoot, ...outsideRoot))); + const atNew = filesAt(newBase); + const atOld = filesAt(oldBase); + const exists = (path: string) => atNew.has(path) || atOld.has(path); + const texts = new Map(); + const theirs = (path: string) => { + // A file that a commit in the range deleted or moved away is read as the old base has it. + if (!texts.has(path)) texts.set(path, git(webkitRoot, "show", `${atNew.has(path) ? newBase : oldBase}:${path}`)); + return texts.get(path)!; + }; + + // An entry that matches nothing must not go unseen. A file that neither end has is fine only when a commit outside + // the history of the old base has it: upstream adds it later, or adds and removes it inside the range. + const wrong = Object.keys(table) + .filter(copy => !tracked.has(copy)) + .map(copy => `${copy} is not a tracked file.`); + for (const path of namedCopies.keys()) { + if (exists(path) || git(webkitRoot, "rev-list", "-1", "--all", `^${oldBase}`, "--", path) !== "") continue; + const removal = git(webkitRoot, "rev-list", "-1", oldBase, "--", path).trim(); + wrong.push( + removal === "" + ? `${path} is in no commit of the clone.` + : `${path} is gone since ${removal.slice(0, 12)}. Name the file that has its code now, or take it out.`, + ); + } + if (wrong.length > 0) throw new Failure(`\`upstreamOf\` in scripts/webcore-copies.ts:\n ${wrong.join("\n ")}`); + + const shares = new Map(); + const namedShare = (copy: string) => { + if (!shares.has(copy)) { + const upstream = table[copy].filter(exists).map(theirs); + shares.set(copy, upstream.length > 0 ? compareToUpstream(ours(copy), upstream.join("\n")).share : undefined); + } + return shares.get(copy); + }; + + const changedFiles = new Map(); + const changedFile = (upstream: string) => { + if (changedFiles.has(upstream)) return changedFiles.get(upstream); + const copies = (namedCopies.get(upstream) ?? []).map(path => ({ path, share: namedShare(path) })); + const name = withoutUnderscores(upstream); + const found = [ + ...(byLooseName.get(name) ?? []), + ...((name.startsWith("JS") && byLooseName.get(name.slice(2))) || []), + ]; + for (const path of found) { + if (Object.hasOwn(table, path)) continue; + if (!exists(upstream)) { + // Added and removed inside the range: there is no text to hold the file against, so it stays listed. + copies.push({ path, share: undefined }); + continue; + } + const { isCopy, share } = compareToUpstream(ours(path), theirs(upstream)); + const sameName = basename(path) === basename(upstream); + if (isCopy && (sameName || (share ?? 0) >= renamedCopyShare)) copies.push({ path, share }); + } + const idl = basename(upstream).endsWith(".idl") ? basename(upstream).slice(0, -".idl".length) : undefined; + const binding = idl === undefined ? undefined : byName.get(`JS${idl}.cpp`); + const file = + copies.length > 0 || binding !== undefined + ? { upstream, gone: !atNew.has(upstream), copies, binding } + : undefined; + changedFiles.set(upstream, file); + return file; + }; + + // Per commit: \0 \0 \0 \0, then one path per line. + // --no-renames lists a moved file under both of its paths. + const format = "--format=%x00%h%x00%cs%x00%B%x00"; + const log = git(webkitRoot, "log", "--no-renames", "--name-only", format, range, "--", upstreamRoot, ...outsideRoot); + const fields = log.split("\0"); + const commits: UpstreamCommit[] = []; + for (let i = 1; i + 3 < fields.length; i += 4) { + const [sha, date, message, paths] = fields.slice(i, i + 4); + const files = trimmedLines(paths) + .map(changedFile) + .filter(file => file !== undefined); + if (files.length > 0) commits.push({ sha, date, title: message.split("\n")[0], files }); + } + return { commits, total }; +} + +export function formatCopyChanges({ commits, total }: CopyChanges, range: string): string { + const lines: string[] = []; + for (const { sha, date, title, files } of commits) { + lines.push(`${sha} ${date} ${title}`); + for (const { upstream, gone, copies, binding } of files) { + const counterparts = copies.map(({ path, share }) => + share === undefined ? path : `${path} (${Math.round(share * 100)}% same)`, + ); + if (binding !== undefined) counterparts.push(`${binding} (binding)`); + lines.push(` ${upstream}${gone ? " (gone)" : ""} -> ${counterparts.join(", ")}`); + } + } + if (commits.length > 0) lines.push(""); + lines.push(`Commits in ${range} that touch the upstream file of a Bun copy: ${commits.length} of ${total}`); + return lines.join("\n"); +} + +/** The command: what it prints for `args`, and its exit code. */ +export function main(args: string[], bunRoot: string, webkitRoot: string) { + const [oldBase, newBase = "upstream/main"] = args; + if (!oldBase || !newBase || !existsSync(webkitRoot)) { + const usage = "usage: bun scripts/webcore-copies.ts []"; + const clone = `It reads the WebKit clone at ${webkitRoot}. Set BUN_WEBKIT_PATH to use another one.`; + return { exitCode: 1, stdout: "", stderr: `${usage}\n${clone}` }; + } + try { + const changes = listCopyChanges(bunRoot, webkitRoot, oldBase, newBase); + return { exitCode: 0, stdout: formatCopyChanges(changes, `${oldBase}..${newBase}`), stderr: "" }; + } catch (error) { + if (!(error instanceof Failure)) throw error; + return { exitCode: 1, stdout: "", stderr: error.message }; + } +} + +if (import.meta.main) { + const bunRoot = dirname(import.meta.dir); + const webkitRoot = process.env.BUN_WEBKIT_PATH || join(bunRoot, "vendor", "WebKit"); + const { exitCode, stdout, stderr } = main(process.argv.slice(2), bunRoot, webkitRoot); + if (stdout !== "") console.log(stdout); + if (stderr !== "") console.error(stderr); + process.exitCode = exitCode; +} diff --git a/test/internal/webcore-copies.test.ts b/test/internal/webcore-copies.test.ts new file mode 100644 index 000000000000..711a4c02411b --- /dev/null +++ b/test/internal/webcore-copies.test.ts @@ -0,0 +1,591 @@ +/** + * scripts/webcore-copies.ts lists the upstream WebKit commits that touch a file Bun has its own copy of. The WebKit + * upgrade routine (.claude/commands/upgrade-webkit.md) runs it, so that an upstream fix to such a file is seen. It + * reads two git repositories: a bun checkout and a WebKit clone. Here both are a few files with the shape of the real + * ones. + */ +import { afterAll, expect, test } from "bun:test"; +import { bunEnv, bunExe, tempDir } from "harness"; +import { chmodSync, readFileSync, rmSync } from "node:fs"; +import { dirname, join } from "node:path"; + +import { + compareToUpstream, + formatCopyChanges, + listCopyChanges, + main, + upstreamOf, +} from "../../scripts/webcore-copies.ts"; + +const root = join(import.meta.dir, "..", ".."); + +function git(cwd: string, args: string[], stdin = "") { + const { exitCode, stdout, stderr } = Bun.spawnSync({ + cmd: ["git", ...args], + cwd, + env: bunEnv, + stdin: Buffer.from(stdin), + stdout: "pipe", + stderr: "pipe", + }); + if (exitCode !== 0) throw new Error(`git ${args.join(" ")} exited with ${exitCode}: ${stderr}`); + return stdout.toString().trim(); +} + +interface Commit { + message: string; + /** The files that the commit writes. `null` removes one. */ + files: Record; + /** A ref that stays at this commit. */ + ref?: string; +} + +/** + * Adds `commits` to `main` with one git process, which makes no use of the machine's git configuration (identity, + * signing, hooks). The author's date is not the committer's, and the objects stay loose files. + */ +function addCommits(dir: string, commits: Commit[], parent = "") { + const data = (text: string) => `data ${Buffer.byteLength(text)}\n${text}\n`; + const stream = commits.map(({ message, files, ref }, index) => + [ + `commit refs/heads/main\nmark :${index + 1}\n`, + "author Test 1756728000 +0000\ncommitter Test 1756900800 +0000\n", + data(message), + index === 0 ? parent : "", + ...Object.entries(files).map(([path, text]) => + text === null ? `D ${path}\n` : `M 100644 inline ${path}\n${data(text)}`, + ), + ref === undefined ? "" : `reset ${ref}\nfrom :${index + 1}\n`, + ].join(""), + ); + git(dir, ["-c", "fastimport.unpackLimit=100000", "fast-import", "--quiet"], stream.join("\n")); +} + +/** A WebKit clone with `commits` on `main`. The branch `base` is the first of them. */ +function clone([base, ...commits]: Commit[]) { + const dir = tempDir("webcore-copies-webkit", {}); + git(String(dir), ["init", "--quiet", "--initial-branch=main"]); + addCommits(String(dir), [{ ...base, ref: "refs/heads/base" }, ...commits]); + return dir; +} + +/** A bun checkout where `files` are tracked. */ +function checkout(files: Record) { + const dir = tempDir("webcore-copies-bun", files); + git(String(dir), ["init", "--quiet", "--initial-branch=main"]); + git(String(dir), ["add", "--all"]); + return dir; +} + +const license = `/* + * Copyright (C) 2016 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + */`; + +// A copy with the name of its upstream file. +const performanceObserver = (register: string, deliver: string) => `${license} +#include "config.h" +#include "PerformanceObserver.h" + +namespace WebCore { + +ExceptionOr PerformanceObserver::observe(Init&& init) +{ + if (!m_registered) { + ${register} + m_registered = true; + } + ${deliver} + return { }; +} + +} // namespace WebCore +`; +const upstreamBefore = performanceObserver( + "protectedPerformance()->registerPerformanceObserver(*this);", + "if (isBuffered)\n deliver();", +); +const upstreamAfter = performanceObserver( + "protectedPerformance()->registerPerformanceObserver(*this);", + "if (isBuffered && m_entriesToDeliver.size())\n protectedPerformance()->scheduleTaskIfNeeded();", +); +// Edited for Bun, and without the upstream fix. +const bunCopy = performanceObserver( + "m_performance->registerPerformanceObserver(*this);", + "if (isBuffered)\n deliver();", +); + +// Bun's src/jsc/bindings/Path.cpp is node:path. WebCore's platform/graphics/Path.cpp draws curves. +const bunPath = `#include "Path.h" + +namespace Bun { + +JSC::JSValue createNodePathBinding(Zig::GlobalObject* globalObject) +{ + return JSC::constructEmptyObject(globalObject); +} + +} // namespace Bun +`; +const webcorePath = (rule: string) => `${license} +#include "config.h" +#include "Path.h" + +namespace WebCore { + +bool Path::contains(const FloatPoint& point, WindRule rule) const +{ + return ${rule}; +} + +} // namespace WebCore +`; + +// A copy with an underscore that upstream's name does not have. +const aesKw = (header: string, failed: string) => `${license} +#include "${header}" + +namespace WebCore { + +static std::optional> cryptUnwrapKey(const Vector& key, const Vector& data) +{ + Vector plainText(data.size() - 8); + if (AES_unwrap_key(aesKey.key(), nullptr, plainText.begin(), data.begin(), data.size()) ${failed}) + return std::nullopt; + return plainText; +} + +} // namespace WebCore +`; + +// A copy without the JS that upstream's name starts with. +const customEvent = (detail: string) => `${license} +#include "config.h" +#include "JSCustomEvent.h" + +namespace WebCore { + +JSC::JSValue JSCustomEvent::detail(JSC::JSGlobalObject& lexicalGlobalObject) const +{ + return cachedPropertyValue(lexicalGlobalObject, *this, wrapped().cachedDetail(), [this] { + return ${detail}; + }); +} + +template +void JSCustomEvent::visitAdditionalChildren(Visitor& visitor) +{ + wrapped().detail().visit(visitor); + wrapped().cachedDetail().visit(visitor); +} + +} // namespace WebCore +`; + +// Two files with nearly the same name, and the same license, that are not related: upstream's has a JS in front. +const upstreamCustom = (world: string) => `${license} +#include "config.h" +#include "JSPerformanceObserver.h" + +bool JSPerformanceObserverOwner::isReachableFromOpaqueRoots(JSC::Handle handle, void*, JSC::AbstractSlotVisitor& visitor, ASCIILiteral* reason) +{ + return JSC::jsCast(handle.slot()->asCell())->wrapped().isRegistered(${world}); +} +`; +const bunCustom = `${license} +#include "PerformanceObserverCustom.h" + +JSC_DEFINE_HOST_FUNCTION(jsPerformanceObserverPrototypeFunction_observe, (JSGlobalObject* globalObject, CallFrame* callFrame)) +{ + auto* thisObject = jsDynamicCast(callFrame->thisValue()); + auto options = convert>(*globalObject, callFrame->argument(0)); + return JSValue::encode(toJS(*globalObject, thisObject->wrapped().observe(WTF::move(options)))); +} + +JSC_DEFINE_HOST_FUNCTION(jsPerformanceObserverPrototypeFunction_disconnect, (JSGlobalObject* globalObject, CallFrame* callFrame)) +{ + auto* thisObject = jsDynamicCast(callFrame->thisValue()); + thisObject->wrapped().disconnect(); + return JSValue::encode(jsUndefined()); +} +`; + +// A copy whose code upstream has in two files. The table names both: no file name leads to the second one. +const cloneSerializer = ` void dumpArrayBufferView(JSObject* object, SerializationReturnCode& code); + bool dumpIfTerminal(JSValue value, SerializationReturnCode& code); + void write(SerializationTag tag) { writeLittleEndian(m_buffer, static_cast(tag)); } + void write(const String& string) { write(string.impl()); } +`; +const serializedScriptValue = `RefPtr SerializedScriptValue::create(JSGlobalObject& globalObject, JSValue value) + auto serializer = CloneSerializer(globalObject, messagePorts, arrayBuffers); + Vector buffer = serializer.takeBuffer(); + return adoptRef(*new SerializedScriptValue(WTF::move(buffer))); +`; +const bunSerializer = "src/jsc/bindings/webcore/SerializedScriptValue.cpp"; +const upstreamSerializer = "Source/JavaScriptCore/runtime/CloneSerializerBase.h"; +const table = { [bunSerializer]: ["Source/WebCore/bindings/js/SerializedScriptValue.cpp", upstreamSerializer] }; + +const renderBox = `${license}\n// Bun has this file too, but not where the copies are.\n`; + +const shas = (text: string) => text.replace(/^[0-9a-f]{4,64} /gm, " "); +const list = (bun: string, webkit: string, oldBase: string, newBase: string, names: Record = {}) => + shas(formatCopyChanges(listCopyChanges(bun, webkit, oldBase, newBase, names), `${oldBase}..${newBase}`)).split("\n"); + +// Ten upstream commits and the Bun files that go with them. Nothing changes these two repositories after this. +let repositories: { bun: string; webkit: string; [Symbol.dispose](): void } | undefined; +afterAll(() => repositories?.[Symbol.dispose]()); +function tenCommits() { + if (repositories !== undefined) return repositories; + const bun = checkout({ + "src/jsc/bindings/webcore/PerformanceObserver.cpp": bunCopy, + "src/jsc/bindings/webcore/JSPerformanceObserver.cpp": `#include "JSPerformanceObserver.h"\n`, + "src/jsc/bindings/webcore/PerformanceObserverCustom.cpp": bunCustom, + "src/jsc/bindings/webcore/CustomEventCustom.cpp": customEvent("wrapped().detail().getValue()"), + "src/jsc/bindings/webcrypto/CryptoAlgorithmAES_KW.cpp": aesKw("CryptoAlgorithmAES_KW.h", "< 0"), + [bunSerializer]: `${license}\n${cloneSerializer}${serializedScriptValue}`, + "src/jsc/bindings/Path.cpp": bunPath, + "src/jsc/bindings/RenderBox.cpp": renderBox, + "src/runtime/RenderBox.cpp": renderBox, + }); + // Tracked, and not in the working tree. + rmSync(join(String(bun), "src/jsc/bindings/RenderBox.cpp")); + + const webkit = clone([ + { + message: "base", + files: { + "Source/WebCore/page/PerformanceObserver.cpp": upstreamBefore, + "Source/WebCore/page/PerformanceObserver.idl": "interface PerformanceObserver {};\n", + "Source/WebCore/bindings/js/JSPerformanceObserverCustom.cpp": upstreamCustom("world"), + "Source/WebCore/bindings/js/JSCustomEventCustom.cpp": customEvent("wrapped().detail().getValue()"), + "Source/WebCore/crypto/algorithms/CryptoAlgorithmAESKW.cpp": aesKw("CryptoAlgorithmAESKW.h", "< 0"), + "Source/WebCore/bindings/js/SerializedScriptValue.cpp": `${license}\n${serializedScriptValue}`, + [upstreamSerializer]: `${license}\n${cloneSerializer}`, + "Source/WebCore/platform/graphics/Path.cpp": webcorePath("false"), + "Source/WebCore/rendering/RenderBox.cpp": renderBox, + "Tools/TestWebKitAPI/PerformanceObserver.cpp": upstreamBefore, + }, + }, + { + message: "Schedule task on buffered PerformanceObserver\nhttps://bugs.webkit.org/show_bug.cgi?id=297765", + files: { + "Source/WebCore/page/PerformanceObserver.cpp": upstreamAfter, + "LayoutTests/performance-api/buffered-does-not-sync-invoke.html": "\n", + }, + }, + { + message: "Not listed: Bun's files of this name are elsewhere, or deleted", + files: { "Source/WebCore/rendering/RenderBox.cpp": `${renderBox}// Changed.\n` }, + }, + { + message: "Not listed: Bun's file of this name is not a copy", + files: { "Source/WebCore/platform/graphics/Path.cpp": webcorePath("rule == WindRule::NonZero") }, + }, + { + message: "Not listed: Bun's file of nearly this name has none of its code", + files: { "Source/WebCore/bindings/js/JSPerformanceObserverCustom.cpp": upstreamCustom("visitor.world()") }, + }, + { + message: "Not listed: the file is not under Source/WebCore, and the table does not name it", + files: { "Tools/TestWebKitAPI/PerformanceObserver.cpp": upstreamAfter }, + }, + { + message: "Expose PerformanceObserver.supportedEntryTypes", + files: { + "Source/WebCore/page/PerformanceObserver.idl": + "interface PerformanceObserver { attribute any supportedEntryTypes; };\n", + }, + }, + { + message: "AES-KW unwrapKey accepts tampered wrapped keys", + files: { "Source/WebCore/crypto/algorithms/CryptoAlgorithmAESKW.cpp": aesKw("CryptoAlgorithmAESKW.h", "<= 0") }, + }, + { + message: "CustomEvent.detail is null before initCustomEvent()", + files: { + "Source/WebCore/bindings/js/JSCustomEventCustom.cpp": customEvent("wrapped().detail().getValue(JSC::jsNull())"), + }, + }, + { + message: "Serialize a resizable ArrayBuffer with its maximum length", + files: { + [upstreamSerializer]: `${license}\n${cloneSerializer} void write(uint64_t);\n`, + "Source/WebCore/bindings/js/SerializedScriptValue.cpp": `${license}\n${serializedScriptValue}// Changed.\n`, + }, + }, + { + message: "Move PerformanceObserver to Modules/performance", + files: { + "Source/WebCore/page/PerformanceObserver.cpp": null, + "Source/WebCore/Modules/performance/PerformanceObserver.cpp": upstreamAfter, + }, + }, + ]); + return (repositories = { + bun: String(bun), + webkit: String(webkit), + [Symbol.dispose]() { + bun[Symbol.dispose](); + webkit[Symbol.dispose](); + }, + }); +} + +test("lists the commits that touch the upstream file of a copy or the IDL of a binding, and no others", () => { + const { bun, webkit } = tenCommits(); + expect(list(bun, webkit, "base", "main", table)).toEqual([ + // A moved file is listed under both paths. `main` does not have the old one, which is compared as `base` has it. + " 2025-09-03 Move PerformanceObserver to Modules/performance", + " Source/WebCore/Modules/performance/PerformanceObserver.cpp -> src/jsc/bindings/webcore/PerformanceObserver.cpp (88% same)", + " Source/WebCore/page/PerformanceObserver.cpp (gone) -> src/jsc/bindings/webcore/PerformanceObserver.cpp (88% same)", + // The two upstream files that the table names for this copy, one of them outside Source/WebCore. Each has half of + // the copy's code, and the share is of both together. The copy is listed once, not again for its name. + " 2025-09-03 Serialize a resizable ArrayBuffer with its maximum length", + " Source/JavaScriptCore/runtime/CloneSerializerBase.h -> src/jsc/bindings/webcore/SerializedScriptValue.cpp (100% same)", + " Source/WebCore/bindings/js/SerializedScriptValue.cpp -> src/jsc/bindings/webcore/SerializedScriptValue.cpp (100% same)", + // Two copies under nearly the name of the upstream file. Neither has the line that the commit changes. + " 2025-09-03 CustomEvent.detail is null before initCustomEvent()", + " Source/WebCore/bindings/js/JSCustomEventCustom.cpp -> src/jsc/bindings/webcore/CustomEventCustom.cpp (91% same)", + " 2025-09-03 AES-KW unwrapKey accepts tampered wrapped keys", + " Source/WebCore/crypto/algorithms/CryptoAlgorithmAESKW.cpp -> src/jsc/bindings/webcrypto/CryptoAlgorithmAES_KW.cpp (75% same)", + " 2025-09-03 Expose PerformanceObserver.supportedEntryTypes", + " Source/WebCore/page/PerformanceObserver.idl -> src/jsc/bindings/webcore/JSPerformanceObserver.cpp (binding)", + // The first line of the message, the committer's date, and only the file that Bun has a copy of. + " 2025-09-03 Schedule task on buffered PerformanceObserver", + " Source/WebCore/page/PerformanceObserver.cpp (gone) -> src/jsc/bindings/webcore/PerformanceObserver.cpp (88% same)", + "", + "Commits in base..main that touch the upstream file of a Bun copy: 6 of 10", + ]); +}); + +test("a range in which no commit touches a copy prints the count alone", () => { + const { bun, webkit } = tenCommits(); + // The four commits in the middle of the ten. + expect(list(bun, webkit, "main~9", "main~5", table)).toEqual([ + "Commits in main~9..main~5 that touch the upstream file of a Bun copy: 0 of 4", + ]); +}); + +test("a range with no commits is an error", () => { + const { bun, webkit } = tenCommits(); + // The arguments the wrong way round, and an old base that was never set. + expect(() => listCopyChanges(bun, webkit, "main", "base", table)).toThrow("No commit is in main..base."); + expect(() => listCopyChanges(bun, webkit, "main", "main", table)).toThrow("No commit is in main..main."); +}); + +test("an upstream file that git cannot read is an error, and not a file without a copy", () => { + const path = "Source/WebCore/page/PerformanceObserver.cpp"; + using bun = checkout({ "src/jsc/bindings/webcore/PerformanceObserver.cpp": bunCopy }); + using webkit = clone([ + { message: "base", files: { [path]: upstreamBefore } }, + { message: "Schedule task on buffered PerformanceObserver", files: { [path]: upstreamAfter } }, + ]); + // What a clone without blobs has when it cannot fetch one: the tree names the file, and its content is not there. + const blob = git(String(webkit), ["rev-parse", `main:${path}`]); + const object = join(String(webkit), ".git", "objects", blob.slice(0, 2), blob.slice(2)); + chmodSync(object, 0o666); + rmSync(object); + expect(() => listCopyChanges(String(bun), String(webkit), "base", "main", {})).toThrow( + `\`git show main:${path}\` failed`, + ); +}); + +test("a table that names a file which is not there is an error", () => { + const webcoreFile = "Source/WebCore/bindings/js/SerializedScriptValue.cpp"; + using bun = checkout({ [bunSerializer]: `${license}\n${cloneSerializer}${serializedScriptValue}` }); + using webkit = clone([ + { + message: "base", + files: { + [upstreamSerializer]: `${license}\n${cloneSerializer}`, + [webcoreFile]: `${license}\n${serializedScriptValue}`, + }, + }, + { + message: "Move the serializer to CloneSerializer.h", + files: { [upstreamSerializer]: null, "Source/JavaScriptCore/runtime/CloneSerializer.h": cloneSerializer }, + ref: "refs/heads/moved", + }, + { message: "Another change", files: { [webcoreFile]: `${license}\n` }, ref: "refs/heads/later" }, + ]); + const [bunRoot, dir] = [String(bun), String(webkit)]; + const removal = git(dir, ["rev-parse", "moved"]).slice(0, 12); + + // The range that removes the file lists it. After that range the entry would match nothing, and nobody would know. + expect(list(bunRoot, dir, "base", "moved", table)).toEqual([ + " 2025-09-03 Move the serializer to CloneSerializer.h", + ` ${upstreamSerializer} (gone) -> ${bunSerializer} (100% same)`, + "", + "Commits in base..moved that touch the upstream file of a Bun copy: 1 of 1", + ]); + expect(() => listCopyChanges(bunRoot, dir, "moved", "later", table)).toThrow( + `\`upstreamOf\` in scripts/webcore-copies.ts:\n ${upstreamSerializer} is gone since ${removal}.`, + ); + + // A name that upstream never had, and a copy that Bun does not have. + const typo = { [bunSerializer]: [...table[bunSerializer], "Source/JavaScriptCore/runtime/CloneSerialiserBase.h"] }; + expect(() => listCopyChanges(bunRoot, dir, "base", "later", typo)).toThrow( + "Source/JavaScriptCore/runtime/CloneSerialiserBase.h is in no commit of the clone.", + ); + expect(() => listCopyChanges(bunRoot, dir, "base", "later", { "src/jsc/bindings/Gone.cpp": [] })).toThrow( + "src/jsc/bindings/Gone.cpp is not a tracked file.", + ); + + // Upstream brings the file back. In the range before that, neither end has it and the table is right. + const reland = { + message: "Reland the serializer", + files: { [upstreamSerializer]: `${license}\n${cloneSerializer}` }, + }; + addCommits(dir, [reland], "from refs/heads/main^0\n"); + expect(list(bunRoot, dir, "moved", "later", table)).toEqual([ + " 2025-09-03 Another change", + ` ${webcoreFile} -> ${bunSerializer} (0% same)`, + "", + "Commits in moved..later that touch the upstream file of a Bun copy: 1 of 1", + ]); +}); + +test("a file that the range adds and removes again is listed without a share", () => { + const path = "Source/WebCore/page/PerformanceTiming.cpp"; + using bun = checkout({ "src/jsc/bindings/webcore/PerformanceTiming.cpp": bunCopy }); + using webkit = clone([ + { message: "base", files: { "Source/WebCore/page/Performance.cpp": upstreamBefore } }, + { message: "Add PerformanceTiming", files: { [path]: upstreamBefore } }, + { message: "Revert: Add PerformanceTiming", files: { [path]: null } }, + ]); + + // Neither end has the file, so there is nothing to hold Bun's file against: not when its name leads to the upstream + // file, and not when the table does. + const listed = [ + " 2025-09-03 Revert: Add PerformanceTiming", + ` ${path} (gone) -> src/jsc/bindings/webcore/PerformanceTiming.cpp`, + " 2025-09-03 Add PerformanceTiming", + ` ${path} (gone) -> src/jsc/bindings/webcore/PerformanceTiming.cpp`, + "", + "Commits in base..main that touch the upstream file of a Bun copy: 2 of 2", + ]; + expect(list(String(bun), String(webkit), "base", "main")).toEqual(listed); + const named = { "src/jsc/bindings/webcore/PerformanceTiming.cpp": [path] }; + expect(list(String(bun), String(webkit), "base", "main", named)).toEqual(listed); +}); + +test("every copy that upstreamOf names is a tracked file", () => { + const copies = Object.keys(upstreamOf); + const { exitCode, stdout, stderr } = Bun.spawnSync({ cmd: ["git", "ls-files", "--", ...copies], cwd: root }); + expect({ exitCode, stderr: stderr.toString(), tracked: stdout.toString().trim().split("\n").sort() }).toEqual({ + exitCode: 0, + stderr: "", + tracked: copies.toSorted(), + }); +}); + +test("a file of the same name is a copy only while it shares a long line with the upstream file", () => { + // Seven of the copy's eight lines of code are in the upstream file. The short lines (`if (isBuffered)`, `deliver();`, + // braces) and the license are not counted. + expect(compareToUpstream(bunCopy, upstreamAfter)).toEqual({ isCopy: true, share: 7 / 8 }); + expect(compareToUpstream(upstreamAfter, upstreamAfter)).toEqual({ isCopy: true, share: 1 }); + // `#include "Path.h"` is in both files, and is too short to make one a copy of the other. + expect(compareToUpstream(bunPath, webcorePath("false"))).toEqual({ isCopy: false, share: undefined }); + // The license alone makes it a copy. Fewer than eight lines of code have no share: the copy, less one line. + expect(compareToUpstream(`${license}\n#pragma once\n`, upstreamAfter)).toEqual({ isCopy: true, share: undefined }); + expect(compareToUpstream(bunCopy.replace(" m_registered = true;\n", ""), upstreamAfter)).toEqual({ + isCopy: true, + share: undefined, + }); + // So a file of nearly the same name needs more than the license: a quarter of its code. + expect(compareToUpstream(bunCustom, upstreamCustom("world"))).toEqual({ isCopy: true, share: 0 }); + // A comment is not code: not one that Bun added, and not a license without an asterisk in front of its lines. + const commented = bunCopy.replace("namespace WebCore {", "// Bun has no document here.\nnamespace WebCore {"); + expect(compareToUpstream(commented, upstreamAfter)).toEqual({ isCopy: true, share: 7 / 8 }); + const plainLicense = license.replaceAll("\n * ", "\n "); + expect( + compareToUpstream(bunCopy.replace(license, plainLicense), upstreamAfter.replace(license, plainLicense)), + ).toEqual({ isCopy: true, share: 7 / 8 }); +}); + +// A clone for this checkout. It has every upstream file that `upstreamOf` names, and the upstream file of one more +// copy, with the copy's own text: so the two are related whatever the copy says. +const realCopy = "src/jsc/bindings/webcore/PerformanceObserver.cpp"; +function cloneForThisCheckout() { + const text = readFileSync(join(root, realCopy), "utf8"); + const named = Object.values(upstreamOf).flat(); + return clone([ + { + message: "base", + files: { + ...Object.fromEntries(named.map(path => [path, "// Not changed.\n"])), + "Source/WebCore/page/PerformanceObserver.cpp": text, + }, + }, + { + message: "Schedule task on buffered PerformanceObserver", + files: { "Source/WebCore/page/PerformanceObserver.cpp": `${text}\n// The fix.\n` }, + ref: "refs/remotes/upstream/main", + }, + ]); +} +const listedForThisCheckout = [ + " 2025-09-03 Schedule task on buffered PerformanceObserver", + ` Source/WebCore/page/PerformanceObserver.cpp -> ${realCopy} (100% same)`, + "", + "Commits in base..upstream/main that touch the upstream file of a Bun copy: 1 of 1", +].join("\n"); + +test("the command prints the list, or says what is wrong", () => { + using webkit = cloneForThisCheckout(); + const command = (args: string[], clone = String(webkit)) => { + const { exitCode, stdout, stderr } = main(args, root, clone); + return { exitCode, stdout: shas(stdout), stderr }; + }; + expect(command(["base"])).toEqual({ exitCode: 0, stdout: listedForThisCheckout, stderr: "" }); + + // `bun scripts/webcore-copies.ts $OLD_BASE upstream/main` with the variable unset. + expect(command(["upstream/main"])).toEqual({ + exitCode: 1, + stdout: "", + stderr: "No commit is in upstream/main..upstream/main. Is the old base set, and is it the older of the two?", + }); + // An argument that is missing or empty, and a clone that is not there. + const usage = (clone: string) => ({ + exitCode: 1, + stdout: "", + stderr: + "usage: bun scripts/webcore-copies.ts []\n" + + `It reads the WebKit clone at ${clone}. Set BUN_WEBKIT_PATH to use another one.`, + }); + const nowhere = join(String(webkit), "nowhere"); + expect([command([]), command(["", "upstream/main"]), command(["base", ""]), command(["base"], nowhere)]).toEqual([ + usage(String(webkit)), + usage(String(webkit)), + usage(String(webkit)), + usage(nowhere), + ]); +}); + +// Each run starts a Bun, which takes seconds for a debug build on a busy machine. +test("the script finds this checkout from where it is, and the clone from $BUN_WEBKIT_PATH", async () => { + using webkit = cloneForThisCheckout(); + const script = async (...args: string[]) => { + await using proc = Bun.spawn({ + cmd: [bunExe(), join(root, "scripts", "webcore-copies.ts"), ...args], + env: { ...bunEnv, BUN_WEBKIT_PATH: String(webkit) }, + cwd: dirname(String(webkit)), + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + return { stdout: shas(stdout), stderr, exitCode }; + }; + expect(await Promise.all([script("base"), script()])).toEqual([ + { stdout: `${listedForThisCheckout}\n`, stderr: "", exitCode: 0 }, + { + stdout: "", + stderr: + "usage: bun scripts/webcore-copies.ts []\n" + + `It reads the WebKit clone at ${webkit}. Set BUN_WEBKIT_PATH to use another one.\n`, + exitCode: 1, + }, + ]); +}, 30_000); From 2ae02870ceeb2023f4ce06650aac290312378840 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 29 Oct 2026 08:26:04 +0000 Subject: [PATCH 2/3] webcore-copies: match by name or table, compare with the old base - A copy is compared with its upstream file as the old base has it, or as the first commit of the range that has it. - "% same" is the part of the upstream file's code that the copy has. - The match by a nearly equal file name is gone. The 16 copies that it found are entries of `upstreamOf`. An entry now adds to the match by name, so it names only the files of another name. - The table check counts only upstream commits after the old base. - A tracked file that is not in the working tree is an error. - Both revisions are required. The checklist runs the script on vendor/WebKit. - The test has no timeout of its own. --- .claude/commands/upgrade-webkit.md | 11 +- scripts/webcore-copies.ts | 212 +++++++------- test/internal/webcore-copies.test.ts | 395 +++++++++++++-------------- 3 files changed, 304 insertions(+), 314 deletions(-) diff --git a/.claude/commands/upgrade-webkit.md b/.claude/commands/upgrade-webkit.md index d1f755a20b1f..511c960c4f50 100644 --- a/.claude/commands/upgrade-webkit.md +++ b/.claude/commands/upgrade-webkit.md @@ -19,12 +19,13 @@ To do that: - Fix the merge conflicts (preserve the fork's Bun-specific changes) - bun run jsc:build:debug — from the bun repo root, builds just JSC (and its `jsc` shell), in bun's own build graph. File lists come from the checkout, but `scripts/build/deps/webkit.ts` restates WebKit's generator commands, compiler flags and option values by hand, and nothing checks them. Read `git diff $OLD_BASE upstream/main -- Source/cmake Source/JavaScriptCore/CMakeLists.txt Source/WTF/wtf/CMakeLists.txt Source/bmalloc/CMakeLists.txt` for changed `add_custom_command`s (→ `gen()`), compiler flags (→ `webkitFlags()`) and `WEBKIT_OPTION_*` / `SET_AND_EXPOSE_TO_BUILD` (→ `rows`), and carry them over in the bun PR - While it compiles, in another task review the JSC commits between $OLD_BASE and upstream/main (Source/JavaScriptCore, Source/WTF, Source/bmalloc). Write up a summary in a file called "webkit-changes.md" -- In the same task, run `bun scripts/webcore-copies.ts $OLD_BASE upstream/main` from the bun repo root. Pass the commit itself if the task's shell does not have `$OLD_BASE`. The script lists the upstream commits in that range that touch a file Bun has its own copy of. Bun does not build Source/WebCore: the files under src/jsc/bindings that come from it (most are in webcore/ and webcrypto/) were copied and edited by hand, and the merge does not change them. An upstream fix reaches a copy only when it is ported. The build is using the bun checkout: this task writes webkit-changes.md and, where a line below says so, `upstreamOf` in the script, and no other file +- In the same task, run `BUN_WEBKIT_PATH=vendor/WebKit bun scripts/webcore-copies.ts $OLD_BASE upstream/main` from the bun repo root. Put the commit id in place of `$OLD_BASE` if the task's shell does not have the variable. `BUN_WEBKIT_PATH` makes the script read the clone of this routine, also when the shell has another value for it. The script lists the upstream commits in that range that touch a file Bun has its own copy of. Bun does not build Source/WebCore. The files under src/jsc/bindings that come from it (most are in webcore/ and webcrypto/) were copied and edited by hand, and the merge does not change them. An upstream fix reaches a copy only when it is ported. The build is reading the bun checkout: in this task, change no file of the checkout but webkit-changes.md and, where a line below says so, `upstreamOf` in the script - For each commit, read `git -C vendor/WebKit show -- ` beside Bun's copy. A behavior change (a bug fix, a spec update) applies when the copy still has the code that it changes, even where the lines no longer match. If the title names a behavior, try it with the installed `bun`. A refactor that changes no behavior needs no port - - "% same" is how much of the copy's code its upstream file also has (all of its upstream files together for a copy in `upstreamOf`; a copy too short to measure has no number). It tells how well the diff lines up with the copy. It falls when either side changes, so a low value is no reason to pass over a commit - - "(binding)" marks Bun's `JS.cpp` for a changed `.idl`: compare the interface - - `upstreamOf` in the script names the upstream files of a copy that its file name does not lead to. When a commit moves code of a copy into another upstream file (it splits the file, or the list marks the file "(gone)" and does not list the copy under a new path), name both files for the copy there, the old one and the new one, and run the script again - - Not listed: changes to upstream's implementation of streams (src/jsc/bindings/webcore/streams is Bun's own; a changed stream `.idl` is listed as a binding), and WebCore code that Bun keeps in a file of another name unless `upstreamOf` names it + - "% same" is how much of the code of the upstream file the copy also has. The upstream file is taken as it was before these commits, or as the commit that adds it has it. A file with fewer than 8 lines of code has no number. The number tells how well a diff of the file lines up with the copy. It is low when Bun has few of the file's functions, and when either side has edited them. A low number is no reason to skip a commit + - "(binding)" marks Bun's `JS.cpp` for a changed `.idl`: compare the change with that binding + - "(gone)" marks an upstream file that the new base does not have, on each commit that touches it. `git -C vendor/WebKit show --stat ` of the newest of those commits shows where its code went + - `upstreamOf` in the script names the upstream files of a copy that have another file name than the copy. Without an entry, only a file of the same name leads to a copy. When a commit moves code of a copy into an upstream file of another name (it renames the file or splits it), name that file for the copy in `upstreamOf` and run the script again. When the script says that a file in `upstreamOf` "is gone since" a commit, name the file that has its code now, or take the file out + - Not listed: changes to upstream's implementation of streams. src/jsc/bindings/webcore/streams is Bun's own. A changed `.idl` of a stream class is listed when Bun has a `JS.cpp` for it. Also not listed: WebCore code that Bun keeps in a file of another name, unless `upstreamOf` names it - In webkit-changes.md, write how many commits the script listed. For each commit that applies, write the number of the issue or PR that already covers it (search first), of the PR that ports it ("this PR" for a port in the upgrade PR), or of a new issue. Write "already in Bun" for a fix that Bun has in another form. Port with a test, after the build below has finished or in a PR of its own - bun run build:local — full Bun build with the local WebKit (same build directory and objects as the JSC build above) - After it compiles, run some code to make sure things work: `bun run build:local -p '42'` diff --git a/scripts/webcore-copies.ts b/scripts/webcore-copies.ts index f5a9580c8cb5..e5dcb7c52b29 100644 --- a/scripts/webcore-copies.ts +++ b/scripts/webcore-copies.ts @@ -3,13 +3,12 @@ * Lists the upstream WebKit commits that touch a file Bun keeps its own copy * of. * - * bun scripts/webcore-copies.ts [] + * bun scripts/webcore-copies.ts * - * Both are revisions in the WebKit clone (vendor/WebKit, or $BUN_WEBKIT_PATH): + * Both are revisions in the WebKit clone ($BUN_WEBKIT_PATH, else vendor/WebKit): * the upstream commit the fork was merged with before, and the one it is - * merged with now (`upstream/main` when left out). The clone needs the history - * between the two. One made with `--filter=blob:none` fetches each file that - * is compared. + * merged with now. The clone needs the history between the two. One made with + * `--filter=blob:none` fetches each file that is compared. * * Bun does not build Source/WebCore. The files under src/jsc/bindings that * come from it (most of them in webcore/ and webcrypto/) were copied by hand @@ -27,22 +26,23 @@ import { basename, dirname, join } from "node:path"; const copiesRoot = "src/jsc/bindings"; const upstreamRoot = "Source/WebCore"; +/** The remote of the clone that is WebKit/WebKit. The upgrade routine asks for this name. */ +const upstreamRemote = "upstream"; /** - * The copies that a file name does not find: Bun's file, then every upstream - * file its code comes from. An entry replaces the match by name, so it also - * has the upstream file of the same name when there is one. + * The upstream files of a copy that its file name does not lead to: Bun's + * file, then each such upstream file. An upstream file with the name of the + * copy is found without an entry. * - * Add an upstream file to an entry, beside the one the code came from, when - * upstream renames or splits a file that has a copy or moves code out of it. - * Add an entry when WebCore code goes into a Bun file of another name. The - * script fails when a file that is named here is not there, so that an entry - * cannot stop matching without notice. + * Add an upstream file here when upstream moves code of a copy into it (it + * renames the copy's file, or splits it), and when WebCore code goes into a + * Bun file of another name. The script fails when upstream no longer has a + * file that is named here, so that an entry cannot stop matching without + * notice. */ export const upstreamOf: Record = { // Upstream moved much of structured clone into JavaScriptCore. Bun's copy still has it in one file. "src/jsc/bindings/webcore/SerializedScriptValue.cpp": [ - "Source/WebCore/bindings/js/SerializedScriptValue.cpp", "Source/JavaScriptCore/runtime/CloneBase.h", "Source/JavaScriptCore/runtime/CloneSerializerBase.h", "Source/JavaScriptCore/runtime/CloneDeserializerBase.h", @@ -50,29 +50,53 @@ export const upstreamOf: Record = { "Source/JavaScriptCore/runtime/StructuredCloneTags.cpp", ], // Upstream split the inline functions off. - "src/jsc/bindings/webcore/EventTarget.h": [ - "Source/WebCore/dom/EventTarget.h", - "Source/WebCore/dom/EventTargetInlines.h", - ], - "src/jsc/bindings/webcore/JSValueInWrappedObject.h": [ - "Source/WebCore/bindings/js/JSValueInWrappedObject.h", - "Source/WebCore/bindings/js/JSValueInWrappedObjectInlines.h", - ], + "src/jsc/bindings/webcore/EventTarget.h": ["Source/WebCore/dom/EventTargetInlines.h"], + "src/jsc/bindings/webcore/JSValueInWrappedObject.h": ["Source/WebCore/bindings/js/JSValueInWrappedObjectInlines.h"], // The OpenSSL half is in a file of its own upstream. Bun wrote its half into the algorithm's file. "src/jsc/bindings/webcrypto/CryptoAlgorithmEd25519.cpp": [ - "Source/WebCore/crypto/algorithms/CryptoAlgorithmEd25519.cpp", "Source/WebCore/crypto/openssl/CryptoAlgorithmEd25519OpenSSL.cpp", ], "src/jsc/bindings/webcrypto/CryptoAlgorithmX25519.cpp": [ - "Source/WebCore/crypto/algorithms/CryptoAlgorithmX25519.cpp", "Source/WebCore/crypto/openssl/CryptoAlgorithmX25519OpenSSL.cpp", ], // Made from the Cocoa file, before upstream had an OpenSSL one. - "src/jsc/bindings/webcrypto/CryptoKeyOKPOpenSSL.cpp": [ - "Source/WebCore/crypto/openssl/CryptoKeyOKPOpenSSL.cpp", - "Source/WebCore/crypto/cocoa/CryptoKeyOKPCocoa.cpp", + "src/jsc/bindings/webcrypto/CryptoKeyOKPOpenSSL.cpp": ["Source/WebCore/crypto/cocoa/CryptoKeyOKPCocoa.cpp"], + // Upstream took the underscore out of these names. + "src/jsc/bindings/webcrypto/CryptoAlgorithmAES_CBC.h": ["Source/WebCore/crypto/algorithms/CryptoAlgorithmAESCBC.h"], + "src/jsc/bindings/webcrypto/CryptoAlgorithmAES_CBC.cpp": [ + "Source/WebCore/crypto/algorithms/CryptoAlgorithmAESCBC.cpp", + ], + "src/jsc/bindings/webcrypto/CryptoAlgorithmAES_CBCOpenSSL.cpp": [ + "Source/WebCore/crypto/openssl/CryptoAlgorithmAESCBCOpenSSL.cpp", + ], + "src/jsc/bindings/webcrypto/CryptoAlgorithmAES_CFB.h": ["Source/WebCore/crypto/algorithms/CryptoAlgorithmAESCFB.h"], + "src/jsc/bindings/webcrypto/CryptoAlgorithmAES_CFB.cpp": [ + "Source/WebCore/crypto/algorithms/CryptoAlgorithmAESCFB.cpp", + ], + "src/jsc/bindings/webcrypto/CryptoAlgorithmAES_CFBOpenSSL.cpp": [ + "Source/WebCore/crypto/openssl/CryptoAlgorithmAESCFBOpenSSL.cpp", + ], + "src/jsc/bindings/webcrypto/CryptoAlgorithmAES_CTR.h": ["Source/WebCore/crypto/algorithms/CryptoAlgorithmAESCTR.h"], + "src/jsc/bindings/webcrypto/CryptoAlgorithmAES_CTR.cpp": [ + "Source/WebCore/crypto/algorithms/CryptoAlgorithmAESCTR.cpp", + ], + "src/jsc/bindings/webcrypto/CryptoAlgorithmAES_CTROpenSSL.cpp": [ + "Source/WebCore/crypto/openssl/CryptoAlgorithmAESCTROpenSSL.cpp", + ], + "src/jsc/bindings/webcrypto/CryptoAlgorithmAES_GCM.h": ["Source/WebCore/crypto/algorithms/CryptoAlgorithmAESGCM.h"], + "src/jsc/bindings/webcrypto/CryptoAlgorithmAES_GCM.cpp": [ + "Source/WebCore/crypto/algorithms/CryptoAlgorithmAESGCM.cpp", + ], + "src/jsc/bindings/webcrypto/CryptoAlgorithmAES_GCMOpenSSL.cpp": [ + "Source/WebCore/crypto/openssl/CryptoAlgorithmAESGCMOpenSSL.cpp", + ], + "src/jsc/bindings/webcrypto/CryptoAlgorithmAES_KW.h": ["Source/WebCore/crypto/algorithms/CryptoAlgorithmAESKW.h"], + "src/jsc/bindings/webcrypto/CryptoAlgorithmAES_KW.cpp": ["Source/WebCore/crypto/algorithms/CryptoAlgorithmAESKW.cpp"], + "src/jsc/bindings/webcrypto/CryptoAlgorithmAES_KWOpenSSL.cpp": [ + "Source/WebCore/crypto/openssl/CryptoAlgorithmAESKWOpenSSL.cpp", ], // WebCore code in a file with a name of Bun's. + "src/jsc/bindings/webcore/CustomEventCustom.cpp": ["Source/WebCore/bindings/js/JSCustomEventCustom.cpp"], "src/jsc/bindings/webcrypto/CryptoDigest.cpp": ["Source/WebCore/PAL/pal/crypto/openssl/CryptoDigestOpenSSL.cpp"], "src/jsc/bindings/BunClientData.h": ["Source/WebCore/bindings/js/WebCoreJSClientData.h"], "src/jsc/bindings/BunClientData.cpp": ["Source/WebCore/bindings/js/WebCoreJSClientData.cpp"], @@ -98,7 +122,7 @@ export interface ChangedFile { upstream: string; /** The new base does not have it: the range deleted it, or moved it. */ gone: boolean; - /** Bun's copies of it, with `compareToUpstream`'s share. */ + /** Bun's copies of it, each with `compareToUpstream`'s share. */ copies: { path: string; share: number | undefined }[]; /** Bun's `JS.cpp`, when `upstream` is `.idl`. */ binding: string | undefined; @@ -122,7 +146,9 @@ export interface CopyChanges { class Failure extends Error {} function git(cwd: string, ...args: string[]): string { - const { exitCode, stdout, stderr } = Bun.spawnSync({ cmd: ["git", ...args], cwd, stdout: "pipe", stderr: "pipe" }); + // Without the option, git prints a path that is not ASCII in quotes, with its bytes as numbers. + const cmd = ["git", "-c", "core.quotePath=false", ...args]; + const { exitCode, stdout, stderr } = Bun.spawnSync({ cmd, cwd, stdout: "pipe", stderr: "pipe" }); if (exitCode !== 0) throw new Failure(`\`git ${args.join(" ")}\` failed in ${cwd}:\n${stderr}`); return stdout.toString(); } @@ -134,48 +160,38 @@ const trimmedLines = (text: string) => .filter(line => line !== ""); /** - * Whether `copy` still has text of `upstream`, and how much. + * Whether `copy` still has text of `upstream`, and how much of it. * * Two files with the same name are not always related: Bun's Path.cpp is * node:path, WebCore's draws curves. `isCopy` asks for one shared line of 40 * characters or more. The license header of a real copy has several, and * `#pragma once`, `namespace WebCore {` or `#include "config.h"` are shorter. * - * `share` is the part of the copy's code that the upstream text has too, where + * `share` is the part of the upstream file's code that the copy has too, where * code is every line of 16 characters or more outside a comment (so not a - * brace, `return true;` or the license). It says how well an upstream diff - * lines up with the copy. It falls when Bun edits the copy and also when - * upstream edits, splits or moves its file, so a low share does not mean that - * a fix has nothing to apply to. It is undefined for a copy with fewer than 8 - * lines of code, where one edited line decides it. + * brace, `return true;` or the license). It says how well a diff of the + * upstream file lines up with the copy. It is low when Bun has few of the + * file's functions and when either side has edited them, so a low share does + * not mean that a fix has nothing to apply to. It is undefined for an upstream + * file with fewer than 8 lines of code, where one edited line decides it. */ export function compareToUpstream(copy: string, upstream: string): { isCopy: boolean; share: number | undefined } { - const theirs = new Set(trimmedLines(upstream)); - const code = trimmedLines(copy.replace(/\/\*[\s\S]*?\*\//g, "")).filter( + const ours = new Set(trimmedLines(copy)); + const code = trimmedLines(upstream.replace(/\/\*[\s\S]*?\*\//g, "")).filter( line => line.length >= 16 && !line.startsWith("//"), ); return { - isCopy: trimmedLines(copy).some(line => line.length >= 40 && theirs.has(line)), - share: code.length >= 8 ? code.filter(line => theirs.has(line)).length / code.length : undefined, + isCopy: trimmedLines(upstream).some(line => line.length >= 40 && ours.has(line)), + share: code.length >= 8 ? code.filter(line => ours.has(line)).length / code.length : undefined, }; } -/** - * A copy can have a name that differs a little from upstream's: - * CryptoAlgorithmAES_KW.cpp is CryptoAlgorithmAESKW.cpp there, and - * CustomEventCustom.cpp is JSCustomEventCustom.cpp. A file whose name differs - * in that way must share a quarter of its code to count. - */ -const withoutUnderscores = (path: string) => basename(path).replaceAll("_", ""); -const renamedCopyShare = 0.25; - /** * The commits in `oldBase..newBase` of the WebKit clone at `webkitRoot` that * touch an upstream file with a counterpart among the tracked files under * src/jsc/bindings of the bun checkout at `bunRoot`. A counterpart is: + * - a file of the same name that `compareToUpstream` takes for a copy, * - a copy that `table` names for the file, - * - a copy found by name: a file of the same name, or of nearly the same name, - * that passes `compareToUpstream`, * - for `.idl`, the binding `JS.cpp`. WebCore generates that file * from the IDL, and Bun's is kept by hand. */ @@ -188,14 +204,15 @@ export function listCopyChanges( ): CopyChanges { const range = `${oldBase}..${newBase}`; const total = Number(git(webkitRoot, "rev-list", "--count", range)); - if (total === 0) throw new Failure(`No commit is in ${range}. Is the old base set, and is it the older of the two?`); + if (total === 0) throw new Failure(`No commit is in ${range}. Is the old base the older of the two?`); - const tracked = new Set( - trimmedLines(git(bunRoot, "ls-files", "--", copiesRoot)).filter(path => existsSync(join(bunRoot, path))), - ); + const tracked = trimmedLines(git(bunRoot, "ls-files", "--", copiesRoot)); + const missing = tracked.filter(path => !existsSync(join(bunRoot, path))); + if (missing.length > 0) { + throw new Failure(`These tracked files are not in the working tree of ${bunRoot}:\n ${missing.join("\n ")}`); + } const ours = (path: string) => readFileSync(join(bunRoot, path), "utf8"); - const byName = new Map([...tracked].map(path => [basename(path), path])); - const byLooseName = Map.groupBy(tracked, withoutUnderscores); + const byName = Map.groupBy(tracked, path => basename(path)); // `table` the other way round. const namedCopies = new Map(); @@ -207,69 +224,54 @@ export function listCopyChanges( // Which upstream files each end has. A failed read of one of them is then an error, not a file that is not there. const filesAt = (rev: string) => new Set(trimmedLines(git(webkitRoot, "ls-tree", "-r", "--name-only", rev, "--", upstreamRoot, ...outsideRoot))); - const atNew = filesAt(newBase); const atOld = filesAt(oldBase); - const exists = (path: string) => atNew.has(path) || atOld.has(path); - const texts = new Map(); - const theirs = (path: string) => { - // A file that a commit in the range deleted or moved away is read as the old base has it. - if (!texts.has(path)) texts.set(path, git(webkitRoot, "show", `${atNew.has(path) ? newBase : oldBase}:${path}`)); - return texts.get(path)!; - }; + const atNew = filesAt(newBase); + const commitsWith = (path: string, ...revisions: string[]) => + trimmedLines(git(webkitRoot, "rev-list", ...revisions, "--", path)); - // An entry that matches nothing must not go unseen. A file that neither end has is fine only when a commit outside - // the history of the old base has it: upstream adds it later, or adds and removes it inside the range. + // An entry that matches nothing must not go unseen. A file that the old base does not have is fine only when an + // upstream commit that comes after the old base has it: one in the range, or a later one on a branch of the + // upstream remote. A commit of the fork does not count. Neither does one on a branch that left before the old + // base, such as a release branch. const wrong = Object.keys(table) - .filter(copy => !tracked.has(copy)) + .filter(copy => !tracked.includes(copy)) .map(copy => `${copy} is not a tracked file.`); + const after = [newBase, `--remotes=${upstreamRemote}`, "--ancestry-path", `^${oldBase}`]; for (const path of namedCopies.keys()) { - if (exists(path) || git(webkitRoot, "rev-list", "-1", "--all", `^${oldBase}`, "--", path) !== "") continue; - const removal = git(webkitRoot, "rev-list", "-1", oldBase, "--", path).trim(); + if (atOld.has(path) || commitsWith(path, "-1", ...after).length > 0) continue; + const [removal] = commitsWith(path, "-1", oldBase); wrong.push( - removal === "" - ? `${path} is in no commit of the clone.` + removal === undefined + ? `${path} is not in the history of ${oldBase}, and no later upstream commit has it.` : `${path} is gone since ${removal.slice(0, 12)}. Name the file that has its code now, or take it out.`, ); } if (wrong.length > 0) throw new Failure(`\`upstreamOf\` in scripts/webcore-copies.ts:\n ${wrong.join("\n ")}`); - const shares = new Map(); - const namedShare = (copy: string) => { - if (!shares.has(copy)) { - const upstream = table[copy].filter(exists).map(theirs); - shares.set(copy, upstream.length > 0 ? compareToUpstream(ours(copy), upstream.join("\n")).share : undefined); + // A copy is held against the text that the commits of the range change: the file as the old base has it, or as the + // first commit of the range that has it. + const texts = new Map(); + const theirs = (path: string) => { + if (!texts.has(path)) { + const [revision] = atOld.has(path) ? [oldBase] : commitsWith(path, "--reverse", range); + texts.set(path, git(webkitRoot, "show", `${revision}:${path}`)); } - return shares.get(copy); + return texts.get(path)!; }; - const changedFiles = new Map(); - const changedFile = (upstream: string) => { - if (changedFiles.has(upstream)) return changedFiles.get(upstream); - const copies = (namedCopies.get(upstream) ?? []).map(path => ({ path, share: namedShare(path) })); - const name = withoutUnderscores(upstream); - const found = [ - ...(byLooseName.get(name) ?? []), - ...((name.startsWith("JS") && byLooseName.get(name.slice(2))) || []), - ]; - for (const path of found) { - if (Object.hasOwn(table, path)) continue; - if (!exists(upstream)) { - // Added and removed inside the range: there is no text to hold the file against, so it stays listed. - copies.push({ path, share: undefined }); - continue; - } + const changedFile = (upstream: string): ChangedFile | undefined => { + const named = namedCopies.get(upstream) ?? []; + const sameName = (byName.get(basename(upstream)) ?? []).filter(path => !named.includes(path)); + const copies: ChangedFile["copies"] = []; + for (const path of [...named, ...sameName]) { const { isCopy, share } = compareToUpstream(ours(path), theirs(upstream)); - const sameName = basename(path) === basename(upstream); - if (isCopy && (sameName || (share ?? 0) >= renamedCopyShare)) copies.push({ path, share }); + // The table says that a file it names is a copy. A file of the same name must show it. + if (isCopy || named.includes(path)) copies.push({ path, share }); } const idl = basename(upstream).endsWith(".idl") ? basename(upstream).slice(0, -".idl".length) : undefined; - const binding = idl === undefined ? undefined : byName.get(`JS${idl}.cpp`); - const file = - copies.length > 0 || binding !== undefined - ? { upstream, gone: !atNew.has(upstream), copies, binding } - : undefined; - changedFiles.set(upstream, file); - return file; + const binding = idl === undefined ? undefined : byName.get(`JS${idl}.cpp`)?.[0]; + if (copies.length === 0 && binding === undefined) return undefined; + return { upstream, gone: !atNew.has(upstream), copies, binding }; }; // Per commit: \0 \0 \0 \0, then one path per line. @@ -307,9 +309,9 @@ export function formatCopyChanges({ commits, total }: CopyChanges, range: string /** The command: what it prints for `args`, and its exit code. */ export function main(args: string[], bunRoot: string, webkitRoot: string) { - const [oldBase, newBase = "upstream/main"] = args; + const [oldBase, newBase] = args; if (!oldBase || !newBase || !existsSync(webkitRoot)) { - const usage = "usage: bun scripts/webcore-copies.ts []"; + const usage = "usage: bun scripts/webcore-copies.ts "; const clone = `It reads the WebKit clone at ${webkitRoot}. Set BUN_WEBKIT_PATH to use another one.`; return { exitCode: 1, stdout: "", stderr: `${usage}\n${clone}` }; } diff --git a/test/internal/webcore-copies.test.ts b/test/internal/webcore-copies.test.ts index 711a4c02411b..7b8e27da7057 100644 --- a/test/internal/webcore-copies.test.ts +++ b/test/internal/webcore-copies.test.ts @@ -4,7 +4,7 @@ * reads two git repositories: a bun checkout and a WebKit clone. Here both are a few files with the shape of the real * ones. */ -import { afterAll, expect, test } from "bun:test"; +import { afterAll, beforeAll, expect, test } from "bun:test"; import { bunEnv, bunExe, tempDir } from "harness"; import { chmodSync, readFileSync, rmSync } from "node:fs"; import { dirname, join } from "node:path"; @@ -41,17 +41,18 @@ interface Commit { } /** - * Adds `commits` to `main` with one git process, which makes no use of the machine's git configuration (identity, - * signing, hooks). The author's date is not the committer's, and the objects stay loose files. + * Adds `commits` to the branch `to` with one git process, which makes no use of the machine's git configuration + * (identity, signing, hooks). The first of them comes after `from`, or after nothing. The author's date is not the + * committer's, and the objects stay loose files. */ -function addCommits(dir: string, commits: Commit[], parent = "") { +function addCommits(dir: string, commits: Commit[], { to = "refs/heads/main", from = "" } = {}) { const data = (text: string) => `data ${Buffer.byteLength(text)}\n${text}\n`; const stream = commits.map(({ message, files, ref }, index) => [ - `commit refs/heads/main\nmark :${index + 1}\n`, + `commit ${to}\nmark :${index + 1}\n`, "author Test 1756728000 +0000\ncommitter Test 1756900800 +0000\n", data(message), - index === 0 ? parent : "", + index === 0 && from !== "" ? `from ${from}\n` : "", ...Object.entries(files).map(([path, text]) => text === null ? `D ${path}\n` : `M 100644 inline ${path}\n${data(text)}`, ), @@ -143,7 +144,7 @@ bool Path::contains(const FloatPoint& point, WindRule rule) const } // namespace WebCore `; -// A copy with an underscore that upstream's name does not have. +// A copy that upstream renamed: its name has an underscore that upstream's no longer has. The table names it. const aesKw = (header: string, failed: string) => `${license} #include "${header}" @@ -159,73 +160,38 @@ static std::optional> cryptUnwrapKey(const Vector& key, } // namespace WebCore `; +const bunAesKw = "src/jsc/bindings/webcrypto/CryptoAlgorithmAES_KW.cpp"; +const upstreamAesKw = "Source/WebCore/crypto/algorithms/CryptoAlgorithmAESKW.cpp"; -// A copy without the JS that upstream's name starts with. -const customEvent = (detail: string) => `${license} -#include "config.h" -#include "JSCustomEvent.h" - -namespace WebCore { - -JSC::JSValue JSCustomEvent::detail(JSC::JSGlobalObject& lexicalGlobalObject) const -{ - return cachedPropertyValue(lexicalGlobalObject, *this, wrapped().cachedDetail(), [this] { - return ${detail}; - }); -} - -template -void JSCustomEvent::visitAdditionalChildren(Visitor& visitor) -{ - wrapped().detail().visit(visitor); - wrapped().cachedDetail().visit(visitor); -} - -} // namespace WebCore -`; - -// Two files with nearly the same name, and the same license, that are not related: upstream's has a JS in front. -const upstreamCustom = (world: string) => `${license} -#include "config.h" -#include "JSPerformanceObserver.h" - -bool JSPerformanceObserverOwner::isReachableFromOpaqueRoots(JSC::Handle handle, void*, JSC::AbstractSlotVisitor& visitor, ASCIILiteral* reason) -{ - return JSC::jsCast(handle.slot()->asCell())->wrapped().isRegistered(${world}); -} -`; -const bunCustom = `${license} -#include "PerformanceObserverCustom.h" - -JSC_DEFINE_HOST_FUNCTION(jsPerformanceObserverPrototypeFunction_observe, (JSGlobalObject* globalObject, CallFrame* callFrame)) -{ - auto* thisObject = jsDynamicCast(callFrame->thisValue()); - auto options = convert>(*globalObject, callFrame->argument(0)); - return JSValue::encode(toJS(*globalObject, thisObject->wrapped().observe(WTF::move(options)))); -} - -JSC_DEFINE_HOST_FUNCTION(jsPerformanceObserverPrototypeFunction_disconnect, (JSGlobalObject* globalObject, CallFrame* callFrame)) -{ - auto* thisObject = jsDynamicCast(callFrame->thisValue()); - thisObject->wrapped().disconnect(); - return JSValue::encode(jsUndefined()); -} -`; - -// A copy whose code upstream has in two files. The table names both: no file name leads to the second one. -const cloneSerializer = ` void dumpArrayBufferView(JSObject* object, SerializationReturnCode& code); - bool dumpIfTerminal(JSValue value, SerializationReturnCode& code); - void write(SerializationTag tag) { writeLittleEndian(m_buffer, static_cast(tag)); } - void write(const String& string) { write(string.impl()); } -`; +// A copy whose code upstream has in two files. Its name leads to the first. The table names the second, which is +// not under Source/WebCore, and names it for Bun's header too. const serializedScriptValue = `RefPtr SerializedScriptValue::create(JSGlobalObject& globalObject, JSValue value) auto serializer = CloneSerializer(globalObject, messagePorts, arrayBuffers); Vector buffer = serializer.takeBuffer(); return adoptRef(*new SerializedScriptValue(WTF::move(buffer))); `; -const bunSerializer = "src/jsc/bindings/webcore/SerializedScriptValue.cpp"; +const cloneSerializer = [ + " void dumpString(const String& string);", + " void dumpBigIntData(JSBigInt* bigInt);", + " void write(SerializationTag tag) { writeLittleEndian(m_buffer, static_cast(tag)); }", + " void write(const String& string) { write(string.impl()); }", + " void dumpArrayBufferView(JSObject* object, SerializationReturnCode& code);", + " void dumpImmediate(JSValue value, SerializationReturnCode& code);", + " bool dumpIfTerminal(JSValue value, SerializationReturnCode& code);", + " bool startObjectInternal(JSObject* object);", +]; +const upstreamSerializedScriptValue = "Source/WebCore/bindings/js/SerializedScriptValue.cpp"; const upstreamSerializer = "Source/JavaScriptCore/runtime/CloneSerializerBase.h"; -const table = { [bunSerializer]: ["Source/WebCore/bindings/js/SerializedScriptValue.cpp", upstreamSerializer] }; +const bunSerializer = "src/jsc/bindings/webcore/SerializedScriptValue.cpp"; +const bunSerializerHeader = "src/jsc/bindings/webcore/SerializedScriptValue.h"; +// Six of the eight lines of upstream's header are in Bun's .cpp. The first two are in Bun's .h, which has no +// license and no long line: only the table says that it is a copy. +const serializerFiles = { + [bunSerializer]: `${license}\n${serializedScriptValue}${cloneSerializer.slice(2).join("\n")}\n`, + [bunSerializerHeader]: `#pragma once\n${cloneSerializer.slice(0, 2).join("\n")}\n`, +}; +const serializerTable = { [bunSerializer]: [upstreamSerializer], [bunSerializerHeader]: [upstreamSerializer] }; +const table = { ...serializerTable, [bunAesKw]: [upstreamAesKw] }; const renderBox = `${license}\n// Bun has this file too, but not where the copies are.\n`; @@ -233,36 +199,31 @@ const shas = (text: string) => text.replace(/^[0-9a-f]{4,64} /gm, " "); const list = (bun: string, webkit: string, oldBase: string, newBase: string, names: Record = {}) => shas(formatCopyChanges(listCopyChanges(bun, webkit, oldBase, newBase, names), `${oldBase}..${newBase}`)).split("\n"); -// Ten upstream commits and the Bun files that go with them. Nothing changes these two repositories after this. -let repositories: { bun: string; webkit: string; [Symbol.dispose](): void } | undefined; -afterAll(() => repositories?.[Symbol.dispose]()); -function tenCommits() { - if (repositories !== undefined) return repositories; +// Eight upstream commits and the Bun files that go with them. No test changes these two repositories. +let repositories: { bun: string; webkit: string }; +const directories: Disposable[] = []; +afterAll(() => directories.forEach(directory => directory[Symbol.dispose]())); +beforeAll(() => { const bun = checkout({ "src/jsc/bindings/webcore/PerformanceObserver.cpp": bunCopy, "src/jsc/bindings/webcore/JSPerformanceObserver.cpp": `#include "JSPerformanceObserver.h"\n`, - "src/jsc/bindings/webcore/PerformanceObserverCustom.cpp": bunCustom, - "src/jsc/bindings/webcore/CustomEventCustom.cpp": customEvent("wrapped().detail().getValue()"), - "src/jsc/bindings/webcrypto/CryptoAlgorithmAES_KW.cpp": aesKw("CryptoAlgorithmAES_KW.h", "< 0"), - [bunSerializer]: `${license}\n${cloneSerializer}${serializedScriptValue}`, + [bunAesKw]: aesKw("CryptoAlgorithmAES_KW.h", "< 0"), + ...serializerFiles, "src/jsc/bindings/Path.cpp": bunPath, - "src/jsc/bindings/RenderBox.cpp": renderBox, "src/runtime/RenderBox.cpp": renderBox, + // Git prints a name like this one in quotes, with its bytes as numbers, unless it is told not to. + "src/jsc/bindings/webcore/Straße.h": "#pragma once\n", }); - // Tracked, and not in the working tree. - rmSync(join(String(bun), "src/jsc/bindings/RenderBox.cpp")); - + directories.push(bun); const webkit = clone([ { message: "base", files: { "Source/WebCore/page/PerformanceObserver.cpp": upstreamBefore, "Source/WebCore/page/PerformanceObserver.idl": "interface PerformanceObserver {};\n", - "Source/WebCore/bindings/js/JSPerformanceObserverCustom.cpp": upstreamCustom("world"), - "Source/WebCore/bindings/js/JSCustomEventCustom.cpp": customEvent("wrapped().detail().getValue()"), - "Source/WebCore/crypto/algorithms/CryptoAlgorithmAESKW.cpp": aesKw("CryptoAlgorithmAESKW.h", "< 0"), - "Source/WebCore/bindings/js/SerializedScriptValue.cpp": `${license}\n${serializedScriptValue}`, - [upstreamSerializer]: `${license}\n${cloneSerializer}`, + [upstreamAesKw]: aesKw("CryptoAlgorithmAESKW.h", "< 0"), + [upstreamSerializedScriptValue]: `${license}\n${serializedScriptValue}`, + [upstreamSerializer]: `${license}\n${cloneSerializer.join("\n")}\n`, "Source/WebCore/platform/graphics/Path.cpp": webcorePath("false"), "Source/WebCore/rendering/RenderBox.cpp": renderBox, "Tools/TestWebKitAPI/PerformanceObserver.cpp": upstreamBefore, @@ -276,17 +237,13 @@ function tenCommits() { }, }, { - message: "Not listed: Bun's files of this name are elsewhere, or deleted", + message: "Not listed: Bun's file of this name is not where the copies are", files: { "Source/WebCore/rendering/RenderBox.cpp": `${renderBox}// Changed.\n` }, }, { message: "Not listed: Bun's file of this name is not a copy", files: { "Source/WebCore/platform/graphics/Path.cpp": webcorePath("rule == WindRule::NonZero") }, }, - { - message: "Not listed: Bun's file of nearly this name has none of its code", - files: { "Source/WebCore/bindings/js/JSPerformanceObserverCustom.cpp": upstreamCustom("visitor.world()") }, - }, { message: "Not listed: the file is not under Source/WebCore, and the table does not name it", files: { "Tools/TestWebKitAPI/PerformanceObserver.cpp": upstreamAfter }, @@ -300,19 +257,13 @@ function tenCommits() { }, { message: "AES-KW unwrapKey accepts tampered wrapped keys", - files: { "Source/WebCore/crypto/algorithms/CryptoAlgorithmAESKW.cpp": aesKw("CryptoAlgorithmAESKW.h", "<= 0") }, - }, - { - message: "CustomEvent.detail is null before initCustomEvent()", - files: { - "Source/WebCore/bindings/js/JSCustomEventCustom.cpp": customEvent("wrapped().detail().getValue(JSC::jsNull())"), - }, + files: { [upstreamAesKw]: aesKw("CryptoAlgorithmAESKW.h", "<= 0") }, }, { message: "Serialize a resizable ArrayBuffer with its maximum length", files: { - [upstreamSerializer]: `${license}\n${cloneSerializer} void write(uint64_t);\n`, - "Source/WebCore/bindings/js/SerializedScriptValue.cpp": `${license}\n${serializedScriptValue}// Changed.\n`, + [upstreamSerializer]: `${license}\n${cloneSerializer.join("\n")}\n void write(uint64_t);\n`, + [upstreamSerializedScriptValue]: `${license}\n${serializedScriptValue}// Changed.\n`, }, }, { @@ -323,53 +274,51 @@ function tenCommits() { }, }, ]); - return (repositories = { - bun: String(bun), - webkit: String(webkit), - [Symbol.dispose]() { - bun[Symbol.dispose](); - webkit[Symbol.dispose](); - }, - }); -} + directories.push(webkit); + repositories = { bun: String(bun), webkit: String(webkit) }; +}); test("lists the commits that touch the upstream file of a copy or the IDL of a binding, and no others", () => { - const { bun, webkit } = tenCommits(); + const { bun, webkit } = repositories; expect(list(bun, webkit, "base", "main", table)).toEqual([ - // A moved file is listed under both paths. `main` does not have the old one, which is compared as `base` has it. + // A moved file is listed under both paths. `main` does not have the old one. `base` does not have the new one, + // which is compared as the commit that adds it has it: with the fix, so the copy has less of it. " 2025-09-03 Move PerformanceObserver to Modules/performance", - " Source/WebCore/Modules/performance/PerformanceObserver.cpp -> src/jsc/bindings/webcore/PerformanceObserver.cpp (88% same)", + " Source/WebCore/Modules/performance/PerformanceObserver.cpp -> src/jsc/bindings/webcore/PerformanceObserver.cpp (70% same)", " Source/WebCore/page/PerformanceObserver.cpp (gone) -> src/jsc/bindings/webcore/PerformanceObserver.cpp (88% same)", - // The two upstream files that the table names for this copy, one of them outside Source/WebCore. Each has half of - // the copy's code, and the share is of both together. The copy is listed once, not again for its name. + // The table names the first file for two copies. Each has its own part of that file. The name of the copy leads + // to the second file, which has fewer than eight lines of code, so no share. " 2025-09-03 Serialize a resizable ArrayBuffer with its maximum length", - " Source/JavaScriptCore/runtime/CloneSerializerBase.h -> src/jsc/bindings/webcore/SerializedScriptValue.cpp (100% same)", - " Source/WebCore/bindings/js/SerializedScriptValue.cpp -> src/jsc/bindings/webcore/SerializedScriptValue.cpp (100% same)", - // Two copies under nearly the name of the upstream file. Neither has the line that the commit changes. - " 2025-09-03 CustomEvent.detail is null before initCustomEvent()", - " Source/WebCore/bindings/js/JSCustomEventCustom.cpp -> src/jsc/bindings/webcore/CustomEventCustom.cpp (91% same)", + ` ${upstreamSerializer} -> ${bunSerializer} (75% same), ${bunSerializerHeader} (25% same)`, + ` ${upstreamSerializedScriptValue} -> ${bunSerializer}`, + // The share is of the file as `base` has it: the copy has seven of its eight lines. It has one fewer of the + // lines that `main` has, because it does not have this fix. " 2025-09-03 AES-KW unwrapKey accepts tampered wrapped keys", - " Source/WebCore/crypto/algorithms/CryptoAlgorithmAESKW.cpp -> src/jsc/bindings/webcrypto/CryptoAlgorithmAES_KW.cpp (75% same)", + ` ${upstreamAesKw} -> ${bunAesKw} (88% same)`, " 2025-09-03 Expose PerformanceObserver.supportedEntryTypes", " Source/WebCore/page/PerformanceObserver.idl -> src/jsc/bindings/webcore/JSPerformanceObserver.cpp (binding)", // The first line of the message, the committer's date, and only the file that Bun has a copy of. " 2025-09-03 Schedule task on buffered PerformanceObserver", " Source/WebCore/page/PerformanceObserver.cpp (gone) -> src/jsc/bindings/webcore/PerformanceObserver.cpp (88% same)", "", - "Commits in base..main that touch the upstream file of a Bun copy: 6 of 10", + "Commits in base..main that touch the upstream file of a Bun copy: 5 of 8", ]); }); test("a range in which no commit touches a copy prints the count alone", () => { - const { bun, webkit } = tenCommits(); - // The four commits in the middle of the ten. - expect(list(bun, webkit, "main~9", "main~5", table)).toEqual([ - "Commits in main~9..main~5 that touch the upstream file of a Bun copy: 0 of 4", + const { bun, webkit } = repositories; + // The three commits that are not listed. + expect(list(bun, webkit, "main~7", "main~4", table)).toEqual([ + "Commits in main~7..main~4 that touch the upstream file of a Bun copy: 0 of 3", + ]); + // The AES-KW commit, without the table: a name that is nearly the same leads to no copy. + expect(list(bun, webkit, "main~3", "main~2")).toEqual([ + "Commits in main~3..main~2 that touch the upstream file of a Bun copy: 0 of 1", ]); }); test("a range with no commits is an error", () => { - const { bun, webkit } = tenCommits(); + const { bun, webkit } = repositories; // The arguments the wrong way round, and an old base that was never set. expect(() => listCopyChanges(bun, webkit, "main", "base", table)).toThrow("No commit is in main..base."); expect(() => listCopyChanges(bun, webkit, "main", "main", table)).toThrow("No commit is in main..main."); @@ -383,90 +332,116 @@ test("an upstream file that git cannot read is an error, and not a file without { message: "Schedule task on buffered PerformanceObserver", files: { [path]: upstreamAfter } }, ]); // What a clone without blobs has when it cannot fetch one: the tree names the file, and its content is not there. - const blob = git(String(webkit), ["rev-parse", `main:${path}`]); + const blob = git(String(webkit), ["rev-parse", `base:${path}`]); const object = join(String(webkit), ".git", "objects", blob.slice(0, 2), blob.slice(2)); chmodSync(object, 0o666); rmSync(object); expect(() => listCopyChanges(String(bun), String(webkit), "base", "main", {})).toThrow( - `\`git show main:${path}\` failed`, + `\`git show base:${path}\` failed`, + ); +}); + +test("a tracked file that the working tree does not have is an error", () => { + const { webkit } = repositories; + const path = "src/jsc/bindings/webcore/PerformanceObserver.cpp"; + using bun = checkout({ [path]: bunCopy, "src/jsc/bindings/webcore/Performance.cpp": bunCopy }); + rmSync(join(String(bun), path)); + // Without the error, the commits that touch its upstream file would not be listed, and nothing would say so. + expect(() => listCopyChanges(String(bun), webkit, "base", "main", {})).toThrow( + `These tracked files are not in the working tree of ${bun}:\n ${path}`, ); }); -test("a table that names a file which is not there is an error", () => { - const webcoreFile = "Source/WebCore/bindings/js/SerializedScriptValue.cpp"; - using bun = checkout({ [bunSerializer]: `${license}\n${cloneSerializer}${serializedScriptValue}` }); +test("a table that names a file which upstream no longer has is an error", () => { + using bun = checkout(serializerFiles); using webkit = clone([ { message: "base", files: { - [upstreamSerializer]: `${license}\n${cloneSerializer}`, - [webcoreFile]: `${license}\n${serializedScriptValue}`, + [upstreamSerializer]: `${license}\n${cloneSerializer.join("\n")}\n`, + [upstreamSerializedScriptValue]: `${license}\n${serializedScriptValue}`, }, }, { message: "Move the serializer to CloneSerializer.h", - files: { [upstreamSerializer]: null, "Source/JavaScriptCore/runtime/CloneSerializer.h": cloneSerializer }, + files: { + [upstreamSerializer]: null, + "Source/JavaScriptCore/runtime/CloneSerializer.h": cloneSerializer.join("\n"), + }, ref: "refs/heads/moved", }, - { message: "Another change", files: { [webcoreFile]: `${license}\n` }, ref: "refs/heads/later" }, + { + message: "Another change", + files: { [upstreamSerializedScriptValue]: `${license}\n// Changed.\n${serializedScriptValue}` }, + ref: "refs/heads/later", + }, ]); - const [bunRoot, dir] = [String(bun), String(webkit)]; + const [bunRoot, dir, table] = [String(bun), String(webkit), serializerTable]; const removal = git(dir, ["rev-parse", "moved"]).slice(0, 12); + const gone = `\`upstreamOf\` in scripts/webcore-copies.ts:\n ${upstreamSerializer} is gone since ${removal}.`; // The range that removes the file lists it. After that range the entry would match nothing, and nobody would know. expect(list(bunRoot, dir, "base", "moved", table)).toEqual([ " 2025-09-03 Move the serializer to CloneSerializer.h", - ` ${upstreamSerializer} (gone) -> ${bunSerializer} (100% same)`, + ` ${upstreamSerializer} (gone) -> ${bunSerializer} (75% same), ${bunSerializerHeader} (25% same)`, "", "Commits in base..moved that touch the upstream file of a Bun copy: 1 of 1", ]); - expect(() => listCopyChanges(bunRoot, dir, "moved", "later", table)).toThrow( - `\`upstreamOf\` in scripts/webcore-copies.ts:\n ${upstreamSerializer} is gone since ${removal}.`, - ); + expect(() => listCopyChanges(bunRoot, dir, "moved", "later", table)).toThrow(gone); + + // Two commits that still have the file do not change that. The upgrade goes to neither: a release branch of + // upstream that left before the move, and the fork, which kept the file when it merged `later`. + const withFile = { message: "Keep the serializer", files: { [upstreamSerializer]: `${license}\n` } }; + addCommits(dir, [withFile], { to: "refs/remotes/upstream/safari-branch", from: "refs/heads/base" }); + addCommits(dir, [withFile], { to: "refs/remotes/origin/main", from: "refs/heads/later" }); + expect(() => listCopyChanges(bunRoot, dir, "moved", "later", table)).toThrow(gone); // A name that upstream never had, and a copy that Bun does not have. - const typo = { [bunSerializer]: [...table[bunSerializer], "Source/JavaScriptCore/runtime/CloneSerialiserBase.h"] }; - expect(() => listCopyChanges(bunRoot, dir, "base", "later", typo)).toThrow( - "Source/JavaScriptCore/runtime/CloneSerialiserBase.h is in no commit of the clone.", + const typo = "Source/JavaScriptCore/runtime/CloneSerialiserBase.h"; + expect(() => listCopyChanges(bunRoot, dir, "base", "later", { [bunSerializer]: [typo] })).toThrow( + `${typo} is not in the history of base, and no later upstream commit has it.`, ); expect(() => listCopyChanges(bunRoot, dir, "base", "later", { "src/jsc/bindings/Gone.cpp": [] })).toThrow( "src/jsc/bindings/Gone.cpp is not a tracked file.", ); - // Upstream brings the file back. In the range before that, neither end has it and the table is right. - const reland = { - message: "Reland the serializer", - files: { [upstreamSerializer]: `${license}\n${cloneSerializer}` }, - }; - addCommits(dir, [reland], "from refs/heads/main^0\n"); - expect(list(bunRoot, dir, "moved", "later", table)).toEqual([ + // Upstream brings the file back after `later`. In the range before that, neither end has it and the table is right. + // The table can also name the file that the copy's name leads to: the copy is then listed once. + const reland = { message: "Reland the serializer", files: { [upstreamSerializer]: `${license}\n` } }; + addCommits(dir, [reland], { to: "refs/remotes/upstream/main", from: "refs/heads/later" }); + const both = { ...table, [bunSerializer]: [upstreamSerializer, upstreamSerializedScriptValue] }; + expect(list(bunRoot, dir, "moved", "later", both)).toEqual([ " 2025-09-03 Another change", - ` ${webcoreFile} -> ${bunSerializer} (0% same)`, + ` ${upstreamSerializedScriptValue} -> ${bunSerializer}`, "", "Commits in moved..later that touch the upstream file of a Bun copy: 1 of 1", ]); }); -test("a file that the range adds and removes again is listed without a share", () => { +test("a file that the range adds and removes again is compared as the commit that adds it has it", () => { const path = "Source/WebCore/page/PerformanceTiming.cpp"; - using bun = checkout({ "src/jsc/bindings/webcore/PerformanceTiming.cpp": bunCopy }); + const unrelated = "Source/WebCore/platform/graphics/Path.cpp"; + using bun = checkout({ + "src/jsc/bindings/webcore/PerformanceTiming.cpp": bunCopy, + "src/jsc/bindings/Path.cpp": bunPath, + }); using webkit = clone([ { message: "base", files: { "Source/WebCore/page/Performance.cpp": upstreamBefore } }, - { message: "Add PerformanceTiming", files: { [path]: upstreamBefore } }, - { message: "Revert: Add PerformanceTiming", files: { [path]: null } }, + { message: "Add PerformanceTiming", files: { [path]: upstreamBefore, [unrelated]: webcorePath("false") } }, + { message: "Revert: Add PerformanceTiming", files: { [path]: null, [unrelated]: null } }, ]); - // Neither end has the file, so there is nothing to hold Bun's file against: not when its name leads to the upstream - // file, and not when the table does. + // Neither end has the two files. Bun's Path.cpp is not a copy of the one that was added, and is not listed. const listed = [ " 2025-09-03 Revert: Add PerformanceTiming", - ` ${path} (gone) -> src/jsc/bindings/webcore/PerformanceTiming.cpp`, + ` ${path} (gone) -> src/jsc/bindings/webcore/PerformanceTiming.cpp (88% same)`, " 2025-09-03 Add PerformanceTiming", - ` ${path} (gone) -> src/jsc/bindings/webcore/PerformanceTiming.cpp`, + ` ${path} (gone) -> src/jsc/bindings/webcore/PerformanceTiming.cpp (88% same)`, "", "Commits in base..main that touch the upstream file of a Bun copy: 2 of 2", ]; expect(list(String(bun), String(webkit), "base", "main")).toEqual(listed); + // The table can name such a file. const named = { "src/jsc/bindings/webcore/PerformanceTiming.cpp": [path] }; expect(list(String(bun), String(webkit), "base", "main", named)).toEqual(listed); }); @@ -482,26 +457,24 @@ test("every copy that upstreamOf names is a tracked file", () => { }); test("a file of the same name is a copy only while it shares a long line with the upstream file", () => { - // Seven of the copy's eight lines of code are in the upstream file. The short lines (`if (isBuffered)`, `deliver();`, - // braces) and the license are not counted. - expect(compareToUpstream(bunCopy, upstreamAfter)).toEqual({ isCopy: true, share: 7 / 8 }); + // The copy has seven of the eight lines of code of the upstream file. The short lines (`if (isBuffered)`, + // `deliver();`, braces) and the license are not counted. The fix is two more lines that the copy does not have. + expect(compareToUpstream(bunCopy, upstreamBefore)).toEqual({ isCopy: true, share: 7 / 8 }); + expect(compareToUpstream(bunCopy, upstreamAfter)).toEqual({ isCopy: true, share: 7 / 10 }); expect(compareToUpstream(upstreamAfter, upstreamAfter)).toEqual({ isCopy: true, share: 1 }); // `#include "Path.h"` is in both files, and is too short to make one a copy of the other. expect(compareToUpstream(bunPath, webcorePath("false"))).toEqual({ isCopy: false, share: undefined }); - // The license alone makes it a copy. Fewer than eight lines of code have no share: the copy, less one line. - expect(compareToUpstream(`${license}\n#pragma once\n`, upstreamAfter)).toEqual({ isCopy: true, share: undefined }); - expect(compareToUpstream(bunCopy.replace(" m_registered = true;\n", ""), upstreamAfter)).toEqual({ - isCopy: true, - share: undefined, - }); - // So a file of nearly the same name needs more than the license: a quarter of its code. - expect(compareToUpstream(bunCustom, upstreamCustom("world"))).toEqual({ isCopy: true, share: 0 }); - // A comment is not code: not one that Bun added, and not a license without an asterisk in front of its lines. - const commented = bunCopy.replace("namespace WebCore {", "// Bun has no document here.\nnamespace WebCore {"); - expect(compareToUpstream(commented, upstreamAfter)).toEqual({ isCopy: true, share: 7 / 8 }); + // The license alone makes it a copy. + expect(compareToUpstream(`${license}\n#pragma once\n`, upstreamBefore)).toEqual({ isCopy: true, share: 0 }); + // An upstream file with fewer than eight lines of code has no share: the file, less one line. + const sevenLines = upstreamBefore.replace(" m_registered = true;\n", ""); + expect(compareToUpstream(bunCopy, sevenLines)).toEqual({ isCopy: true, share: undefined }); + // A comment is not code: not a line of its own, and not a license without an asterisk in front of its lines. + const commented = upstreamBefore.replace("namespace WebCore {", "// Workers have no document.\nnamespace WebCore {"); + expect(compareToUpstream(bunCopy, commented)).toEqual({ isCopy: true, share: 7 / 8 }); const plainLicense = license.replaceAll("\n * ", "\n "); expect( - compareToUpstream(bunCopy.replace(license, plainLicense), upstreamAfter.replace(license, plainLicense)), + compareToUpstream(bunCopy.replace(license, plainLicense), upstreamBefore.replace(license, plainLicense)), ).toEqual({ isCopy: true, share: 7 / 8 }); }); @@ -539,24 +512,31 @@ test("the command prints the list, or says what is wrong", () => { const { exitCode, stdout, stderr } = main(args, root, clone); return { exitCode, stdout: shas(stdout), stderr }; }; - expect(command(["base"])).toEqual({ exitCode: 0, stdout: listedForThisCheckout, stderr: "" }); - - // `bun scripts/webcore-copies.ts $OLD_BASE upstream/main` with the variable unset. - expect(command(["upstream/main"])).toEqual({ + expect(command(["base", "upstream/main"])).toEqual({ exitCode: 0, stdout: listedForThisCheckout, stderr: "" }); + expect(command(["upstream/main", "base"])).toEqual({ exitCode: 1, stdout: "", - stderr: "No commit is in upstream/main..upstream/main. Is the old base set, and is it the older of the two?", + stderr: "No commit is in upstream/main..base. Is the old base the older of the two?", }); - // An argument that is missing or empty, and a clone that is not there. + + // A revision that is missing or empty, and a clone that is not there. The checklist's command has one argument + // when `$OLD_BASE` is not set. const usage = (clone: string) => ({ exitCode: 1, stdout: "", stderr: - "usage: bun scripts/webcore-copies.ts []\n" + + "usage: bun scripts/webcore-copies.ts \n" + `It reads the WebKit clone at ${clone}. Set BUN_WEBKIT_PATH to use another one.`, }); const nowhere = join(String(webkit), "nowhere"); - expect([command([]), command(["", "upstream/main"]), command(["base", ""]), command(["base"], nowhere)]).toEqual([ + expect([ + command([]), + command(["upstream/main"]), + command(["", "upstream/main"]), + command(["base", ""]), + command(["base", "upstream/main"], nowhere), + ]).toEqual([ + usage(String(webkit)), usage(String(webkit)), usage(String(webkit)), usage(String(webkit)), @@ -564,28 +544,35 @@ test("the command prints the list, or says what is wrong", () => { ]); }); -// Each run starts a Bun, which takes seconds for a debug build on a busy machine. +async function script(clone: string, ...args: string[]) { + await using proc = Bun.spawn({ + cmd: [bunExe(), join(root, "scripts", "webcore-copies.ts"), ...args], + env: { ...bunEnv, BUN_WEBKIT_PATH: clone }, + cwd: dirname(clone), + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + return { stdout: shas(stdout), stderr, exitCode }; +} + +// These two are not concurrent: two debug builds of Bun that start together take longer than one after the other. test("the script finds this checkout from where it is, and the clone from $BUN_WEBKIT_PATH", async () => { using webkit = cloneForThisCheckout(); - const script = async (...args: string[]) => { - await using proc = Bun.spawn({ - cmd: [bunExe(), join(root, "scripts", "webcore-copies.ts"), ...args], - env: { ...bunEnv, BUN_WEBKIT_PATH: String(webkit) }, - cwd: dirname(String(webkit)), - stdout: "pipe", - stderr: "pipe", - }); - const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); - return { stdout: shas(stdout), stderr, exitCode }; - }; - expect(await Promise.all([script("base"), script()])).toEqual([ - { stdout: `${listedForThisCheckout}\n`, stderr: "", exitCode: 0 }, - { - stdout: "", - stderr: - "usage: bun scripts/webcore-copies.ts []\n" + - `It reads the WebKit clone at ${webkit}. Set BUN_WEBKIT_PATH to use another one.\n`, - exitCode: 1, - }, - ]); -}, 30_000); + expect(await script(String(webkit), "base", "upstream/main")).toEqual({ + stdout: `${listedForThisCheckout}\n`, + stderr: "", + exitCode: 0, + }); +}); + +test("the script prints what is wrong to stderr, and exits with 1", async () => { + using directory = tempDir("webcore-copies-no-arguments", {}); + expect(await script(String(directory))).toEqual({ + stdout: "", + stderr: + "usage: bun scripts/webcore-copies.ts \n" + + `It reads the WebKit clone at ${directory}. Set BUN_WEBKIT_PATH to use another one.\n`, + exitCode: 1, + }); +}); From aba2b0c36a217eded82c156d2328f8377f8d1bc5 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 29 Oct 2026 12:52:17 +0000 Subject: [PATCH 3/3] webcore-copies: spawn git with Bun.spawn in the test, say how to find moved code - The test's git helper uses Bun.spawn and awaits the pipes and the exit code together. The builders of the repositories are async. - The "(gone)" line of the checklist says when `git show --stat` shows where the code went, and gives a `git grep` for the other case. --- .claude/commands/upgrade-webkit.md | 2 +- test/internal/webcore-copies.test.ts | 78 +++++++++++++--------------- 2 files changed, 38 insertions(+), 42 deletions(-) diff --git a/.claude/commands/upgrade-webkit.md b/.claude/commands/upgrade-webkit.md index 511c960c4f50..a692c1cf5dc8 100644 --- a/.claude/commands/upgrade-webkit.md +++ b/.claude/commands/upgrade-webkit.md @@ -23,7 +23,7 @@ To do that: - For each commit, read `git -C vendor/WebKit show -- ` beside Bun's copy. A behavior change (a bug fix, a spec update) applies when the copy still has the code that it changes, even where the lines no longer match. If the title names a behavior, try it with the installed `bun`. A refactor that changes no behavior needs no port - "% same" is how much of the code of the upstream file the copy also has. The upstream file is taken as it was before these commits, or as the commit that adds it has it. A file with fewer than 8 lines of code has no number. The number tells how well a diff of the file lines up with the copy. It is low when Bun has few of the file's functions, and when either side has edited them. A low number is no reason to skip a commit - "(binding)" marks Bun's `JS.cpp` for a changed `.idl`: compare the change with that binding - - "(gone)" marks an upstream file that the new base does not have, on each commit that touches it. `git -C vendor/WebKit show --stat ` of the newest of those commits shows where its code went + - "(gone)" marks an upstream file that the new base does not have, on each commit that touches it. `git -C vendor/WebKit show --stat ` of the newest of those commits shows where its code went, if that commit also adds the new file. If an earlier commit added it, `git -C vendor/WebKit grep -l '' -- Source` finds it - `upstreamOf` in the script names the upstream files of a copy that have another file name than the copy. Without an entry, only a file of the same name leads to a copy. When a commit moves code of a copy into an upstream file of another name (it renames the file or splits it), name that file for the copy in `upstreamOf` and run the script again. When the script says that a file in `upstreamOf` "is gone since" a commit, name the file that has its code now, or take the file out - Not listed: changes to upstream's implementation of streams. src/jsc/bindings/webcore/streams is Bun's own. A changed `.idl` of a stream class is listed when Bun has a `JS.cpp` for it. Also not listed: WebCore code that Bun keeps in a file of another name, unless `upstreamOf` names it - In webkit-changes.md, write how many commits the script listed. For each commit that applies, write the number of the issue or PR that already covers it (search first), of the PR that ports it ("this PR" for a port in the upgrade PR), or of a new issue. Write "already in Bun" for a fix that Bun has in another form. Port with a test, after the build below has finished or in a PR of its own diff --git a/test/internal/webcore-copies.test.ts b/test/internal/webcore-copies.test.ts index 7b8e27da7057..1f897c0ead94 100644 --- a/test/internal/webcore-copies.test.ts +++ b/test/internal/webcore-copies.test.ts @@ -19,8 +19,8 @@ import { const root = join(import.meta.dir, "..", ".."); -function git(cwd: string, args: string[], stdin = "") { - const { exitCode, stdout, stderr } = Bun.spawnSync({ +async function git(cwd: string, args: string[], stdin = "") { + await using proc = Bun.spawn({ cmd: ["git", ...args], cwd, env: bunEnv, @@ -28,8 +28,9 @@ function git(cwd: string, args: string[], stdin = "") { stdout: "pipe", stderr: "pipe", }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); if (exitCode !== 0) throw new Error(`git ${args.join(" ")} exited with ${exitCode}: ${stderr}`); - return stdout.toString().trim(); + return stdout.trim(); } interface Commit { @@ -45,7 +46,7 @@ interface Commit { * (identity, signing, hooks). The first of them comes after `from`, or after nothing. The author's date is not the * committer's, and the objects stay loose files. */ -function addCommits(dir: string, commits: Commit[], { to = "refs/heads/main", from = "" } = {}) { +async function addCommits(dir: string, commits: Commit[], { to = "refs/heads/main", from = "" } = {}) { const data = (text: string) => `data ${Buffer.byteLength(text)}\n${text}\n`; const stream = commits.map(({ message, files, ref }, index) => [ @@ -59,22 +60,22 @@ function addCommits(dir: string, commits: Commit[], { to = "refs/heads/main", fr ref === undefined ? "" : `reset ${ref}\nfrom :${index + 1}\n`, ].join(""), ); - git(dir, ["-c", "fastimport.unpackLimit=100000", "fast-import", "--quiet"], stream.join("\n")); + await git(dir, ["-c", "fastimport.unpackLimit=100000", "fast-import", "--quiet"], stream.join("\n")); } /** A WebKit clone with `commits` on `main`. The branch `base` is the first of them. */ -function clone([base, ...commits]: Commit[]) { +async function clone([base, ...commits]: Commit[]) { const dir = tempDir("webcore-copies-webkit", {}); - git(String(dir), ["init", "--quiet", "--initial-branch=main"]); - addCommits(String(dir), [{ ...base, ref: "refs/heads/base" }, ...commits]); + await git(String(dir), ["init", "--quiet", "--initial-branch=main"]); + await addCommits(String(dir), [{ ...base, ref: "refs/heads/base" }, ...commits]); return dir; } /** A bun checkout where `files` are tracked. */ -function checkout(files: Record) { +async function checkout(files: Record) { const dir = tempDir("webcore-copies-bun", files); - git(String(dir), ["init", "--quiet", "--initial-branch=main"]); - git(String(dir), ["add", "--all"]); + await git(String(dir), ["init", "--quiet", "--initial-branch=main"]); + await git(String(dir), ["add", "--all"]); return dir; } @@ -203,8 +204,8 @@ const list = (bun: string, webkit: string, oldBase: string, newBase: string, nam let repositories: { bun: string; webkit: string }; const directories: Disposable[] = []; afterAll(() => directories.forEach(directory => directory[Symbol.dispose]())); -beforeAll(() => { - const bun = checkout({ +beforeAll(async () => { + const bun = await checkout({ "src/jsc/bindings/webcore/PerformanceObserver.cpp": bunCopy, "src/jsc/bindings/webcore/JSPerformanceObserver.cpp": `#include "JSPerformanceObserver.h"\n`, [bunAesKw]: aesKw("CryptoAlgorithmAES_KW.h", "< 0"), @@ -215,7 +216,7 @@ beforeAll(() => { "src/jsc/bindings/webcore/Straße.h": "#pragma once\n", }); directories.push(bun); - const webkit = clone([ + const webkit = await clone([ { message: "base", files: { @@ -324,15 +325,15 @@ test("a range with no commits is an error", () => { expect(() => listCopyChanges(bun, webkit, "main", "main", table)).toThrow("No commit is in main..main."); }); -test("an upstream file that git cannot read is an error, and not a file without a copy", () => { +test("an upstream file that git cannot read is an error, and not a file without a copy", async () => { const path = "Source/WebCore/page/PerformanceObserver.cpp"; - using bun = checkout({ "src/jsc/bindings/webcore/PerformanceObserver.cpp": bunCopy }); - using webkit = clone([ + using bun = await checkout({ "src/jsc/bindings/webcore/PerformanceObserver.cpp": bunCopy }); + using webkit = await clone([ { message: "base", files: { [path]: upstreamBefore } }, { message: "Schedule task on buffered PerformanceObserver", files: { [path]: upstreamAfter } }, ]); // What a clone without blobs has when it cannot fetch one: the tree names the file, and its content is not there. - const blob = git(String(webkit), ["rev-parse", `base:${path}`]); + const blob = await git(String(webkit), ["rev-parse", `base:${path}`]); const object = join(String(webkit), ".git", "objects", blob.slice(0, 2), blob.slice(2)); chmodSync(object, 0o666); rmSync(object); @@ -341,10 +342,10 @@ test("an upstream file that git cannot read is an error, and not a file without ); }); -test("a tracked file that the working tree does not have is an error", () => { +test("a tracked file that the working tree does not have is an error", async () => { const { webkit } = repositories; const path = "src/jsc/bindings/webcore/PerformanceObserver.cpp"; - using bun = checkout({ [path]: bunCopy, "src/jsc/bindings/webcore/Performance.cpp": bunCopy }); + using bun = await checkout({ [path]: bunCopy, "src/jsc/bindings/webcore/Performance.cpp": bunCopy }); rmSync(join(String(bun), path)); // Without the error, the commits that touch its upstream file would not be listed, and nothing would say so. expect(() => listCopyChanges(String(bun), webkit, "base", "main", {})).toThrow( @@ -352,9 +353,9 @@ test("a tracked file that the working tree does not have is an error", () => { ); }); -test("a table that names a file which upstream no longer has is an error", () => { - using bun = checkout(serializerFiles); - using webkit = clone([ +test("a table that names a file which upstream no longer has is an error", async () => { + using bun = await checkout(serializerFiles); + using webkit = await clone([ { message: "base", files: { @@ -377,7 +378,7 @@ test("a table that names a file which upstream no longer has is an error", () => }, ]); const [bunRoot, dir, table] = [String(bun), String(webkit), serializerTable]; - const removal = git(dir, ["rev-parse", "moved"]).slice(0, 12); + const removal = (await git(dir, ["rev-parse", "moved"])).slice(0, 12); const gone = `\`upstreamOf\` in scripts/webcore-copies.ts:\n ${upstreamSerializer} is gone since ${removal}.`; // The range that removes the file lists it. After that range the entry would match nothing, and nobody would know. @@ -392,8 +393,8 @@ test("a table that names a file which upstream no longer has is an error", () => // Two commits that still have the file do not change that. The upgrade goes to neither: a release branch of // upstream that left before the move, and the fork, which kept the file when it merged `later`. const withFile = { message: "Keep the serializer", files: { [upstreamSerializer]: `${license}\n` } }; - addCommits(dir, [withFile], { to: "refs/remotes/upstream/safari-branch", from: "refs/heads/base" }); - addCommits(dir, [withFile], { to: "refs/remotes/origin/main", from: "refs/heads/later" }); + await addCommits(dir, [withFile], { to: "refs/remotes/upstream/safari-branch", from: "refs/heads/base" }); + await addCommits(dir, [withFile], { to: "refs/remotes/origin/main", from: "refs/heads/later" }); expect(() => listCopyChanges(bunRoot, dir, "moved", "later", table)).toThrow(gone); // A name that upstream never had, and a copy that Bun does not have. @@ -408,7 +409,7 @@ test("a table that names a file which upstream no longer has is an error", () => // Upstream brings the file back after `later`. In the range before that, neither end has it and the table is right. // The table can also name the file that the copy's name leads to: the copy is then listed once. const reland = { message: "Reland the serializer", files: { [upstreamSerializer]: `${license}\n` } }; - addCommits(dir, [reland], { to: "refs/remotes/upstream/main", from: "refs/heads/later" }); + await addCommits(dir, [reland], { to: "refs/remotes/upstream/main", from: "refs/heads/later" }); const both = { ...table, [bunSerializer]: [upstreamSerializer, upstreamSerializedScriptValue] }; expect(list(bunRoot, dir, "moved", "later", both)).toEqual([ " 2025-09-03 Another change", @@ -418,14 +419,14 @@ test("a table that names a file which upstream no longer has is an error", () => ]); }); -test("a file that the range adds and removes again is compared as the commit that adds it has it", () => { +test("a file that the range adds and removes again is compared as the commit that adds it has it", async () => { const path = "Source/WebCore/page/PerformanceTiming.cpp"; const unrelated = "Source/WebCore/platform/graphics/Path.cpp"; - using bun = checkout({ + using bun = await checkout({ "src/jsc/bindings/webcore/PerformanceTiming.cpp": bunCopy, "src/jsc/bindings/Path.cpp": bunPath, }); - using webkit = clone([ + using webkit = await clone([ { message: "base", files: { "Source/WebCore/page/Performance.cpp": upstreamBefore } }, { message: "Add PerformanceTiming", files: { [path]: upstreamBefore, [unrelated]: webcorePath("false") } }, { message: "Revert: Add PerformanceTiming", files: { [path]: null, [unrelated]: null } }, @@ -446,14 +447,9 @@ test("a file that the range adds and removes again is compared as the commit tha expect(list(String(bun), String(webkit), "base", "main", named)).toEqual(listed); }); -test("every copy that upstreamOf names is a tracked file", () => { +test("every copy that upstreamOf names is a tracked file", async () => { const copies = Object.keys(upstreamOf); - const { exitCode, stdout, stderr } = Bun.spawnSync({ cmd: ["git", "ls-files", "--", ...copies], cwd: root }); - expect({ exitCode, stderr: stderr.toString(), tracked: stdout.toString().trim().split("\n").sort() }).toEqual({ - exitCode: 0, - stderr: "", - tracked: copies.toSorted(), - }); + expect((await git(root, ["ls-files", "--", ...copies])).split("\n").sort()).toEqual(copies.toSorted()); }); test("a file of the same name is a copy only while it shares a long line with the upstream file", () => { @@ -481,7 +477,7 @@ test("a file of the same name is a copy only while it shares a long line with th // A clone for this checkout. It has every upstream file that `upstreamOf` names, and the upstream file of one more // copy, with the copy's own text: so the two are related whatever the copy says. const realCopy = "src/jsc/bindings/webcore/PerformanceObserver.cpp"; -function cloneForThisCheckout() { +async function cloneForThisCheckout() { const text = readFileSync(join(root, realCopy), "utf8"); const named = Object.values(upstreamOf).flat(); return clone([ @@ -506,8 +502,8 @@ const listedForThisCheckout = [ "Commits in base..upstream/main that touch the upstream file of a Bun copy: 1 of 1", ].join("\n"); -test("the command prints the list, or says what is wrong", () => { - using webkit = cloneForThisCheckout(); +test("the command prints the list, or says what is wrong", async () => { + using webkit = await cloneForThisCheckout(); const command = (args: string[], clone = String(webkit)) => { const { exitCode, stdout, stderr } = main(args, root, clone); return { exitCode, stdout: shas(stdout), stderr }; @@ -558,7 +554,7 @@ async function script(clone: string, ...args: string[]) { // These two are not concurrent: two debug builds of Bun that start together take longer than one after the other. test("the script finds this checkout from where it is, and the clone from $BUN_WEBKIT_PATH", async () => { - using webkit = cloneForThisCheckout(); + using webkit = await cloneForThisCheckout(); expect(await script(String(webkit), "base", "upstream/main")).toEqual({ stdout: `${listedForThisCheckout}\n`, stderr: "",