From 40e27e4a2fae82b3e501f50e51348bc887ce00dd Mon Sep 17 00:00:00 2001 From: Michael Johnston Date: Sat, 5 Sep 2026 10:59:35 -0700 Subject: [PATCH] feat(lastcode): resume repaired nightly checkpoints --- docs/lastcode/release.md | 35 ++- scripts/lastcode-checkpoint-recovery.test.ts | 232 +++++++++++++++ scripts/lastcode-checkpoint.ts | 290 +++++++++++++++++-- 3 files changed, 528 insertions(+), 29 deletions(-) create mode 100644 scripts/lastcode-checkpoint-recovery.test.ts diff --git a/docs/lastcode/release.md b/docs/lastcode/release.md index 0c4acbbeac37..bd076c0ff962 100644 --- a/docs/lastcode/release.md +++ b/docs/lastcode/release.md @@ -159,7 +159,40 @@ use the `lc-wait-for-checkpoint` trust entry. The command loads from updated observer after the primary checkout is refreshed. If the action is missing or disabled, report that setup problem rather than reverting to a sleep loop. -### Validating a checkpoint +### Resuming a repaired checkpoint + +When a retained nightly rebase is complete but validation required additional +commits, do not delete the worktree and rely on rerere: Git only remembers +conflict resolutions, not subsequent repairs. Commit the repairs and incorporate +any downstream merges made since that attempt before selecting its exact head: + +```bash +pnpm lastcode:checkpoint -- --select-recovery --recovery-source +pnpm lastcode:checkpoint:service run-now +``` + +`--recovery-source` is the operator's assertion that the repaired tree includes +that exact LastCode main revision. Inspect the replay and subsequent commits +before making it; upstream ancestry alone cannot prove this after a rebase. +Selection requires a clean, completed `sync/nightly/` worktree and is +bound to its head, nightly, and current source. The service skips only that +nightly's rebase and reruns the full checkpoint smoke gate. It publishes the +immutable tag and promotes main together with an atomic push leased against the +selected source commit. Open LastCode PRs prevent publication in the service's +normal promotion mode. Selected recovery cannot disable validation or +be automatically superseded. A changed head or source requires inspection and +selection again. Failed validation retains the worktree and selection. + +Use **Wait for Checkpoint** immediately after requesting the service run, and +end the turn. After publication, use **Build Local Package** on the exact new +installable tag; selection and publication do not install or restart the app. +Selected recovery processes only its selected nightly. Request another service +run afterward to continue remaining nightlies from the repaired main branch. +If publication succeeded but cleanup was interrupted, a retry recognizes the +matching immutable tag represented on main and finishes cleanup without +republishing. + +### Validating a checkpoint manually A release build uses a different full-CI context because rebasing intentionally rewrites ancestry. Check out the immutable checkpoint or revision and run: diff --git a/scripts/lastcode-checkpoint-recovery.test.ts b/scripts/lastcode-checkpoint-recovery.test.ts new file mode 100644 index 000000000000..059d8f68fd78 --- /dev/null +++ b/scripts/lastcode-checkpoint-recovery.test.ts @@ -0,0 +1,232 @@ +// @effect-diagnostics nodeBuiltinImport:off -- Host-side disposable Git fixtures. +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import { describe, expect, it, onTestFinished } from "vite-plus/test"; + +import { + assertRecoverySelection, + checkpointFailureDisposition, + parseRecoverySelection, + recoveryPublicationArgs, + type RecoverySelection, +} from "./lastcode-checkpoint.ts"; + +const NIGHTLY_TAG = "v0.0.39-nightly.20260905.1286"; +const SOURCE_COMMIT = "b".repeat(40); + +function git(repository: string, args: ReadonlyArray): string { + return NodeChildProcess.execFileSync("git", args, { + cwd: repository, + encoding: "utf8", + }).trim(); +} + +function fixture(): { + readonly repository: string; + readonly selection: RecoverySelection; +} { + const repository = NodeFS.mkdtempSync( + NodePath.join(NodeOS.tmpdir(), "lastcode-checkpoint-recovery-"), + ); + onTestFinished(() => NodeFS.rmSync(repository, { force: true, recursive: true })); + git(repository, ["init", "--initial-branch=main"]); + git(repository, ["config", "user.email", "checkpoint@example.com"]); + git(repository, ["config", "user.name", "Checkpoint Test"]); + NodeFS.writeFileSync(NodePath.join(repository, "fixture.txt"), "upstream\n"); + git(repository, ["add", "fixture.txt"]); + git(repository, ["commit", "-m", "upstream nightly"]); + git(repository, ["tag", NIGHTLY_TAG]); + git(repository, ["checkout", "-b", `sync/nightly/${NIGHTLY_TAG}`]); + NodeFS.appendFileSync(NodePath.join(repository, "fixture.txt"), "repaired\n"); + git(repository, ["commit", "-am", "repair nightly rebase"]); + + return { + repository, + selection: { + head: git(repository, ["rev-parse", "HEAD"]), + sourceCommit: SOURCE_COMMIT, + nightlyTag: NIGHTLY_TAG, + }, + }; +} + +function publicationFixture(): { + readonly remote: string; + readonly repository: string; + readonly selection: RecoverySelection; + readonly tag: string; +} { + const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "lastcode-recovery-publication-")); + onTestFinished(() => NodeFS.rmSync(root, { force: true, recursive: true })); + const remote = NodePath.join(root, "remote.git"); + const repository = NodePath.join(root, "repository"); + NodeFS.mkdirSync(repository); + git(root, ["init", "--bare", remote]); + git(repository, ["init", "--initial-branch=source"]); + git(repository, ["config", "user.email", "checkpoint@example.com"]); + git(repository, ["config", "user.name", "Checkpoint Test"]); + git(repository, ["remote", "add", "origin", remote]); + NodeFS.writeFileSync(NodePath.join(repository, "fixture.txt"), "source\n"); + git(repository, ["add", "fixture.txt"]); + git(repository, ["commit", "-m", "source main"]); + const sourceCommit = git(repository, ["rev-parse", "HEAD"]); + git(repository, ["push", "origin", "HEAD:refs/heads/lastcode/main"]); + git(repository, ["checkout", "-b", `sync/nightly/${NIGHTLY_TAG}`]); + NodeFS.appendFileSync(NodePath.join(repository, "fixture.txt"), "repaired\n"); + git(repository, ["commit", "-am", "repaired checkpoint"]); + const head = git(repository, ["rev-parse", "HEAD"]); + const tag = `lastcode/checkpoint/${NIGHTLY_TAG}`; + git(repository, ["tag", tag, head]); + return { + remote, + repository, + selection: { head, sourceCommit, nightlyTag: NIGHTLY_TAG }, + tag, + }; +} + +describe("checkpoint recovery selection", () => { + it("atomically publishes the repaired tag and promotes its exact head", () => { + const { remote, repository, selection, tag } = publicationFixture(); + const result = NodeChildProcess.spawnSync( + "git", + recoveryPublicationArgs("origin", tag, selection), + { cwd: repository, encoding: "utf8" }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(git(remote, ["rev-parse", "refs/heads/lastcode/main"])).toBe(selection.head); + expect(git(remote, ["rev-parse", `${tag}^{commit}`])).toBe(selection.head); + }); + + it("publishes neither ref when main advanced beyond the selected source", () => { + const { remote, repository, selection, tag } = publicationFixture(); + git(repository, ["checkout", "source"]); + NodeFS.writeFileSync(NodePath.join(repository, "advanced.txt"), "new main work\n"); + git(repository, ["add", "advanced.txt"]); + git(repository, ["commit", "-m", "advance main"]); + const advanced = git(repository, ["rev-parse", "HEAD"]); + git(repository, ["push", "origin", "HEAD:refs/heads/lastcode/main"]); + + const result = NodeChildProcess.spawnSync( + "git", + recoveryPublicationArgs("origin", tag, selection), + { cwd: repository, encoding: "utf8" }, + ); + const remoteTag = NodeChildProcess.spawnSync( + "git", + ["--git-dir", remote, "rev-parse", "--verify", `refs/tags/${tag}`], + { encoding: "utf8" }, + ); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("stale info"); + expect(git(remote, ["rev-parse", "refs/heads/lastcode/main"])).toBe(advanced); + expect(remoteTag.status).not.toBe(0); + }); + + it("retains committed repairs when publication fails after deleting its local tag", () => { + expect(checkpointFailureDisposition("pending-tag", "recovery", true, true)).toEqual({ + cleanup: false, + recoveryBranch: "recovery", + }); + }); + it("parses exact full commits and a nightly tag", () => { + const selection = { + head: "a".repeat(40), + sourceCommit: SOURCE_COMMIT, + nightlyTag: NIGHTLY_TAG, + }; + expect(parseRecoverySelection(selection)).toEqual(selection); + }); + + it("rejects malformed selections and abbreviated commits", () => { + expect(() => parseRecoverySelection(null)).toThrow("Invalid recovery selection"); + expect(() => + parseRecoverySelection({ + head: "abc1234", + sourceCommit: SOURCE_COMMIT, + nightlyTag: NIGHTLY_TAG, + }), + ).toThrow("full commits"); + expect(() => + parseRecoverySelection({ + head: "a".repeat(40), + sourceCommit: "deadbeef", + nightlyTag: NIGHTLY_TAG, + }), + ).toThrow("full commits"); + expect(() => + parseRecoverySelection({ + head: "a".repeat(40), + sourceCommit: SOURCE_COMMIT, + nightlyTag: "nightly-latest", + }), + ).toThrow("exact nightly tag"); + }); + + it("accepts a clean committed repaired branch containing the selected nightly", () => { + const { repository, selection } = fixture(); + expect(() => assertRecoverySelection(repository, selection, SOURCE_COMMIT)).not.toThrow(); + }); + + it("rejects tracked and untracked recovery changes", () => { + const tracked = fixture(); + NodeFS.appendFileSync(NodePath.join(tracked.repository, "fixture.txt"), "dirty\n"); + expect(() => + assertRecoverySelection(tracked.repository, tracked.selection, SOURCE_COMMIT), + ).toThrow("must be clean"); + + const untracked = fixture(); + NodeFS.writeFileSync(NodePath.join(untracked.repository, "untracked.txt"), "not committed\n"); + expect(() => + assertRecoverySelection(untracked.repository, untracked.selection, SOURCE_COMMIT), + ).toThrow("must be clean"); + }); + + it("rejects a different recovery head, branch, or source commit", () => { + const wrongHead = fixture(); + expect(() => + assertRecoverySelection( + wrongHead.repository, + { ...wrongHead.selection, head: "c".repeat(40) }, + SOURCE_COMMIT, + ), + ).toThrow("head or branch changed"); + + const wrongBranch = fixture(); + git(wrongBranch.repository, ["branch", "--move", "sync/nightly/unselected"]); + expect(() => + assertRecoverySelection(wrongBranch.repository, wrongBranch.selection, SOURCE_COMMIT), + ).toThrow("head or branch changed"); + + const wrongSource = fixture(); + expect(() => + assertRecoverySelection(wrongSource.repository, wrongSource.selection, "d".repeat(40)), + ).toThrow("Recovery source changed"); + }); + + it("rejects a missing or uncontained upstream nightly", () => { + const { repository, selection } = fixture(); + const missingNightly = "v0.0.39-nightly.20260905.1287"; + git(repository, ["branch", "--move", `sync/nightly/${missingNightly}`]); + expect(() => + assertRecoverySelection( + repository, + { ...selection, nightlyTag: missingNightly }, + SOURCE_COMMIT, + ), + ).toThrow("does not contain the selected upstream nightly"); + }); + + it("rejects a recovery while rebase state remains", () => { + const { repository, selection } = fixture(); + NodeFS.mkdirSync(NodePath.join(repository, ".git", "rebase-merge")); + expect(() => assertRecoverySelection(repository, selection, SOURCE_COMMIT)).toThrow( + "rebase completed", + ); + }); +}); diff --git a/scripts/lastcode-checkpoint.ts b/scripts/lastcode-checkpoint.ts index b45b67f660a7..70dc0e2c2026 100644 --- a/scripts/lastcode-checkpoint.ts +++ b/scripts/lastcode-checkpoint.ts @@ -8,6 +8,7 @@ import * as NodePath from "node:path"; import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import * as Effect from "effect/Effect"; +import { acquirePortableLock } from "./lastcode-lock.mjs"; import { appendCheckpointRun, @@ -51,6 +52,8 @@ interface CheckpointOptions { readonly supersedeFailedRecovery: boolean; readonly upstreamRemote: string; readonly pushRemote: string; + readonly selectRecovery?: string; + readonly recoverySource?: string; } interface CheckpointRef { @@ -767,8 +770,9 @@ export function checkpointFailureDisposition( pendingCheckpointTag: string | undefined, recoveryBranch: string, tagDeleted = true, + preserveRecovery = false, ): { readonly cleanup: boolean; readonly recoveryBranch?: string } { - return pendingCheckpointTag && tagDeleted + return pendingCheckpointTag && tagDeleted && !preserveRecovery ? { cleanup: true } : { cleanup: false, recoveryBranch }; } @@ -805,6 +809,8 @@ function parseArgs(argv: ReadonlyArray): CheckpointOptions { let supersedeFailedRecovery = false; let upstreamRemote = DEFAULT_UPSTREAM_REMOTE; let pushRemote = DEFAULT_PUSH_REMOTE; + let selectRecovery: string | undefined; + let recoverySource: string | undefined; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; @@ -817,18 +823,29 @@ function parseArgs(argv: ReadonlyArray): CheckpointOptions { else if (arg === "--supersede-failed-recovery") supersedeFailedRecovery = true; else if (arg === "--promote") promotion = "always"; else if (arg === "--promote-if-no-open-prs") promotion = "if-no-open-prs"; - else if (arg === "--source-ref" || arg === "--upstream-remote" || arg === "--push-remote") { + else if ( + arg === "--source-ref" || + arg === "--upstream-remote" || + arg === "--push-remote" || + arg === "--select-recovery" || + arg === "--recovery-source" + ) { const value = argv[index + 1]; if (!value) throw new Error(`Missing value for ${arg}.`); if (arg === "--source-ref") sourceRef = value; else if (arg === "--upstream-remote") upstreamRemote = value; - else pushRemote = value; + else if (arg === "--push-remote") pushRemote = value; + else if (arg === "--select-recovery") selectRecovery = value; + else recoverySource = value; index += 1; } else { throw new Error(`Unknown argument '${arg}'.`); } } + if (Boolean(selectRecovery) !== Boolean(recoverySource)) { + throw new Error("--select-recovery and --recovery-source must be supplied together."); + } return { dryRun, fetch, @@ -840,6 +857,8 @@ function parseArgs(argv: ReadonlyArray): CheckpointOptions { supersedeFailedRecovery, upstreamRemote, pushRemote, + ...(selectRecovery ? { selectRecovery } : {}), + ...(recoverySource ? { recoverySource } : {}), }; } @@ -1295,10 +1314,144 @@ export function runCarrySetShadowAfterPublication( } } +export interface RecoverySelection { + readonly head: string; + readonly sourceCommit: string; + readonly nightlyTag: string; +} + +export function recoveryPublicationArgs( + remote: string, + tag: string, + selection: RecoverySelection, +): ReadonlyArray { + return [ + "push", + "--no-verify", + "--atomic", + `--force-with-lease=refs/heads/lastcode/main:${selection.sourceCommit}`, + remote, + tag, + `${selection.head}:refs/heads/lastcode/main`, + ]; +} + +function releasePublishedRecovery( + repoRoot: string, + worktree: string, + selectionPath: string, + selection: RecoverySelection, +): void { + if (NodeFS.existsSync(worktree)) { + assertRecoverySelection(worktree, selection, selection.sourceCommit); + run(repoRoot, "git", ["worktree", "remove", worktree]); + } + const branchRef = `refs/heads/sync/nightly/${selection.nightlyTag}`; + const branchHead = git(repoRoot, ["rev-parse", "--verify", branchRef], { allowFailure: true }); + if (branchHead) git(repoRoot, ["update-ref", "-d", branchRef, selection.head]); + NodeFS.unlinkSync(selectionPath); +} + +export function parseRecoverySelection(value: unknown): RecoverySelection { + if (value === null || typeof value !== "object") throw new Error("Invalid recovery selection."); + const input = value as Record; + if ( + typeof input.head !== "string" || + !/^[a-f0-9]{40}$/.test(input.head) || + typeof input.sourceCommit !== "string" || + !/^[a-f0-9]{40}$/.test(input.sourceCommit) || + typeof input.nightlyTag !== "string" || + !parseNightlyTag(input.nightlyTag) + ) { + throw new Error("Recovery selection requires full commits and an exact nightly tag."); + } + return { head: input.head, sourceCommit: input.sourceCommit, nightlyTag: input.nightlyTag }; +} + +export function assertRecoverySelection( + worktree: string, + selection: RecoverySelection, + sourceCommit: string, +): void { + if (selection.sourceCommit !== sourceCommit) + throw new Error("Recovery source changed; incorporate new main commits and select again."); + if ( + git(worktree, ["rev-parse", "HEAD"]) !== selection.head || + git(worktree, ["branch", "--show-current"]) !== `sync/nightly/${selection.nightlyTag}` + ) { + throw new Error("Retained recovery head or branch changed; select again."); + } + if ( + rebaseInProgress(worktree) || + git(worktree, ["status", "--porcelain", "--untracked-files=all"]) + ) { + throw new Error("Recovery must be clean with its rebase completed and repairs committed."); + } + if (!isAncestor(worktree, selection.nightlyTag, selection.head)) { + throw new Error("Recovery does not contain the selected upstream nightly."); + } +} + function main(argv: ReadonlyArray): void { const options = parseArgs(argv); - const hostPlatform = Effect.runSync(HostProcessPlatform); const repoRoot = git(process.cwd(), ["rev-parse", "--show-toplevel"]); + const commonDirectory = git(repoRoot, [ + "rev-parse", + "--path-format=absolute", + "--git-common-dir", + ]); + const release = acquirePortableLock( + commonDirectory, + "lastcode-checkpoint", + "checkpoint operation", + ); + try { + runCheckpoint( + repoRoot, + options, + NodePath.join(commonDirectory, "lastcode-recovery-selection.json"), + ); + } finally { + release(); + } +} + +function runCheckpoint(repoRoot: string, options: CheckpointOptions, selectionPath: string): void { + const hostPlatform = Effect.runSync(HostProcessPlatform); + if (options.selectRecovery && options.recoverySource) { + const worktree = resolveAutomationWorktree(repoRoot); + const branch = git(worktree, ["branch", "--show-current"]); + const selection = parseRecoverySelection({ + head: options.selectRecovery, + sourceCommit: options.recoverySource, + nightlyTag: branch.replace(/^sync\/nightly\//, ""), + }); + // Selection is an explicit assertion that the repaired tree includes this source. + run(repoRoot, "git", ["fetch", options.pushRemote, "lastcode/main"]); + assertRecoverySelection( + worktree, + selection, + git(repoRoot, ["rev-parse", `${options.sourceRef}^{commit}`]), + ); + if (!options.dryRun) { + const temporaryPath = `${selectionPath}.${process.pid}.tmp`; + NodeFS.writeFileSync(temporaryPath, `${JSON.stringify(selection)}\n`, { + mode: 0o600, + flush: true, + }); + NodeFS.renameSync(temporaryPath, selectionPath); + } + console.log( + `[lastcode:checkpoint] ${options.dryRun ? "Would select" : "Selected"} repaired ${selection.nightlyTag} at ${selection.head}. Request a service run, then use Wait for Checkpoint.`, + ); + return; + } + const selection = NodeFS.existsSync(selectionPath) + ? parseRecoverySelection(JSON.parse(NodeFS.readFileSync(selectionPath, "utf8"))) + : undefined; + if (selection && (!options.smoke || !options.pushTags || options.promotion === "never")) { + throw new Error("Selected recovery requires smoke validation, --push-tags, and promotion."); + } git(repoRoot, ["config", "rerere.enabled", "true"]); git(repoRoot, ["config", "rerere.autoupdate", "true"]); @@ -1342,6 +1495,35 @@ function main(argv: ReadonlyArray): void { const sourceCommit = git(repoRoot, ["rev-parse", `${options.sourceRef}^{commit}`]); const checkpoints = listCheckpointRefs(repoRoot); + // A crash after pushing but before clearing the selection must not republish or + // rebase the repaired commit. The published immutable tag now preserves it. + if ( + selection && + checkpoints.some( + (checkpoint) => + checkpoint.checkpointTag === checkpointTagFromNightlyTag(selection.nightlyTag) && + checkpoint.commit === selection.head && + checkpoint.sourceCommit === selection.sourceCommit, + ) + ) { + if (!isAncestor(repoRoot, selection.head, sourceCommit)) + throw new Error( + "Published recovery is not represented on main; inspect before releasing it.", + ); + if (!options.dryRun) + releasePublishedRecovery( + repoRoot, + resolveAutomationWorktree(repoRoot), + selectionPath, + selection, + ); + console.log( + "[lastcode:checkpoint] Selected recovery was already published; released its retained worktree. Run the service again to continue.", + ); + return; + } + if (selection) + assertRecoverySelection(resolveAutomationWorktree(repoRoot), selection, sourceCommit); const installables = listInstallableRefs(repoRoot); const sourceAncestor = latestCheckpointAncestor(repoRoot, checkpoints, options.sourceRef); const sourceNightlyTags = splitLines( @@ -1353,7 +1535,7 @@ function main(argv: ReadonlyArray): void { nightlyTags, recoverySupersessionMode({ dryRun: options.dryRun, - enabled: options.supersedeFailedRecovery, + enabled: options.supersedeFailedRecovery && !selection, }), ); const plan = resolveCheckpointPlan({ @@ -1365,6 +1547,14 @@ function main(argv: ReadonlyArray): void { sourceRef: options.sourceRef, ...(supersededNightly ? { supersedeThroughNightlyTag: supersededNightly.tag } : {}), }); + if ( + selection && + (plan.bootstrapCheckpoint || plan.missingNightlies[0]?.tag !== selection.nightlyTag) + ) { + throw new Error( + "Selected recovery is not the next unpublished checkpoint; inspect before selecting again.", + ); + } console.log(`[lastcode:checkpoint] Source: ${plan.candidateRef}`); console.log(`[lastcode:checkpoint] Upstream base: ${plan.baseNightly.tag}`); @@ -1489,7 +1679,7 @@ function main(argv: ReadonlyArray): void { } const worktree = resolveAutomationWorktree(repoRoot); - if (NodeFS.existsSync(worktree)) { + if (NodeFS.existsSync(worktree) && !selection) { throw new Error( `Nightly sync worktree already exists at ${worktree}. Resolve or remove it first.`, ); @@ -1498,11 +1688,14 @@ function main(argv: ReadonlyArray): void { const firstNightly = plan.missingNightlies[0]; if (!firstNightly) throw new Error("Missing first nightly checkpoint."); let branch = `sync/nightly/${firstNightly.tag}`; - if (git(repoRoot, ["show-ref", "--verify", `refs/heads/${branch}`], { allowFailure: true })) { + if ( + !selection && + git(repoRoot, ["show-ref", "--verify", `refs/heads/${branch}`], { allowFailure: true }) + ) { throw new Error(`Recovery branch ${branch} already exists.`); } - run(repoRoot, "git", worktreeAddArgs(branch, worktree, candidateRef)); + if (!selection) run(repoRoot, "git", worktreeAddArgs(branch, worktree, candidateRef)); let completed = false; let pendingCheckpointTag: string | undefined; let attempt: @@ -1515,7 +1708,7 @@ function main(argv: ReadonlyArray): void { let failurePhase: "publication" | "rebase" | "smoke" | undefined; try { let baseTag = plan.baseNightly.tag; - for (const nightly of plan.missingNightlies) { + for (const nightly of selection ? plan.missingNightlies.slice(0, 1) : plan.missingNightlies) { const recoveryBranch = `sync/nightly/${nightly.tag}`; if (branch !== recoveryBranch) { run(worktree, "git", ["branch", "--move", recoveryBranch]); @@ -1530,10 +1723,21 @@ function main(argv: ReadonlyArray): void { }; console.log(`[lastcode:checkpoint] Rebasing LastCode from ${baseTag} onto ${nightly.tag}...`); failurePhase = "rebase"; - rebaseOnto(worktree, nightly.tag, baseTag); + if (selection && nightly.tag === selection.nightlyTag) { + assertRecoverySelection(worktree, selection, sourceCommit); + console.log(`[lastcode:checkpoint] Validating selected repaired head ${selection.head}.`); + } else { + rebaseOnto(worktree, nightly.tag, baseTag); + } candidateCommit = git(repoRoot, ["rev-parse", "HEAD"], { cwd: worktree }); failurePhase = "smoke"; if (options.smoke) runSmokeGate(repoRoot, worktree); + if ( + git(worktree, ["rev-parse", "HEAD"]) !== candidateCommit || + git(worktree, ["status", "--porcelain", "--untracked-files=all"]) + ) { + throw new Error("Checkpoint changed during validation; retain and inspect the worktree."); + } failurePhase = "publication"; const finishedAtMs = Date.now(); const timing = { @@ -1552,21 +1756,34 @@ function main(argv: ReadonlyArray): void { ); pendingCheckpointTag = checkpointTag; if (options.pushTags) { - run( - repoRoot, - "git", - checkpointTagPushArgs( - options.pushRemote, - checkpointTag, - options.smoke - ? { kind: "smoke" } - : { - kind: "pre-push", - candidateCommit, - checkoutHead: git(repoRoot, ["rev-parse", "HEAD"]), - }, - ), - ); + if (selection) { + if (options.promotion === "if-no-open-prs" && openPullRequestCount(repoRoot) > 0) { + throw new Error( + "Open LastCode PRs prevent repaired checkpoint publication; retained for retry.", + ); + } + run( + repoRoot, + "git", + recoveryPublicationArgs(options.pushRemote, checkpointTag, selection), + ); + } else { + run( + repoRoot, + "git", + checkpointTagPushArgs( + options.pushRemote, + checkpointTag, + options.smoke + ? { kind: "smoke" } + : { + kind: "pre-push", + candidateCommit, + checkoutHead: git(repoRoot, ["rev-parse", "HEAD"]), + }, + ), + ); + } } pendingCheckpointTag = undefined; appendCheckpointRun({ @@ -1589,7 +1806,12 @@ function main(argv: ReadonlyArray): void { const tagDeleted = pendingCheckpointTag ? deleteCheckpointTag(repoRoot, pendingCheckpointTag) : true; - const disposition = checkpointFailureDisposition(pendingCheckpointTag, branch, tagDeleted); + const disposition = checkpointFailureDisposition( + pendingCheckpointTag, + branch, + tagDeleted, + selection !== undefined, + ); completed = disposition.cleanup; if (attempt) { const finishedAtMs = Date.now(); @@ -1638,11 +1860,23 @@ function main(argv: ReadonlyArray): void { throw error; } finally { if (completed) { - run(repoRoot, "git", ["worktree", "remove", worktree]); - git(repoRoot, ["update-ref", "-d", `refs/heads/${branch}`]); + if (selection) { + releasePublishedRecovery(repoRoot, worktree, selectionPath, selection); + } else { + run(repoRoot, "git", ["worktree", "remove", worktree]); + git(repoRoot, ["update-ref", "-d", `refs/heads/${branch}`]); + } } } + if (selection) { + runCarrySetShadowAfterPublication(repoRoot, newestProducedInstallableTag); + console.log( + "[lastcode:checkpoint] Repaired checkpoint published and promoted. Run the service again for later nightlies.", + ); + return; + } + runPromotionThenShadow( () => promoteCheckpoint(