diff --git a/.github/scripts/__tests__/helpers/keepalive-presence-server.js b/.github/scripts/__tests__/helpers/keepalive-presence-server.js new file mode 100644 index 000000000..1573cfcd0 --- /dev/null +++ b/.github/scripts/__tests__/helpers/keepalive-presence-server.js @@ -0,0 +1,92 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); + +// Keep immutable snapshots: a cache publication changes the branch commit, while +// only an index writer changes the attempt subtree. No process-local helper state. +function presenceServer({ count = 1001, missing = null } = {}) { + const sha = (value) => value.toString(16).padStart(40, '0'); + const inventories = new Map(); + const commits = new Map(); + const trees = new Map(); + const blobs = new Map(); + const calls = []; + let hook = () => {}; + let version = 0; + let indexVersion = 0; + const entries = []; + function addIndex(prNumber, runId) { + const ownerAttempt = `owner/repo:${runId}:1`; + const index = { version: 1, repository: 'owner/repo', owner_attempt: ownerAttempt, + pr_number: prNumber, generation: 'a'.repeat(64), + receipt: { id: 'b'.repeat(64), claim_digest: 'c'.repeat(64), owner_attempt: ownerAttempt, + head_sha: 'd'.repeat(40), provider: 'codex', consumed_at: '2026-10-01T00:00:00.000Z' } }; + const blobSha = sha(500000 + runId); + blobs.set(blobSha, { sha: blobSha, encoding: 'base64', + content: Buffer.from(JSON.stringify(index)).toString('base64') }); + entries.push({ path: crypto.createHash('sha256').update(ownerAttempt).digest('hex') + '.json', + type: 'blob', sha: blobSha }); + indexVersion += 1; + } + for (let i = 1; i <= count; i += 1) addIndex(9000, i); + function publish() { + version += 1; + const indexTree = sha(400000 + indexVersion); + trees.set(indexTree, { truncated: false, tree: entries.map((entry) => ({ ...entry })) }); + const githubTree = sha(300000 + version); + trees.set(githubTree, { truncated: false, tree: missing === 'attempts' ? [] : + [{ path: 'keepalive-authority-attempts', type: 'tree', sha: indexTree }] }); + const rootTree = sha(200000 + version); + trees.set(rootTree, { truncated: false, tree: missing === 'github' ? [] : + [{ path: '.github', type: 'tree', sha: githubTree }] }); + commits.set(sha(100000 + version), { tree: { sha: rootTree }, inventories: new Map(inventories) }); + } + publish(); + const request = async (method, path, body) => { + calls.push({ method, path }); + let response; + if (path.includes('/contents/.github/keepalive-authority-presence/')) { + const key = path.split('keepalive-authority-presence/')[1].split('?')[0]; + if (method === 'PUT') { + assert.equal(body.sha, undefined, 'inventory must be create-only'); + if (inventories.has(key)) throw Object.assign(new Error('existing'), { status: 422 }); + inventories.set(key, body.content); + publish(); + response = {}; + } else { + assert.equal(method, 'GET'); + const ref = path.split('?ref=')[1]; + const snapshot = ref === 'keepalive-authority-state' ? inventories : commits.get(ref)?.inventories; + if (!snapshot?.has(key)) throw Object.assign(new Error('missing'), { status: 404 }); + response = { sha: sha(600000), encoding: 'base64', content: snapshot.get(key) }; + } + } else { + assert.equal(method, 'GET'); + const key = path.split('/').pop(); + if (path.includes('/git/ref/')) response = { object: { type: 'commit', sha: sha(100000 + version) } }; + else if (path.includes('/git/commits/')) response = { tree: commits.get(key)?.tree }; + else if (path.includes('/git/trees/')) response = trees.get(key); + else if (path.includes('/git/blobs/')) response = blobs.get(key); + assert.ok(response, `Unexpected request ${method} ${path}`); + } + await hook({ method, path, body }); + return response; + }; + return { + request, + calls, + setHook(value) { hook = value; }, + addAttempt(prNumber = 44) { + missing = null; + addIndex(prNumber, 9999); + publish(); + }, + stats() { + return { blobs: calls.filter((call) => call.path.includes('/git/blobs/')).length, + writes: calls.filter((call) => call.method === 'PUT').length, calls: calls.length }; + }, + }; +} + +module.exports = { presenceServer }; diff --git a/.github/scripts/__tests__/keepalive-attempt-presence.test.js b/.github/scripts/__tests__/keepalive-attempt-presence.test.js new file mode 100644 index 000000000..945cdd89c --- /dev/null +++ b/.github/scripts/__tests__/keepalive-attempt-presence.test.js @@ -0,0 +1,156 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); + +function fixture({ count = 1, target = 42, truncate = false, mutate = () => {}, + missingDirectory = false, unreadable = false, wrongBlob = false, badBase64 = false, + malformedTree = false, missingBranch = false, putStatus = 0, corruptPresence = false } = {}) { + const calls = []; + const hashes = ['1', '2', '3', '4'].map((x) => x.repeat(40)); + const blobs = new Map(); + const inventories = new Map(); + const entries = Array.from({ length: count }, (_, i) => { + const owner = `owner/repo:${i + 1}:1`; + const name = crypto.createHash('sha256').update(owner).digest('hex') + '.json'; + const sha = (i + 10).toString(16).padStart(40, '0'); + const index = { version: 1, repository: 'owner/repo', owner_attempt: owner, + pr_number: i === count - 1 ? target : 43, generation: 'a'.repeat(64), + receipt: { id: 'b'.repeat(64), claim_digest: 'c'.repeat(64), owner_attempt: owner, + head_sha: 'd'.repeat(40), provider: 'codex', consumed_at: '2026-10-01T00:00:00.000Z' } }; + mutate(index); + blobs.set(sha, { sha: wrongBlob ? 'f'.repeat(40) : sha, encoding: 'base64', + content: Buffer.from(JSON.stringify(index)).toString('base64') + (badBase64 ? '!' : '') }); + return { path: name, type: 'blob', sha }; + }); + const request = async (method, path, body) => { + calls.push(path); + if (path.includes('/contents/.github/keepalive-authority-presence/')) { + const key = path.split('keepalive-authority-presence/')[1].split('?')[0]; + if (method === 'PUT') { + if (putStatus) { + const attempted = JSON.parse(Buffer.from(body.content, 'base64').toString('utf8')); + if (putStatus === 409) inventories.set(key, { ...attempted, positive_prs: [999] }); + throw Object.assign(new Error('write failed'), { status: putStatus }); + } + inventories.set(key, JSON.parse(Buffer.from(body.content, 'base64').toString('utf8'))); + return {}; + } + assert.equal(method, 'GET'); + if (!inventories.has(key)) throw Object.assign(new Error('missing inventory'), { status: 404 }); + return { sha: 'e'.repeat(40), encoding: 'base64', + content: corruptPresence ? 'not base64!' : Buffer.from(JSON.stringify(inventories.get(key))).toString('base64') }; + } + assert.equal(method, 'GET'); + if (path.endsWith('/git/ref/heads/keepalive-authority-state')) { + if (missingBranch) throw Object.assign(new Error('unavailable branch'), { status: 404 }); + return { object: { type: 'commit', sha: hashes[0] } }; + } + if (path.endsWith(`/git/commits/${hashes[0]}`)) return { tree: { sha: hashes[1] } }; + if (path.endsWith(`/git/trees/${hashes[1]}`)) return { truncated: false, + tree: [{ path: '.github', type: 'tree', sha: hashes[2] }] }; + if (path.endsWith(`/git/trees/${hashes[2]}`)) return { truncated: false, + tree: missingDirectory ? [] : [{ path: 'keepalive-authority-attempts', type: 'tree', sha: hashes[3] }] }; + if (path.endsWith(`/git/trees/${hashes[3]}`)) return { truncated: truncate, + tree: malformedTree ? [...entries, { path: 'bad', type: 'blob', sha: 'invalid' }] : entries }; + const sha = path.split('/git/blobs/')[1]; + if (blobs.has(sha)) { + if (unreadable) throw Object.assign(new Error('unavailable blob'), { status: 404 }); + return blobs.get(sha); + } + throw new Error(`Unexpected request ${path}`); + }; + return { request, calls }; +} + +for (const surface of ['../keepalive_authority_state.js', + '../../../templates/consumer-repo/.github/scripts/keepalive_authority_state.js']) { + const { hasAttemptIndexesForPr } = require(surface); + test(`${surface}: finds an index beyond the Contents API directory limit`, async () => { + const { request, calls } = fixture({ count: 1001 }); + assert.equal(await hasAttemptIndexesForPr(request, 'owner/repo', 42), true); + const blobReads = calls.filter((p) => p.includes('/git/blobs/')).length; + assert.equal(await hasAttemptIndexesForPr(request, 'owner/repo', 42), true); + assert.equal(await hasAttemptIndexesForPr(request, 'owner/repo', 43), true); + assert.equal(await hasAttemptIndexesForPr(request, 'owner/repo', 44), false); + assert.equal(calls.filter((p) => p.includes('/git/blobs/')).length, blobReads); + assert.ok(calls.filter((p) => p.includes('/git/ref/')).length >= 1); + assert.ok(calls.every((p) => !p.includes('/contents/.github/keepalive-authority-attempts/'))); + }); + test(`${surface}: proves absence only from complete validated evidence`, async () => { + for (const options of [{ target: 43 }, { count: 0 }, { missingDirectory: true }]) { + assert.equal(await hasAttemptIndexesForPr(fixture(options).request, 'owner/repo', 42), false); + } + }); + test(`${surface}: rejects incomplete, unreadable, or malformed evidence`, async () => { + for (const options of [ + { truncate: true }, { malformedTree: true }, { unreadable: true }, { wrongBlob: true }, + { badBase64: true }, { missingBranch: true }, + { mutate: (index) => { index.repository = 'other/repo'; } }, + { mutate: (index) => { index.pr_number = '42'; } }, + { mutate: (index) => { index.receipt = null; } }, + { mutate: (index) => { index.owner_attempt = 'owner/repo:999:1'; index.receipt.owner_attempt = index.owner_attempt; } }, + ]) { + await assert.rejects(hasAttemptIndexesForPr(fixture(options).request, 'owner/repo', 42)); + } + }); + test(`${surface}: rejects lost, conflicting, or malformed inventory publication`, async () => { + for (const options of [{ putStatus: 500 }, { putStatus: 409 }, { corruptPresence: true }]) { + await assert.rejects(hasAttemptIndexesForPr(fixture(options).request, 'owner/repo', 42)); + } + }); +} + + +// A legacy writer never touches presence records. Its new attempt still changes +// the immutable subtree key, so old negative inventory cannot hide the attempt. +for (const surface of ['../keepalive_authority_state.js', + '../../../templates/consumer-repo/.github/scripts/keepalive_authority_state.js']) { + function changingServer(trigger) { + const base = fixture({ target: 43 }); + let changed = false; + let armed = false; + const owner = 'owner/repo:9999:1'; + const index = { version: 1, repository: 'owner/repo', owner_attempt: owner, + pr_number: 44, generation: 'a'.repeat(64), + receipt: { id: 'b'.repeat(64), claim_digest: 'c'.repeat(64), owner_attempt: owner, + head_sha: 'd'.repeat(40), provider: 'codex', consumed_at: '2026-10-01T00:00:00.000Z' } }; + const request = async (method, url, body) => { + if (url.endsWith('/git/trees/' + '5'.repeat(40))) { + return { truncated: false, tree: [{ path: crypto.createHash('sha256').update(owner).digest('hex') + '.json', + type: 'blob', sha: '6'.repeat(40) }] }; + } + if (url.endsWith('/git/blobs/' + '6'.repeat(40))) { + return { sha: '6'.repeat(40), encoding: 'base64', content: Buffer.from(JSON.stringify(index)).toString('base64') }; + } + const response = await base.request(method, url, body); + if (url.endsWith('/git/trees/' + '3'.repeat(40)) && changed) { + return { truncated: false, tree: [{ path: 'keepalive-authority-attempts', type: 'tree', sha: '5'.repeat(40) }] }; + } + if (armed && ((trigger === 'warm' && method === 'GET' && url.includes('/keepalive-authority-presence/')) || + (trigger === 'backfill' && url.includes('/git/blobs/')) || + (trigger === 'publication' && method === 'PUT'))) changed = true; + return response; + }; + return { request, arm: () => { armed = true; }, advance: () => { changed = true; } }; + } + test(`${surface}: an older writer invalidates negative presence by changing the subtree`, async () => { + let { hasAttemptIndexesForPr } = require(surface); + const server = changingServer(); + assert.equal(await hasAttemptIndexesForPr(server.request, 'owner/repo', 44), false); + server.advance(); + delete require.cache[require.resolve(surface)]; + ({ hasAttemptIndexesForPr } = require(surface)); + assert.equal(await hasAttemptIndexesForPr(server.request, 'owner/repo', 44), true); + }); + test(`${surface}: concurrent changes fail closed during warm read, backfill and publication`, async () => { + const { hasAttemptIndexesForPr } = require(surface); + for (const phase of ['warm', 'backfill', 'publication']) { + const server = changingServer(phase); + if (phase === 'warm') assert.equal(await hasAttemptIndexesForPr(server.request, 'owner/repo', 44), false); + server.arm(); + await assert.rejects(hasAttemptIndexesForPr(server.request, 'owner/repo', 44), /changed during/); + } + }); +} diff --git a/.github/scripts/__tests__/keepalive-authority-state.test.js b/.github/scripts/__tests__/keepalive-authority-state.test.js index a8a4e3f0d..5d7b47e9d 100644 --- a/.github/scripts/__tests__/keepalive-authority-state.test.js +++ b/.github/scripts/__tests__/keepalive-authority-state.test.js @@ -22,6 +22,103 @@ const prNumber = 42; const fingerprint = 'a'.repeat(64); const headSha = 'd'.repeat(40); const ownerAttempt = 'owner/repo:100:1'; +const { presenceServer } = require('./helpers/keepalive-presence-server.js'); + +for (const directory of ['..', '../../../templates/consumer-repo/.github/scripts']) { + function freshHelper(filename) { + const authority = require.resolve(`${directory}/keepalive_authority_state.js`); + delete require.cache[authority]; + const target = require.resolve(`${directory}/${filename}`); + delete require.cache[target]; + return require(target); + } + + test(`${directory}: separate default reporters reuse positive and negative inventory`, async () => { + const server = presenceServer(); + const replay = async (number) => { + const { replayReporterAuthority } = freshHelper('keepalive_reporter_applicability.js'); + // Exercise both production defaults, including the real pinned ledger read. + return replayReporterAuthority({ github: {}, context: { repo: { owner: 'owner', repo: 'repo' } }, + prNumber: number, makeRequest: () => server.request }); + }; + assert.deepEqual(await replay(42), { prNumber: 42, results: [] }); + assert.equal(server.stats().blobs, 1001); + assert.equal(server.stats().writes, 1); + for (const number of [43, 44, 9000, 45, 9000]) { + const before = server.stats(); + if (number === 9000) await assert.rejects(replay(number), /ledger is missing/); + else assert.deepEqual(await replay(number), { prNumber: number, results: [] }); + assert.equal(server.stats().blobs, before.blobs); + assert.equal(server.stats().writes, before.writes); + assert.ok(server.stats().calls - before.calls <= 15, 'warm calls must be bounded'); + } + // An older writer only publishes an index; it knows nothing about inventories. + server.addAttempt(44); + await assert.rejects(replay(44), /ledger is missing/); + assert.equal(server.stats().blobs, 2003); + const settled = server.stats(); + await assert.rejects(replay(44), /ledger is missing/); + await replay(45); + assert.equal(server.stats().blobs, settled.blobs); + assert.equal(server.stats().writes, settled.writes); + }); + + test(`${directory}: separate backfill writers converge on the same complete inventory`, async () => { + const server = presenceServer({ count: 20 }); + const first = freshHelper('keepalive_authority_state.js'); + const second = freshHelper('keepalive_authority_state.js'); + assert.deepEqual(await Promise.all([ + first.hasAttemptIndexesForPr(server.request, repository, 44), + second.hasAttemptIndexesForPr(server.request, repository, 9000), + ]), [false, true]); + assert.equal(server.stats().writes, 2, 'the second create must reconcile its 422'); + const before = server.stats(); + assert.equal(await freshHelper('keepalive_authority_state.js') + .hasAttemptIndexesForPr(server.request, repository, 44), false); + assert.equal(server.stats().blobs, before.blobs); + }); + + test(`${directory}: partial backfill and landed lost response deny the current read`, async () => { + for (const phase of ['scan', 'publication']) { + const server = presenceServer({ count: 20 }); + let failed = false; + server.setHook(({ method, path }) => { + if (!failed && (phase === 'scan' ? path.includes('/git/blobs/') : method === 'PUT')) { + failed = true; + throw status(503); + } + }); + await assert.rejects(freshHelper('keepalive_authority_state.js') + .hasAttemptIndexesForPr(server.request, repository, 44), /HTTP 503/); + assert.equal(server.stats().writes, phase === 'scan' ? 0 : 1); + const before = server.stats(); + assert.equal(await freshHelper('keepalive_authority_state.js') + .hasAttemptIndexesForPr(server.request, repository, 44), false); + // A failed scan publishes nothing. A landed lost-response write is usable + // only by a later independent reader that validates the settled snapshot. + assert.equal(server.stats().blobs - before.blobs, phase === 'scan' ? 20 : 0); + } + }); + + for (const missing of ['github', 'attempts']) { + test(`${directory}: first older-writer index cannot hide behind absent ${missing}`, async () => { + const server = presenceServer({ count: 0, missing }); + const { hasAttemptIndexesForPr } = freshHelper('keepalive_authority_state.js'); + assert.equal(await hasAttemptIndexesForPr(server.request, repository, 44), false); + let advanced = false; + server.setHook(({ path }) => { + const absentTree = missing === 'github' ? 200001 : 300001; + if (!advanced && path.endsWith(absentTree.toString(16).padStart(40, '0'))) { + advanced = true; + server.addAttempt(44); + } + }); + await assert.rejects(hasAttemptIndexesForPr(server.request, repository, 44), /changed during/); + assert.equal(server.stats().writes, 0, 'uncertain absence must not be published'); + assert.equal(await hasAttemptIndexesForPr(server.request, repository, 44), true); + }); + } +} function replayTreeRequest({ missingRef = false, truncated = false, ledgerPresent = false, blobUnavailable = false, malformedBase64 = false, diff --git a/.github/scripts/__tests__/keepalive-reporter-applicability.test.js b/.github/scripts/__tests__/keepalive-reporter-applicability.test.js index 29feaee1c..6f7e9e33f 100644 --- a/.github/scripts/__tests__/keepalive-reporter-applicability.test.js +++ b/.github/scripts/__tests__/keepalive-reporter-applicability.test.js @@ -406,6 +406,11 @@ test('replay is a no-op when an ordinary PR has no authority ledger', async () = assert.equal(number, 42); return null; }, + hasAttemptIndexes: async (_request, repository, number) => { + assert.equal(repository, 'stranske/repo'); + assert.equal(number, 42); + return false; + }, makeRequest: () => 'request', }); assert.deepEqual(result, { prNumber: 42, results: [] }); @@ -421,6 +426,25 @@ test('replay still fails closed when an authority ledger read fails', async () = }), /ledger unavailable/); }); +test('replay fails closed when an authority candidate PR has a missing ledger but has attempt indexes', async () => { + await assert.rejects(replayReporterAuthority({ + github: { request: async () => { throw new Error('no run lookup expected'); } }, + context: { repo: { owner: 'stranske', repo: 'repo' } }, + prNumber: 42, + readAuthority: async (_request, repository, number) => { + assert.equal(repository, 'stranske/repo'); + assert.equal(number, 42); + return null; + }, + hasAttemptIndexes: async (_request, repository, number) => { + assert.equal(repository, 'stranske/repo'); + assert.equal(number, 42); + return true; // PR has attempt indexes, so it's an authority candidate + }, + makeRequest: () => 'request', + }), /Authority ledger is missing for a PR with attempt indexes/); +}); + test('replay rejects a present receipt with an invalid owner attempt', async () => { await assert.rejects(replayReporterAuthority({ github: {}, diff --git a/.github/scripts/keepalive_authority_state.js b/.github/scripts/keepalive_authority_state.js index ef15ecdc0..66f78d112 100644 --- a/.github/scripts/keepalive_authority_state.js +++ b/.github/scripts/keepalive_authority_state.js @@ -76,6 +76,11 @@ function attemptPath(repository, ownerAttempt) { return `/repos/${String(repository).toLowerCase()}/contents/.github/keepalive-authority-attempts/${key}.json`; } +function presencePath(repository, indexTreeSha) { + if (!HEAD.test(String(indexTreeSha))) throw new Error('Invalid authority attempt tree SHA'); + return `/repos/${String(repository).toLowerCase()}/contents/.github/keepalive-authority-presence/${indexTreeSha}.json`; +} + function validAttemptIndex(index, repository, ownerAttempt) { return index?.version === 1 && index.repository === String(repository).toLowerCase() && index.owner_attempt === ownerAttempt && Number.isSafeInteger(index.pr_number) && @@ -985,9 +990,156 @@ async function findAuthorityPrForAttempt({ request, repository, ownerAttempt }) return { prNumber: index.pr_number, state }; } +async function hasAttemptIndexesForPr(request, repository, prNumber) { + pathFor(repository, prNumber); + const snapshot = await attemptIndexTree(request, repository); + if (snapshot.treeSha === null) { + // The first index may be created by a legacy writer after this snapshot. + // Missing directories need the same freshness fence as cached negatives. + const current = await attemptIndexTree(request, snapshot.repo); + if (current.treeSha !== null) { + throw new Error('Authority attempt indexes changed during absence read'); + } + return false; + } + let inventory = await readAttemptPresence(request, snapshot.repo, snapshot.treeSha, snapshot.commitSha); + if (!inventory) { + const positives = await scanAttemptIndexes(request, snapshot); + // A writer may have added an index while this complete scan was in flight. + // Never publish an absence result for a different subtree. + const current = await attemptIndexTree(request, snapshot.repo); + if (!current || current.treeSha !== snapshot.treeSha) { + throw new Error('Authority attempt indexes changed during inventory backfill'); + } + inventory = await createAttemptPresence(request, snapshot, positives); + const settled = await attemptIndexTree(request, snapshot.repo); + if (!settled || settled.treeSha !== snapshot.treeSha) { + throw new Error('Authority attempt indexes changed during inventory publication'); + } + } + const current = await attemptIndexTree(request, snapshot.repo); + if (!current || current.treeSha !== snapshot.treeSha) { + throw new Error('Authority attempt indexes changed during presence read'); + } + return inventory.positive_prs.includes(Number(prNumber)); +} + +async function attemptIndexTree(request, repository) { + const repo = String(repository).toLowerCase(); + const ref = await request('GET', `/repos/${repo}/git/ref/heads/${BRANCH}`); + const commitSha = ref?.object?.sha; + if (ref?.object?.type !== 'commit' || !HEAD.test(String(commitSha))) { + throw new Error('Authority attempt branch did not resolve to a commit'); + } + const commit = await request('GET', `/repos/${repo}/git/commits/${commitSha}`); + let treeSha = commit?.tree?.sha; + const segments = ['.github', 'keepalive-authority-attempts']; + // Non-recursive trees avoid the Contents API's 1,000-entry directory limit. + // Every later read uses immutable SHAs from this one branch snapshot. + for (const segment of [...segments, null]) { + if (!HEAD.test(String(treeSha))) throw new Error('Invalid authority attempt tree SHA'); + const tree = await request('GET', `/repos/${repo}/git/trees/${treeSha}`); + if (tree?.truncated !== false || !Array.isArray(tree.tree) || + !tree.tree.every((entry) => entry && typeof entry.path === 'string' && + entry.path.length > 0 && !entry.path.includes('/') && + ['blob', 'tree', 'commit'].includes(entry.type) && HEAD.test(String(entry.sha))) || + new Set(tree.tree.map((entry) => entry.path)).size !== tree.tree.length) { + throw new Error('Incomplete or malformed authority attempt tree'); + } + if (segment !== null) { + const entry = tree.tree.find((item) => item.path === segment); + if (!entry) return { repo, commitSha, treeSha: null }; + if (entry.type !== 'tree') throw new Error('Authority attempt directory is not a tree'); + treeSha = entry.sha; + continue; + } + return { repo, commitSha, treeSha }; + } + throw new Error('Authority attempt tree traversal did not settle'); +} + +async function scanAttemptIndexes(request, snapshot) { + const tree = await request('GET', `/repos/${snapshot.repo}/git/trees/${snapshot.treeSha}`); + if (tree?.truncated !== false || !Array.isArray(tree.tree)) { + throw new Error('Incomplete or malformed authority attempt tree'); + } + const positives = new Set(); + for (const entry of tree.tree) { + if (!/^[0-9a-f]{64}\.json$/.test(entry.path) || entry.type !== 'blob') { + throw new Error('Invalid authority attempt index path'); + } + const blob = await request('GET', `/repos/${snapshot.repo}/git/blobs/${entry.sha}`); + if (blob?.sha !== entry.sha || blob?.encoding !== 'base64' || + typeof blob.content !== 'string') { + throw new Error('Invalid authority attempt index metadata'); + } + const encoded = blob.content.replace(/\s/g, ''); + if (!encoded || Buffer.from(encoded, 'base64').toString('base64') !== encoded) { + throw new Error('Malformed authority attempt index base64'); + } + const index = JSON.parse(Buffer.from(encoded, 'base64').toString('utf8')); + if (!validAttemptIndex(index, snapshot.repo, index?.owner_attempt) || + attemptPath(snapshot.repo, index.owner_attempt).split('/').pop() !== entry.path) { + throw new Error('Invalid authority attempt index'); + } + positives.add(index.pr_number); + } + return [...positives].sort((left, right) => left - right); +} + +function decodePresence(file, repository, treeSha) { + if (!HEAD.test(String(file?.sha)) || file?.encoding !== 'base64' || typeof file.content !== 'string') { + throw new Error('Invalid authority attempt presence metadata'); + } + const encoded = file.content.replace(/\s/g, ''); + if (!encoded || Buffer.from(encoded, 'base64').toString('base64') !== encoded) { + throw new Error('Malformed authority attempt presence base64'); + } + let inventory; + try { inventory = JSON.parse(Buffer.from(encoded, 'base64').toString('utf8')); } catch (error) { + throw new Error(`Malformed authority attempt presence: ${error.message}`); + } + if (inventory?.version !== 1 || inventory.repository !== repository || + inventory.index_tree_sha !== treeSha || !Array.isArray(inventory.positive_prs) || + !inventory.positive_prs.every((value) => Number.isSafeInteger(value) && value > 0) || + new Set(inventory.positive_prs).size !== inventory.positive_prs.length || + inventory.positive_prs.some((value, index) => index && inventory.positive_prs[index - 1] >= value)) { + throw new Error('Invalid authority attempt presence'); + } + return inventory; +} + +async function readAttemptPresence(request, repository, treeSha, ref = BRANCH) { + try { + return decodePresence(await request('GET', `${presencePath(repository, treeSha)}?ref=${ref}`), repository, treeSha); + } catch (error) { + if (error.status === 404) return null; + throw error; + } +} + +async function createAttemptPresence(request, snapshot, positives) { + const inventory = { version: 1, repository: snapshot.repo, index_tree_sha: snapshot.treeSha, + positive_prs: positives }; + try { + await request('PUT', presencePath(snapshot.repo, snapshot.treeSha), { branch: BRANCH, + message: `keepalive authority presence ${snapshot.treeSha}`, + content: Buffer.from(`${JSON.stringify(inventory)}\n`).toString('base64') }); + } catch (error) { + if (![409, 422].includes(error.status)) throw error; + } + const settled = await readAttemptPresence(request, snapshot.repo, snapshot.treeSha, BRANCH); + if (!settled) throw new Error('Authority attempt presence write was not confirmed'); + if (JSON.stringify(settled.positive_prs) !== JSON.stringify(positives)) { + throw new Error('Authority attempt presence conflicts with validated inventory'); + } + return settled; +} + module.exports = { authorityAttemptOwnsRecoveryReceipt, findAuthorityPrForAttempt, + hasAttemptIndexesForPr, reconcileFailedAuthorityAttempt, BRANCH, beginChallenge, diff --git a/.github/scripts/keepalive_reporter_applicability.js b/.github/scripts/keepalive_reporter_applicability.js index 9b3cdedb6..94bdc176b 100644 --- a/.github/scripts/keepalive_reporter_applicability.js +++ b/.github/scripts/keepalive_reporter_applicability.js @@ -2,6 +2,7 @@ const { findAuthorityPrForAttempt, + hasAttemptIndexesForPr, readAuthorityStateForReplay, reconcileFailedAuthorityAttempt, requester, @@ -152,6 +153,7 @@ async function replayReporterAuthority({ writerLogin, maxPasses = 3, readAuthority = readAuthorityStateForReplay, + hasAttemptIndexes = hasAttemptIndexesForPr, lookupTarget = findAuthorityPrForAttempt, reconcileAttempt = reconcileFailedAuthorityAttempt, projectRecovery = projectRecoveredAuthorityState, @@ -170,9 +172,18 @@ async function replayReporterAuthority({ const seen = new Set(); for (let pass = 0; pass < maxPasses; pass += 1) { const authority = await readAuthority(request, repository, number); - // Most keepalive PRs never enter the challenge path and have no ledger. Only - // absence proved from a complete, pinned authority tree is an empty replay. - if (authority === null) break; + // Most keepalive PRs never enter the challenge path and have no ledger. + // A confirmed 404 (null from readAuthorityStateForReplay) is only accepted after + // verifying the PR is ordinary (no attempt indexes). Authority candidates fail closed: + // a missing ledger for a PR with attempt indexes indicates a reconciliation problem + // that must not be silently abandoned. + if (authority === null) { + const hasIndexes = await hasAttemptIndexes(request, repository, number); + if (hasIndexes) { + throw new Error('Authority ledger is missing for a PR with attempt indexes; cannot abandon reconciliation'); + } + break; + } const { state } = authority; const attempts = []; for (const receipt of [state.receipt, state.released_receipt, state.recovered_receipt]) { diff --git a/.github/sync-manifest.yml b/.github/sync-manifest.yml index 7da3bfa9c..33ec8ce01 100644 --- a/.github/sync-manifest.yml +++ b/.github/sync-manifest.yml @@ -454,7 +454,7 @@ scripts: description: "Selects due automation-owned authority challenges for the hourly keepalive sweep" - source: .github/scripts/keepalive_authority_state.js - description: "PR-wide conditional authority generation, immutable attempt index, pinned-tree replay reads, exact-attempt release retry, and single-use receipt ledger" + description: "PR-wide conditional authority generation, immutable attempt index, pinned-tree replay reads, exact-attempt release retry, single-use receipt ledger, and attempt-index presence check for PR classification" - source: .github/scripts/keepalive_worker_evidence.js description: "Attempt-bound worker job evidence from the originating registry revision for keepalive authority recovery" diff --git a/docs/evidence/issue-3793-presence/README.md b/docs/evidence/issue-3793-presence/README.md new file mode 100644 index 000000000..1d1a60f8d --- /dev/null +++ b/docs/evidence/issue-3793-presence/README.md @@ -0,0 +1,37 @@ +# Issue 3793: durable attempt presence, existing PR 3792 + +The retained consumer proof compares the original exact head +`6102cc666ac99371d2196f038969ca481b314f20` with the repaired actual consumer helper. +All GitHub responses are simulated; no live API load is generated. The server +retains inventories across helper module reloads and pins their visibility to +commit snapshots. Three ledgerless negative PRs previously required **3,003** +index blob reads. Repair performs one **1,001**-blob complete migration and one +inventory write, then **zero** additional index blob reads across separate later +helper instances. A cached positive PR with a missing ledger still fails. + +Run from a checkout with the original commit available locally: + +```sh +node docs/evidence/issue-3793-presence/consumer-proof.js +node --test .github/scripts/__tests__/keepalive-attempt-presence.test.js .github/scripts/__tests__/keepalive-authority-state.test.js .github/scripts/__tests__/keepalive-reporter-applicability.test.js +``` + +`consumer-proof.json` records actual measured mock API counts, not an estimate. +`focused-green.log` records 87 passing authority/presence/reporter tests, including +both source/template surfaces. Mutable-tree tests prove older-writer +negative-to-positive invalidation and rejection of changes during warm reads, +backfill, and publication. Publication failure cases include lost response, +conflicting inventory, and malformed readback. Inventories are create-only; +conflicting publication is accepted only if the complete validated set agrees. + +Deliberate control: remove the final warm-read subtree recheck from the production +source helper, then run the `concurrent changes` regression. `warm-guard-red.log` +records exit 1 and the missing rejection; the source was restored byte-for-byte +before the retained 87-test GREEN run. This is an actual source mutation, not a +fixture-only failure or an invented assertion transcript. + +Source/template helper bytes match. Template sync, completeness and drift checks +pass. This packet establishes the local bounded protocol repair; the Major thread +and source issue remain open pending exact-head reviewer disposition, complete +hosted CI/topology, seven-minute review floor, squash merge, and actual issue-bound +`verify:compare`. No deployed protection or provider PASS is claimed here. diff --git a/docs/evidence/issue-3793-presence/consumer-proof.js b/docs/evidence/issue-3793-presence/consumer-proof.js new file mode 100644 index 000000000..574edb8b1 --- /dev/null +++ b/docs/evidence/issue-3793-presence/consumer-proof.js @@ -0,0 +1,56 @@ +'use strict'; +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const cp = require('node:child_process'); +const Module = require('node:module'); +const path = require('node:path'); +const root = path.resolve(__dirname, '../../..'); +const directory = path.join(root, 'templates/consumer-repo/.github/scripts'); +function server() { + const inventories = new Map(); const history = new Map([[1, new Map()]]); const blobs = new Map(); + let version = 1; let reads = 0; let writes = 0; + const sha = (n) => n.toString(16).padStart(40, '0'); + const indexSha = sha(400000); + const entries = Array.from({length:1001},(_,i) => { + const owner = `owner/repo:${i+1}:1`; const blobSha = sha(i+500000); + const value = {version:1,repository:'owner/repo',owner_attempt:owner,pr_number:9000,generation:'a'.repeat(64),receipt:{id:'b'.repeat(64),claim_digest:'c'.repeat(64),owner_attempt:owner,head_sha:'d'.repeat(40),provider:'codex',consumed_at:'2026-10-01T00:00:00.000Z'}}; + blobs.set(blobSha,{sha:blobSha,encoding:'base64',content:Buffer.from(JSON.stringify(value)).toString('base64')}); + return {path:crypto.createHash('sha256').update(owner).digest('hex')+'.json',type:'blob',sha:blobSha}; + }); + const request = async(method,url,body) => { + if (url.includes('/contents/.github/keepalive-authority-presence/')) { + const key = url.split('keepalive-authority-presence/')[1].split('?')[0]; + if(method==='PUT') { assert.equal(body.sha,undefined); if(inventories.has(key)) throw Object.assign(new Error('existing'),{status:422}); inventories.set(key,body.content); writes++; version++; history.set(version,new Map(inventories)); return {}; } + const ref = url.split('?ref=')[1]; const snapshot = ref==='keepalive-authority-state' ? inventories : history.get(parseInt(ref,16)-100000); + if(!snapshot || !snapshot.has(key)) throw Object.assign(new Error('missing'),{status:404}); + return {sha:sha(600000),encoding:'base64',content:snapshot.get(key)}; + } + assert.equal(method,'GET'); + if(url.includes('/git/ref/'))return {object:{type:'commit',sha:sha(100000+version)}}; + if(url.includes('/git/commits/'))return {tree:{sha:sha(200000+parseInt(url.split('/').pop(),16)-100000)}}; + if(url.includes('/git/trees/')) { + const n=parseInt(url.split('/').pop(),16); + if(n>200000 && n<300000)return {truncated:false,tree:[{path:'.github',type:'tree',sha:sha(n+100000)}]}; + if(n>300000 && n<400000)return {truncated:false,tree:[{path:'keepalive-authority-attempts',type:'tree',sha:indexSha}]}; + if(n===400000)return {truncated:false,tree:entries}; + } + const blob=blobs.get(url.split('/git/blobs/')[1]); assert.ok(blob,url); reads++; return blob; + }; + return {request,stats:()=>({blob_reads:reads,inventory_writes:writes})}; +} +(async()=>{ + const oldFile=path.join(directory,'keepalive_authority_state.js'); const oldModule=new Module(oldFile);oldModule.filename=oldFile;oldModule.paths=Module._nodeModulePaths(directory); + oldModule._compile(cp.execFileSync('git',['-C',root,'show','6102cc666ac99371d2196f038969ca481b314f20:templates/consumer-repo/.github/scripts/keepalive_authority_state.js'],{encoding:'utf8'}),oldFile); + const before=server();for(const pr of [42,43,44])assert.equal(await oldModule.exports.hasAttemptIndexesForPr(before.request,'owner/repo',pr),false);assert.equal(before.stats().blob_reads,3003); + const after=server(); + for(const pr of [42,43,44]) { + for(const filename of ['keepalive_authority_state.js','keepalive_reporter_applicability.js'])delete require.cache[require.resolve(path.join(directory,filename))]; + const {replayReporterAuthority}=require(path.join(directory,'keepalive_reporter_applicability.js')); + assert.deepEqual(await replayReporterAuthority({github:{},context:{repo:{owner:'owner',repo:'repo'}},prNumber:pr,readAuthority:async()=>null,makeRequest:()=>after.request}),{prNumber:pr,results:[]}); + } + assert.equal(after.stats().blob_reads,1001);assert.equal(after.stats().inventory_writes,1); + const {replayReporterAuthority}=require(path.join(directory,'keepalive_reporter_applicability.js')); + await assert.rejects(replayReporterAuthority({github:{},context:{repo:{owner:'owner',repo:'repo'}},prNumber:9000,readAuthority:async()=>null,makeRequest:()=>after.request}),/ledger is missing/); + assert.equal(after.stats().blob_reads,1001); + console.log(JSON.stringify({before:before.stats(),after:after.stats(),warm_blob_reads:0,negative_prs:3,positive_missing_ledger:'rejected',consumer_default_helper:'actual default helper across module reloads',live_api_load:0},null,2)); +})().catch(e=>{console.error(e);process.exitCode=1;}); diff --git a/docs/evidence/issue-3793-presence/consumer-proof.json b/docs/evidence/issue-3793-presence/consumer-proof.json new file mode 100644 index 000000000..811efccdb --- /dev/null +++ b/docs/evidence/issue-3793-presence/consumer-proof.json @@ -0,0 +1,15 @@ +{ + "before": { + "blob_reads": 3003, + "inventory_writes": 0 + }, + "after": { + "blob_reads": 1001, + "inventory_writes": 1 + }, + "warm_blob_reads": 0, + "negative_prs": 3, + "positive_missing_ledger": "rejected", + "consumer_default_helper": "actual default helper across module reloads", + "live_api_load": 0 +} diff --git a/docs/evidence/issue-3793-presence/focused-green.log b/docs/evidence/issue-3793-presence/focused-green.log new file mode 100644 index 000000000..9fdae411c --- /dev/null +++ b/docs/evidence/issue-3793-presence/focused-green.log @@ -0,0 +1,95 @@ +✔ ../keepalive_authority_state.js: finds an index beyond the Contents API directory limit (23.203042ms) +✔ ../keepalive_authority_state.js: proves absence only from complete validated evidence (0.439042ms) +✔ ../keepalive_authority_state.js: rejects incomplete, unreadable, or malformed evidence (0.839166ms) +✔ ../keepalive_authority_state.js: rejects lost, conflicting, or malformed inventory publication (0.378209ms) +✔ ../../../templates/consumer-repo/.github/scripts/keepalive_authority_state.js: finds an index beyond the Contents API directory limit (18.232083ms) +✔ ../../../templates/consumer-repo/.github/scripts/keepalive_authority_state.js: proves absence only from complete validated evidence (0.287625ms) +✔ ../../../templates/consumer-repo/.github/scripts/keepalive_authority_state.js: rejects incomplete, unreadable, or malformed evidence (0.537041ms) +✔ ../../../templates/consumer-repo/.github/scripts/keepalive_authority_state.js: rejects lost, conflicting, or malformed inventory publication (0.277333ms) +✔ ../keepalive_authority_state.js: an older writer invalidates negative presence by changing the subtree (0.794666ms) +✔ ../keepalive_authority_state.js: concurrent changes fail closed during warm read, backfill and publication (0.970042ms) +✔ ../../../templates/consumer-repo/.github/scripts/keepalive_authority_state.js: an older writer invalidates negative presence by changing the subtree (0.714125ms) +✔ ../../../templates/consumer-repo/.github/scripts/keepalive_authority_state.js: concurrent changes fail closed during warm read, backfill and publication (0.512125ms) +✔ replay proves an absent PR ledger from a complete pinned authority tree (1.113209ms) +✔ replay does not reinterpret an unavailable authority branch as no ledger (0.312042ms) +✔ replay rejects a truncated authority tree instead of proving absence (0.106333ms) +✔ replay rejects malformed entries instead of treating them as proof of absence (0.17975ms) +✔ replay accepts valid unrelated entries beside the authority path (0.128625ms) +✔ replay reads a present ledger from the pinned blob (1.297208ms) +✔ replay does not reinterpret an unreadable pinned ledger as absent (0.141209ms) +✔ replay rejects non-canonical base64 in a present pinned ledger (0.148292ms) +✔ replay rejects invalid receipts in every ledger slot (0.556333ms) +✔ one generation grants once across attempts, providers, heads and nonces (2.535875ms) +✔ preparation is non-authorizing and can be conditionally released before reservation (0.584917ms) +✔ an exact release retry is idempotent but cannot release a replacement receipt (0.54425ms) +✔ released availability rotates for a changed boundary while retaining receipt lineage (0.880958ms) +✔ released availability rotates an expired window while retaining receipt lineage (0.442334ms) +✔ beginChallenge reaps an expired orphaned preparation with exact index and PR evidence (0.853916ms) +✔ beginChallenge migrates an expired legacy preparation after exact index backfill (1.492458ms) +✔ legacy preparation migration accepts an exact index retry but rejects conflicts (1.46025ms) +✔ legacy preparation migration converges on a concurrent valid release (0.434458ms) +✔ lost preparation response remains recoverable after expiry (0.442ms) +✔ expired preparation stays fail-closed without exact index or eligible PR evidence (1.238959ms) +✔ expired preparation stays fail-closed when its persisted claim is inconsistent (0.263834ms) +✔ finalization winning the expired-preparation race preserves the consumed receipt (0.518667ms) +✔ expired prepared release creates a fresh due window without refunding its attempt (0.615709ms) +✔ expired earlier release refreshes once and retains the original receipt lineage (1.161333ms) +✔ expired preparation cannot rotate while the exact-head PR has a human blocker (0.305458ms) +✔ lost response after expired release is accepted only from settled exact ledger state (0.390042ms) +✔ a generation is never reused for a different originating head (0.164042ms) +✔ head changed during consumption spends receipt but denies grant (0.31675ms) +✔ head changed before the ledger PUT still denies the grant (0.27425ms) +✔ routing label changed during consumption denies the grant (0.235209ms) +✔ head changed during confirmation never reports trusted confirmation (0.490792ms) +✔ reconciliation rejects a replacement available head before reading its null receipt (0.157792ms) +✔ workflow reporter reopens with the persisted claim and failed run identity (0.395584ms) +✔ failed-run reconciliation is independent of summary state and requires positive non-start (0.419084ms) +✔ a corrupted direct attempt index denies finalization and missing-PR recovery (0.261916ms) +✔ lost index-create response denies this preparation but permits exact later retry (0.299083ms) +✔ consumed receipt only reopens for its exact attempt with a proven unstarted worker (0.303375ms) +✔ recovered retry revalidates PR state and refreshes an expired window (3.723791ms) +✔ recovered window refresh settles an exact committed write after response loss (0.546834ms) +✔ unavailable PR read after confirmation preserves the confirmed challenge (0.348167ms) +✔ confirmed receipt never reopens after a same-head hard label removal (0.290792ms) +✔ expired consumed generation cannot be replaced without positive non-start (0.4705ms) +✔ expired confirmed generation cannot be reopened by initialization (0.372959ms) +✔ two racing consumers produce at most one grant through the conditional SHA (0.417625ms) +✔ ambiguous write consumes if it landed but never grants a second execution (0.764458ms) +✔ missing or corrupt authoritative state never grants (0.232416ms) +✔ .github/workflows/agents-keepalive-loop.yml: ordinary unassociated dispatch recovers its canonical PR target (2.166625ms) +✔ .github/workflows/agents-keepalive-loop.yml: authority candidate without index fails closed (1.012375ms) +✔ .github/workflows/agents-keepalive-loop.yml: malformed or legacy binding cannot become ordinary (6.532041ms) +✔ templates/consumer-repo/.github/workflows/agents-81-gate-followups.yml: ordinary unassociated dispatch recovers its canonical PR target (0.623209ms) +✔ templates/consumer-repo/.github/workflows/agents-81-gate-followups.yml: authority candidate without index fails closed (0.613833ms) +✔ templates/consumer-repo/.github/workflows/agents-81-gate-followups.yml: malformed or legacy binding cannot become ordinary (7.250625ms) +✔ verified index wins over an ordinary title and needs no title lookup (0.173958ms) +✔ unavailable or corrupt index lookup never becomes an ordinary skip (0.172916ms) +✔ wrong originating workflow or event cannot claim ordinary routing (0.518542ms) +✔ originating revision without the classification contract fails closed (0.307791ms) +✔ contract-shaped text outside the top-level run-name cannot authorize a target (0.191084ms) +✔ associated runs avoid the unassociated locator altogether (0.075666ms) +✔ delayed released recovery is located, reconciled, and projected after summary retry (1.9075ms) +✔ delayed reopened recovery is located, reconciled, and projected after summary retry (0.202625ms) +✔ 404 reconciliation preserves the owner-attempt binding for normal reporting (1.500875ms) +✔ non-released reconciliation preserves the owner-attempt binding for normal reporting (0.114708ms) +✔ non-released delayed reporter stops when a newer attempt owns the running summary (0.123333ms) +✔ recovery rejects unknown worker evidence before authority mutation (0.073333ms) +✔ recovery rejects a delayed summary marker for a different attempt or generation (0.163542ms) +✔ superseded recovery stops reporter processing without comment mutation (0.084667ms) +✔ owner attempt parsing is repository-bound and canonical (0.127791ms) +✔ a later replay wake recovers a dropped middle reporter from the current receipt (0.337292ms) +✔ replay is a no-op when an ordinary PR has no authority ledger (0.08025ms) +✔ replay still fails closed when an authority ledger read fails (0.0645ms) +✔ replay fails closed when an authority candidate PR has a missing ledger but has attempt indexes (0.062959ms) +✔ replay rejects a present receipt with an invalid owner attempt (0.052708ms) +✔ replay fails closed when reconciliation remains uncertain (0.161917ms) +✔ replay fails closed when exact-attempt worker evidence is unknown (0.117042ms) +✔ replay defers an exact active attempt without worker reads or state writes (0.118625ms) +ℹ tests 87 +ℹ suites 0 +ℹ pass 87 +ℹ fail 0 +ℹ cancelled 0 +ℹ skipped 0 +ℹ todo 0 +ℹ duration_ms 104.042291 diff --git a/docs/evidence/issue-3793-presence/warm-guard-red.log b/docs/evidence/issue-3793-presence/warm-guard-red.log new file mode 100644 index 000000000..cade0d93b --- /dev/null +++ b/docs/evidence/issue-3793-presence/warm-guard-red.log @@ -0,0 +1,25 @@ +✖ ../keepalive_authority_state.js: concurrent changes fail closed during warm read, backfill and publication (3.861667ms) +✔ ../../../templates/consumer-repo/.github/scripts/keepalive_authority_state.js: concurrent changes fail closed during warm read, backfill and publication (1.459416ms) +ℹ tests 2 +ℹ suites 0 +ℹ pass 1 +ℹ fail 1 +ℹ cancelled 0 +ℹ skipped 0 +ℹ todo 0 +ℹ duration_ms 59.228708 + +✖ failing tests: + +test at .github/scripts/__tests__/keepalive-attempt-presence.test.js:147:3 +✖ ../keepalive_authority_state.js: concurrent changes fail closed during warm read, backfill and publication (3.861667ms) + AssertionError [ERR_ASSERTION]: Missing expected rejection. + at async TestContext. (/Users/teacher/.codex/automations/reviewed-repo-orphan-pr-steward/worktrees/workflows-3792-repair/.github/scripts/__tests__/keepalive-attempt-presence.test.js:153:7) + at async Test.run (node:internal/test_runner/test:1069:7) + at async startSubtestAfterBootstrap (node:internal/test_runner/harness:332:3) { + generatedMessage: false, + code: 'ERR_ASSERTION', + actual: undefined, + expected: /changed during/, + operator: 'rejects' + } diff --git a/docs/keepalive/GoalsAndPlumbing.md b/docs/keepalive/GoalsAndPlumbing.md index 4f17660ff..28e15255d 100644 --- a/docs/keepalive/GoalsAndPlumbing.md +++ b/docs/keepalive/GoalsAndPlumbing.md @@ -125,7 +125,7 @@ PR head and labels live outside the authority ledger, so their reads and ledger After a failed or cancelled originating run, the workflow-run reporter reads that run's exact job attempt and the agent registry from that run's exact head commit. A worker step that actually entered is treated as started; a fully terminal skipped worker path is positively not-started; incomplete jobs, API errors, missing job evidence, or unavailable originating registry are unknown. Unknown evidence fails the reporter before summary or fingerprint persistence so a retry remains possible. Only the not-started case may reconcile the matching receipt and owner attempt independently of summary `running` state or a fresh authority-failure classification. A prepared receipt can then be released and an unconfirmed consumed receipt can be reopened after a same-head PR read; confirmed receipts remain spent. A retry of an already reopened receipt revalidates the immutable attempt index, exact PR head, soft attention label, and absence of `needs-human` before returning success. If its 24-hour window expired, the retry conditionally rotates the generation and retains the original plus recent recovered generations as bounded lineage; summary projection accepts only that receipt-bound recovered lineage. A fresh preparation clears the lineage so a different attempt cannot inherit it. The reporter's consumed-receipt primitive requires non-start proof, so an ordinary summary or confirmation caller cannot bypass this boundary. Expired initialization is a separate preparation-only recovery: because `prepared` has not granted execution, exact claim/index and PR-state validation plus the conditional SHA are sufficient, while any consumed or conflicting state stays fail-closed. The reporter then projects the settled generation, due/expiry times, and non-running status to a matching trusted summary; retrying a lost projection response is idempotent. A running summary is bound to its exact repository, run ID, and run attempt, so a delayed reporter cannot clear a newer run that happens to share the same generation. Immediately before projection, the reporter re-reads the authoritative ledger and treats any changed or non-available settled receipt as superseded without falling through to the ordinary summary writer. A sweep-triggered run may have no webhook PR association; the reporter reads the exact attempt locator directly and verifies the receipt ID and bindings in the authoritative PR ledger without scanning historical PRs. Missing, conflicting or uncertain lookup fails explicitly for owner reconciliation rather than asserting no worker executed. Associated runs use their known PR directly. Consumer fingerprinting includes run ID and attempt so a rerun cannot be skipped as identical to its predecessor. A missing or inconclusive job read never refunds consumed authority. -Reporter concurrency is only a mutual-exclusion boundary; GitHub may replace an older pending job when several runs share one group. Every reporter wake therefore reconciles the PR ledger's current `receipt`, `released_receipt`, and `recovered_receipt` obligations before handling its triggering event. Each owner attempt is verified through its immutable index, exact run attempt, originating-head registry, and exact worker evidence. The hourly sweep independently dispatches a PR-bound reporter replay before ordinary keepalive evaluation, so a cancelled last reporter or a crash before projection does not require another producer completion. Replay is bounded to three ledger rereads and never scans historical indexes. It resolves the authority branch to one commit, walks complete non-recursive trees to the PR path, and treats the path's verified absence from that pinned snapshot as a successful no-op. Every tree entry and every present receipt must be structurally valid and repository-bound before absence or replay can be concluded. A missing or inaccessible branch, commit, tree, or present blob, as well as a truncated or malformed tree, fails closed instead of impersonating absence. A receipt whose exact indexed workflow attempt is still queued or running returns `deferred-active-attempt` without worker-evidence reads, reconciliation, projection, or state writes. Missing runs, unknown worker evidence, unsupported run states, changed heads, hard human holds, inaccessible storage, and an `uncertain` reconciliation fail the replay rather than completing successfully. Consumer fingerprint debounce cannot suppress this manual replay path. The guarantee is eventual reconciliation of current ledger obligations, not an audit acknowledgment for every superseded historical completion. +Reporter concurrency is only a mutual-exclusion boundary; GitHub may replace an older pending job when several runs share one group. Every reporter wake therefore reconciles the PR ledger's current `receipt`, `released_receipt`, and `recovered_receipt` obligations before handling its triggering event. Each owner attempt is verified through its immutable index, exact run attempt, originating-head registry, and exact worker evidence. The hourly sweep independently dispatches a PR-bound reporter replay before ordinary keepalive evaluation, so a cancelled last reporter or a crash before projection does not require another producer completion. Replay is bounded to three ledger rereads. Only when a ledger is missing, it scans the immutable attempt-index directory from a pinned commit using complete non-recursive trees and SHA-bound blobs to prove whether the PR has any prior attempts; truncated trees, unreadable blobs, or invalid indexes fail closed. Replay does not scan historical indexes for other purposes. It resolves the authority branch to one commit, walks complete non-recursive trees to the PR path, and treats the path's verified absence from that pinned snapshot as a potential no-op. Every tree entry and every present receipt must be structurally valid and repository-bound before absence or replay can be concluded. A confirmed missing ledger is only accepted as a successful no-op after verifying the PR has no attempt indexes (i.e., it is ordinary); a PR with attempt indexes fails closed to prevent silently abandoning reconciliation for one that previously entered the challenge path. A missing or inaccessible branch, commit, tree, or present blob, as well as a truncated or malformed tree, fails closed instead of impersonating absence. A receipt whose exact indexed workflow attempt is still queued or running returns `deferred-active-attempt` without worker-evidence reads, reconciliation, projection, or state writes. Missing runs, unknown worker evidence, unsupported run states, changed heads, hard human holds, inaccessible storage, and an `uncertain` reconciliation fail the replay rather than completing successfully. Consumer fingerprint debounce cannot suppress this manual replay path. The guarantee is eventual reconciliation of current ledger obligations, not an audit acknowledgment for every superseded historical completion. Reporter mutation is serialized by the resolved PR, including for unassociated `workflow_dispatch` runs. A read-only resolver job classifies the originating @@ -520,3 +520,44 @@ field empty, which keeps "no drainable path stated" from ever reading as "nothin Constants live in `scripts/runner_lib/core.py`: `UNPRODUCTIVE_COMPLETION_RETRY_LIMIT` and `UNPRODUCTIVE_COMPLETION_COOLDOWN_SECONDS`. + + +### Durable missing-ledger attempt presence + +The missing-ledger replay guard uses a durable complete presence inventory on the +`keepalive-authority-state` branch, under +`.github/keepalive-authority-presence/.json`. The key is the +immutable `.github/keepalive-authority-attempts` subtree, not the branch commit: +writing an inventory does not invalidate itself. The inventory stores the sorted +set of PRs with validated indexes; absence from that set is a negative result only +for that exact complete tree. It grants no execution authority and never replaces +ledger or receipt reconciliation for a positive PR. + +On a cache miss, the reporter traverses complete non-truncated pinned Git trees, +validates every index blob and its receipt/filename/repository binding, and creates +the inventory without an overwrite SHA. A concurrent create is accepted only when +readback exactly matches the independently computed set. Partial, malformed, +unavailable, lost-response, conflicting or unconfirmed writes fail closed. Cache +reads use the snapshot commit and recheck the current index subtree before return; +backfill and publication also recheck it. Any writer, including an older writer, +that creates an attempt index changes the subtree key. An older negative inventory +therefore cannot certify absence in the newer tree. Retry rebuilds that complete +new tree rather than updating a partial positive-only marker. + +When the pinned snapshot has no `.github` or attempt directory, the reader +rechecks the current branch's complete trees before returning absence. Creation +of the first attempt directory during that read is uncertain and fails closed; +the next independent read migrates the newly present subtree. A failed scan +publishes nothing. A publication whose response is lost denies the current read +even if it landed; a later reader may reuse it only after validating the durable +inventory and the current subtree. Concurrent create-only backfills may converge +on an identical independently validated inventory after a 409/422 response. + +The first migration scan remains proportional to legacy indexes. Once persisted, +separate later reporter instances reuse one inventory without reading every index +blob again; tree and inventory API calls remain bounded independently of index +count. Migration requires the existing dedicated reporter App's contents-write +permission. The read-only target classifier does not invoke this migration path. +An inaccessible writer is an automation error, never successful absence. Retained +inventories are evidence for immutable trees; this change does not garbage-collect +the authority branch or claim that all broader recovery acceptance is complete. diff --git a/templates/consumer-repo/.github/scripts/keepalive_authority_state.js b/templates/consumer-repo/.github/scripts/keepalive_authority_state.js index ef15ecdc0..66f78d112 100644 --- a/templates/consumer-repo/.github/scripts/keepalive_authority_state.js +++ b/templates/consumer-repo/.github/scripts/keepalive_authority_state.js @@ -76,6 +76,11 @@ function attemptPath(repository, ownerAttempt) { return `/repos/${String(repository).toLowerCase()}/contents/.github/keepalive-authority-attempts/${key}.json`; } +function presencePath(repository, indexTreeSha) { + if (!HEAD.test(String(indexTreeSha))) throw new Error('Invalid authority attempt tree SHA'); + return `/repos/${String(repository).toLowerCase()}/contents/.github/keepalive-authority-presence/${indexTreeSha}.json`; +} + function validAttemptIndex(index, repository, ownerAttempt) { return index?.version === 1 && index.repository === String(repository).toLowerCase() && index.owner_attempt === ownerAttempt && Number.isSafeInteger(index.pr_number) && @@ -985,9 +990,156 @@ async function findAuthorityPrForAttempt({ request, repository, ownerAttempt }) return { prNumber: index.pr_number, state }; } +async function hasAttemptIndexesForPr(request, repository, prNumber) { + pathFor(repository, prNumber); + const snapshot = await attemptIndexTree(request, repository); + if (snapshot.treeSha === null) { + // The first index may be created by a legacy writer after this snapshot. + // Missing directories need the same freshness fence as cached negatives. + const current = await attemptIndexTree(request, snapshot.repo); + if (current.treeSha !== null) { + throw new Error('Authority attempt indexes changed during absence read'); + } + return false; + } + let inventory = await readAttemptPresence(request, snapshot.repo, snapshot.treeSha, snapshot.commitSha); + if (!inventory) { + const positives = await scanAttemptIndexes(request, snapshot); + // A writer may have added an index while this complete scan was in flight. + // Never publish an absence result for a different subtree. + const current = await attemptIndexTree(request, snapshot.repo); + if (!current || current.treeSha !== snapshot.treeSha) { + throw new Error('Authority attempt indexes changed during inventory backfill'); + } + inventory = await createAttemptPresence(request, snapshot, positives); + const settled = await attemptIndexTree(request, snapshot.repo); + if (!settled || settled.treeSha !== snapshot.treeSha) { + throw new Error('Authority attempt indexes changed during inventory publication'); + } + } + const current = await attemptIndexTree(request, snapshot.repo); + if (!current || current.treeSha !== snapshot.treeSha) { + throw new Error('Authority attempt indexes changed during presence read'); + } + return inventory.positive_prs.includes(Number(prNumber)); +} + +async function attemptIndexTree(request, repository) { + const repo = String(repository).toLowerCase(); + const ref = await request('GET', `/repos/${repo}/git/ref/heads/${BRANCH}`); + const commitSha = ref?.object?.sha; + if (ref?.object?.type !== 'commit' || !HEAD.test(String(commitSha))) { + throw new Error('Authority attempt branch did not resolve to a commit'); + } + const commit = await request('GET', `/repos/${repo}/git/commits/${commitSha}`); + let treeSha = commit?.tree?.sha; + const segments = ['.github', 'keepalive-authority-attempts']; + // Non-recursive trees avoid the Contents API's 1,000-entry directory limit. + // Every later read uses immutable SHAs from this one branch snapshot. + for (const segment of [...segments, null]) { + if (!HEAD.test(String(treeSha))) throw new Error('Invalid authority attempt tree SHA'); + const tree = await request('GET', `/repos/${repo}/git/trees/${treeSha}`); + if (tree?.truncated !== false || !Array.isArray(tree.tree) || + !tree.tree.every((entry) => entry && typeof entry.path === 'string' && + entry.path.length > 0 && !entry.path.includes('/') && + ['blob', 'tree', 'commit'].includes(entry.type) && HEAD.test(String(entry.sha))) || + new Set(tree.tree.map((entry) => entry.path)).size !== tree.tree.length) { + throw new Error('Incomplete or malformed authority attempt tree'); + } + if (segment !== null) { + const entry = tree.tree.find((item) => item.path === segment); + if (!entry) return { repo, commitSha, treeSha: null }; + if (entry.type !== 'tree') throw new Error('Authority attempt directory is not a tree'); + treeSha = entry.sha; + continue; + } + return { repo, commitSha, treeSha }; + } + throw new Error('Authority attempt tree traversal did not settle'); +} + +async function scanAttemptIndexes(request, snapshot) { + const tree = await request('GET', `/repos/${snapshot.repo}/git/trees/${snapshot.treeSha}`); + if (tree?.truncated !== false || !Array.isArray(tree.tree)) { + throw new Error('Incomplete or malformed authority attempt tree'); + } + const positives = new Set(); + for (const entry of tree.tree) { + if (!/^[0-9a-f]{64}\.json$/.test(entry.path) || entry.type !== 'blob') { + throw new Error('Invalid authority attempt index path'); + } + const blob = await request('GET', `/repos/${snapshot.repo}/git/blobs/${entry.sha}`); + if (blob?.sha !== entry.sha || blob?.encoding !== 'base64' || + typeof blob.content !== 'string') { + throw new Error('Invalid authority attempt index metadata'); + } + const encoded = blob.content.replace(/\s/g, ''); + if (!encoded || Buffer.from(encoded, 'base64').toString('base64') !== encoded) { + throw new Error('Malformed authority attempt index base64'); + } + const index = JSON.parse(Buffer.from(encoded, 'base64').toString('utf8')); + if (!validAttemptIndex(index, snapshot.repo, index?.owner_attempt) || + attemptPath(snapshot.repo, index.owner_attempt).split('/').pop() !== entry.path) { + throw new Error('Invalid authority attempt index'); + } + positives.add(index.pr_number); + } + return [...positives].sort((left, right) => left - right); +} + +function decodePresence(file, repository, treeSha) { + if (!HEAD.test(String(file?.sha)) || file?.encoding !== 'base64' || typeof file.content !== 'string') { + throw new Error('Invalid authority attempt presence metadata'); + } + const encoded = file.content.replace(/\s/g, ''); + if (!encoded || Buffer.from(encoded, 'base64').toString('base64') !== encoded) { + throw new Error('Malformed authority attempt presence base64'); + } + let inventory; + try { inventory = JSON.parse(Buffer.from(encoded, 'base64').toString('utf8')); } catch (error) { + throw new Error(`Malformed authority attempt presence: ${error.message}`); + } + if (inventory?.version !== 1 || inventory.repository !== repository || + inventory.index_tree_sha !== treeSha || !Array.isArray(inventory.positive_prs) || + !inventory.positive_prs.every((value) => Number.isSafeInteger(value) && value > 0) || + new Set(inventory.positive_prs).size !== inventory.positive_prs.length || + inventory.positive_prs.some((value, index) => index && inventory.positive_prs[index - 1] >= value)) { + throw new Error('Invalid authority attempt presence'); + } + return inventory; +} + +async function readAttemptPresence(request, repository, treeSha, ref = BRANCH) { + try { + return decodePresence(await request('GET', `${presencePath(repository, treeSha)}?ref=${ref}`), repository, treeSha); + } catch (error) { + if (error.status === 404) return null; + throw error; + } +} + +async function createAttemptPresence(request, snapshot, positives) { + const inventory = { version: 1, repository: snapshot.repo, index_tree_sha: snapshot.treeSha, + positive_prs: positives }; + try { + await request('PUT', presencePath(snapshot.repo, snapshot.treeSha), { branch: BRANCH, + message: `keepalive authority presence ${snapshot.treeSha}`, + content: Buffer.from(`${JSON.stringify(inventory)}\n`).toString('base64') }); + } catch (error) { + if (![409, 422].includes(error.status)) throw error; + } + const settled = await readAttemptPresence(request, snapshot.repo, snapshot.treeSha, BRANCH); + if (!settled) throw new Error('Authority attempt presence write was not confirmed'); + if (JSON.stringify(settled.positive_prs) !== JSON.stringify(positives)) { + throw new Error('Authority attempt presence conflicts with validated inventory'); + } + return settled; +} + module.exports = { authorityAttemptOwnsRecoveryReceipt, findAuthorityPrForAttempt, + hasAttemptIndexesForPr, reconcileFailedAuthorityAttempt, BRANCH, beginChallenge, diff --git a/templates/consumer-repo/.github/scripts/keepalive_reporter_applicability.js b/templates/consumer-repo/.github/scripts/keepalive_reporter_applicability.js index 9b3cdedb6..94bdc176b 100644 --- a/templates/consumer-repo/.github/scripts/keepalive_reporter_applicability.js +++ b/templates/consumer-repo/.github/scripts/keepalive_reporter_applicability.js @@ -2,6 +2,7 @@ const { findAuthorityPrForAttempt, + hasAttemptIndexesForPr, readAuthorityStateForReplay, reconcileFailedAuthorityAttempt, requester, @@ -152,6 +153,7 @@ async function replayReporterAuthority({ writerLogin, maxPasses = 3, readAuthority = readAuthorityStateForReplay, + hasAttemptIndexes = hasAttemptIndexesForPr, lookupTarget = findAuthorityPrForAttempt, reconcileAttempt = reconcileFailedAuthorityAttempt, projectRecovery = projectRecoveredAuthorityState, @@ -170,9 +172,18 @@ async function replayReporterAuthority({ const seen = new Set(); for (let pass = 0; pass < maxPasses; pass += 1) { const authority = await readAuthority(request, repository, number); - // Most keepalive PRs never enter the challenge path and have no ledger. Only - // absence proved from a complete, pinned authority tree is an empty replay. - if (authority === null) break; + // Most keepalive PRs never enter the challenge path and have no ledger. + // A confirmed 404 (null from readAuthorityStateForReplay) is only accepted after + // verifying the PR is ordinary (no attempt indexes). Authority candidates fail closed: + // a missing ledger for a PR with attempt indexes indicates a reconciliation problem + // that must not be silently abandoned. + if (authority === null) { + const hasIndexes = await hasAttemptIndexes(request, repository, number); + if (hasIndexes) { + throw new Error('Authority ledger is missing for a PR with attempt indexes; cannot abandon reconciliation'); + } + break; + } const { state } = authority; const attempts = []; for (const receipt of [state.receipt, state.released_receipt, state.recovered_receipt]) {