diff --git a/.changeset/review-live-producer-staging.md b/.changeset/review-live-producer-staging.md new file mode 100644 index 00000000..7cf3c638 --- /dev/null +++ b/.changeset/review-live-producer-staging.md @@ -0,0 +1,5 @@ +--- +"review": minor +--- + +The live producer (live A/B plan, phase 2): everything needed to run the REAL model sub-agents over a live-enabled corpus case. `eval/agent-extract.ts` parses `review.md`'s `## agent:` sections into name/description/model/prompt as pure data (string in, no fs), so an A/B can extract the baseline arm from a `git show` of the merge-base; parsing is strict and lists every malformed section at once, and an integration test runs it over the real `review.md`. `eval/live-stage.ts` materializes the production `/tmp/gh-aw/review/` staging layout for a case (pr-context.json, full/pr/full-stripped diffs, files.json with hasPatch, review-files.json, provenance.json, routing.json, out/, and the post-change checkout) and rewrites extracted prompts to point at it. `eval/live-producer.ts` dispatches the default finders plus the router's lenses through an injected runner seam, maps all three output contracts (label-shape reviewers, structured-schema lenses, the claim-validator's three-state verdicts) into the exact `RecordedFinding`/`CaseVerification` shapes the deterministic runner consumes, stages `claims.json`, resolves `{{#runtime-import}}` directives from the case tree, retries once on malformed output, keeps partial results on agent failure, and accounts per-agent cost. `eval/live-runner.ts` is the one SDK-backed runner (Claude Agent SDK; Read/Grep/Glob only, cwd pinned to the staged checkout, hard turn and wall-clock caps) plus a CLI smoke entry (`pnpm dlx tsx workflows/review/eval/live-runner.ts --case `, requires ANTHROPIC_API_KEY). diff --git a/package.json b/package.json index 3d5bed5e..3fcc1079 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "build": "tsc -p actions/tsconfig.json" }, "devDependencies": { + "@anthropic-ai/claude-agent-sdk": "^0.3.205", "@changesets/cli": "^2.29.8", "@khanacademy/eslint-config": "^0.1.0", "@swc-node/register": "^1.11.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f679e932..019cacac 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: devDependencies: + '@anthropic-ai/claude-agent-sdk': + specifier: ^0.3.205 + version: 0.3.205(@anthropic-ai/sdk@0.110.0(zod@4.4.3))(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(zod@4.4.3) '@changesets/cli': specifier: ^2.29.8 version: 2.29.8(@types/node@25.3.3) @@ -108,6 +111,63 @@ importers: packages: + '@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.205': + resolution: {integrity: sha512-lrfJ4eVtzfPkCpbSkBOGSMQCBbvmW6nbPzgHE4IwMN3scZlpuFMUFqh2aaJa/X2SAcWD9H2S0t2WWvSRgM7BjA==} + cpu: [arm64] + os: [darwin] + + '@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.205': + resolution: {integrity: sha512-G6ETPmL5mNzJ2DFsWxG3jmsmrXgZX1N2ZCJvxaGUUpjTsKZJ4Tup1cWYvcd/m7o5fYZmx9REmgzTwsAIc1fdPQ==} + cpu: [x64] + os: [darwin] + + '@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.205': + resolution: {integrity: sha512-91fgdG4aTnQ29sKOcUqgH4+tKCW2ut6PWGRSYmXNDbROasJm1rAlPdzC5brdu/e4c0CDSNV6TWyE5JCjaS/jlQ==} + cpu: [arm64] + os: [linux] + + '@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.205': + resolution: {integrity: sha512-CXzySK3PV3EizCRPXnxPqeaAtgrBFDnMFOVpMe36oC3U16yDb1b1tAJGqZi/7uFrVvAiaXvnSFxhUWnDDSaO+A==} + cpu: [arm64] + os: [linux] + + '@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.205': + resolution: {integrity: sha512-vvsb7GlnA8CTSVvvTkrXjcSeRKqxSM7p/tU3Od9ICAZeWHglptekEyzLEApzLuLbI5ewfFF/F0q3NwOBbo18dg==} + cpu: [x64] + os: [linux] + + '@anthropic-ai/claude-agent-sdk-linux-x64@0.3.205': + resolution: {integrity: sha512-siS+1iNqBSlGFZZvJY6+mhzZ/6/ec/TbX9GMuwmTF0E6fxGhIIp797jJxR1q8r6FAq7d39mEoRNhC0Ffo60uNQ==} + cpu: [x64] + os: [linux] + + '@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.205': + resolution: {integrity: sha512-SpP5zF68weFez/6pKrGzq/UVAJDMDNphWqmkLfOpWTDBL5xy6XlIZw5Bl4EXoVnfi2VLFkwuffNeFe+9SdX7kw==} + cpu: [arm64] + os: [win32] + + '@anthropic-ai/claude-agent-sdk-win32-x64@0.3.205': + resolution: {integrity: sha512-kg2kkXyeSoFLruO3Ic2IruLxzBR0xCUtmlJHdWi3SYW7JhAKNJg4fcrdJsWcardmEw23Y2UDGDJbRyxqSVx6wg==} + cpu: [x64] + os: [win32] + + '@anthropic-ai/claude-agent-sdk@0.3.205': + resolution: {integrity: sha512-ft6iBw9kXudsusiXNpeybIPBJ07Z3tqp1ROSg5cEJqgA+9i+JJj2sRfQth+QD+lyenbbAU8yPieLxIimvfBhtw==} + engines: {node: '>=18.0.0'} + peerDependencies: + '@anthropic-ai/sdk': '>=0.93.0' + '@modelcontextprotocol/sdk': ^1.29.0 + zod: ^4.0.0 + + '@anthropic-ai/sdk@0.110.0': + resolution: {integrity: sha512-hOP4bNYXDFHDxxiEgzlILXrxZIYCDnhe8sry0RDRKD/QnsEpvZcQpablCdm9X/WuD/YgOiSIkkqsL1mLLlTqJw==} + hasBin: true + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + peerDependenciesMeta: + zod: + optional: true + '@ardatan/aggregate-error@0.0.6': resolution: {integrity: sha512-vyrkEHG1jrukmzTPtyWB4NLPauUw5bQeg4uhn8f+1SSynmrOcyvlb1GKQjjgoBzElLdfXCRYX8UnBlhklOHYRQ==} engines: {node: '>=8'} @@ -456,6 +516,12 @@ packages: peerDependencies: graphql: ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 + '@hono/node-server@1.19.14': + resolution: {integrity: sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==} + engines: {node: '>=18.14.1'} + peerDependencies: + hono: ^4 + '@humanwhocodes/config-array@0.9.5': resolution: {integrity: sha512-ObyMyWxZiCu/yTisA7uzx81s40xR2fD5Cg/2Kq7G02ajkNubJf6BopgDTmDyc3U7sXpNKM8cYOw7s7Tyr+DnCw==} engines: {node: '>=10.10.0'} @@ -552,6 +618,16 @@ packages: '@microsoft/fetch-event-source@2.0.1': resolution: {integrity: sha512-W6CLUJ2eBMw3Rec70qrsEW0jOm/3twwJv21mrmj2yORiaVmVYGS4sSS5yUwvQc1ZlDLYGPnClVWmUUMagKNsfA==} + '@modelcontextprotocol/sdk@1.29.0': + resolution: {integrity: sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ==} + engines: {node: '>=18'} + peerDependencies: + '@cfworker/json-schema': ^4.1.1 + zod: ^3.25 || ^4.0 + peerDependenciesMeta: + '@cfworker/json-schema': + optional: true + '@napi-rs/wasm-runtime@1.1.1': resolution: {integrity: sha512-p64ah1M1ld8xjWv3qbvFwHiFVWrq1yFvV4f7w+mzaqiR4IlSgkqhcRdHwsGgomwzBH51sRY4NEowLxnaBjcW/A==} @@ -780,6 +856,9 @@ packages: '@rtsao/scc@1.1.0': resolution: {integrity: sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g==} + '@stablelib/base64@1.0.1': + resolution: {integrity: sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==} + '@standard-schema/spec@1.0.0': resolution: {integrity: sha512-m2bOd0f2RT9k8QJx1JN85cZYyH1RqFBdlwtkSlf4tBDYLCiiZnv1fIIwacK6cqwXavOydf0NPToMQgpKq+dVlA==} @@ -993,6 +1072,10 @@ packages: resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} engines: {node: '>=6.5'} + accepts@2.0.0: + resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} + engines: {node: '>= 0.6'} + acorn-jsx@5.3.2: resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} peerDependencies: @@ -1003,9 +1086,20 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + ajv-formats@3.0.1: + resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==} + peerDependencies: + ajv: ^8.0.0 + peerDependenciesMeta: + ajv: + optional: true + ajv@6.12.6: resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} + ajv@8.20.0: + resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} + ansi-colors@4.1.3: resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} engines: {node: '>=6'} @@ -1128,6 +1222,10 @@ packages: resolution: {integrity: sha512-pbnl5XzGBdrFU/wT4jqmJVPn2B6UHPBOhzMQkY/SPUPB6QtUXtmBHBIwCbXJol93mOpGMnQyP/+BB19q04xj7g==} engines: {node: '>=4'} + body-parser@2.3.0: + resolution: {integrity: sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==} + engines: {node: '>=18'} + brace-expansion@1.1.11: resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} @@ -1148,6 +1246,10 @@ packages: buffer@5.7.1: resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + bytes@3.1.2: + resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} + engines: {node: '>= 0.8'} + call-bind-apply-helpers@1.0.2: resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} engines: {node: '>= 0.4'} @@ -1224,6 +1326,30 @@ packages: concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} + content-disposition@1.1.0: + resolution: {integrity: sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==} + engines: {node: '>=18'} + + content-type@1.0.5: + resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} + engines: {node: '>= 0.6'} + + content-type@2.0.0: + resolution: {integrity: sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==} + engines: {node: '>=18'} + + cookie-signature@1.2.2: + resolution: {integrity: sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==} + engines: {node: '>=6.6.0'} + + cookie@0.7.2: + resolution: {integrity: sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==} + engines: {node: '>= 0.6'} + + cors@2.8.6: + resolution: {integrity: sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==} + engines: {node: '>= 0.10'} + cosmiconfig-toml-loader@1.0.0: resolution: {integrity: sha512-H/2gurFWVi7xXvCyvsWRLCMekl4tITJcX0QEsDMpzxtuxDyM59xLatYNg4s/k9AA/HdtCYfj2su8mgA0GSDLDA==} @@ -1316,6 +1442,10 @@ packages: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} + depd@2.0.0: + resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} + engines: {node: '>= 0.8'} + detect-indent@6.1.0: resolution: {integrity: sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA==} engines: {node: '>=8'} @@ -1340,12 +1470,19 @@ packages: resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} engines: {node: '>= 0.4'} + ee-first@1.1.1: + resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} + emoji-regex@8.0.0: resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} emoji-regex@9.2.2: resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} + encodeurl@2.0.0: + resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} + engines: {node: '>= 0.8'} + enquirer@2.4.1: resolution: {integrity: sha512-rRqJg/6gd538VHvR3PSrdRBb/1Vy2YfzHqzvbhGIQpDRKIa4FgV/54b5Q1xYSxOOwKvjXweS26E0Q+nAMwp2pQ==} engines: {node: '>=8.6'} @@ -1396,6 +1533,9 @@ packages: engines: {node: '>=18'} hasBin: true + escape-html@1.0.3: + resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} + escape-string-regexp@1.0.5: resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} engines: {node: '>=0.8.0'} @@ -1570,6 +1710,10 @@ packages: resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} engines: {node: '>=0.10.0'} + etag@1.8.1: + resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} + engines: {node: '>= 0.6'} + event-target-shim@5.0.1: resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} engines: {node: '>=6'} @@ -1577,10 +1721,28 @@ packages: eventemitter3@3.1.2: resolution: {integrity: sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q==} + eventsource-parser@3.1.0: + resolution: {integrity: sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==} + engines: {node: '>=18.0.0'} + + eventsource@3.0.7: + resolution: {integrity: sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==} + engines: {node: '>=18.0.0'} + expect-type@1.2.2: resolution: {integrity: sha512-JhFGDVJ7tmDJItKhYgJCGLOWjuK9vPxiXoUFLwLDc99NlmklilbiQJwoctZtt13+xMw91MCk/REan6MWHqDjyA==} engines: {node: '>=12.0.0'} + express-rate-limit@8.5.2: + resolution: {integrity: sha512-5Kb34ipNX694DH48vN9irak1Qx30nb0PLYHXfJgw4YEjiC3ZEmZJhwOp+VfiCYwFzvFTdB9QkArYS5kXa2cx2A==} + engines: {node: '>= 16'} + peerDependencies: + express: '>= 4.11' + + express@5.2.1: + resolution: {integrity: sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==} + engines: {node: '>= 18'} + extendable-error@0.1.7: resolution: {integrity: sha512-UOiS2in6/Q0FK0R0q6UY9vYpQ21mr/Qn1KOnte7vsACuNJf514WvCCUHSRCPcgjPT2bAhNIJdlE6bVap1GKmeg==} @@ -1604,6 +1766,12 @@ packages: fast-levenshtein@2.0.6: resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} + fast-sha256@1.3.0: + resolution: {integrity: sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==} + + fast-uri@3.1.3: + resolution: {integrity: sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg==} + fastq@1.13.0: resolution: {integrity: sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==} @@ -1628,6 +1796,10 @@ packages: resolution: {integrity: sha512-GdvMk8r98FmuSLErSnWzfRIylY1OWLMxlN6YmG6/JZp3dIBZUzcEGyoiSu+hM+bnwGuUS1msQ7raGB5iWeSP/g==} engines: {node: '>=0.10.0'} + finalhandler@2.1.1: + resolution: {integrity: sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==} + engines: {node: '>= 18.0.0'} + find-up@2.1.0: resolution: {integrity: sha512-NWzkk0jSJtTt08+FBFMvXoeZnOJD+jTtsRmBYbAIzJdX6l7dLgR7CTubCM5/eDdPUBvLCeVasP1brfVR/9/EZQ==} engines: {node: '>=4'} @@ -1651,6 +1823,14 @@ packages: resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} engines: {node: '>= 6'} + forwarded@0.2.0: + resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} + engines: {node: '>= 0.6'} + + fresh@2.0.0: + resolution: {integrity: sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==} + engines: {node: '>= 0.8'} + fs-extra@7.0.1: resolution: {integrity: sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==} engines: {node: '>=6 <7 || >=8'} @@ -1800,12 +1980,20 @@ packages: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} + hono@4.12.28: + resolution: {integrity: sha512-YwUvVpSF7m1yOblFPrU3Hbo8XhPheBoiyfGuII6z19LnOr6JpDnyyp7LFNrfV56wS8tpvtBFGRISHN02pDdLOA==} + engines: {node: '>=16.9.0'} + hosted-git-info@2.8.9: resolution: {integrity: sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==} html-escaper@2.0.2: resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} + http-errors@2.0.1: + resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} + engines: {node: '>= 0.8'} + human-id@4.1.3: resolution: {integrity: sha512-tsYlhAYpjCKa//8rXZ9DqKEawhPoSytweBC2eNvcaDK+57RZLHGqNs3PZTQO6yekLFSuvA6AlnAfrw1uBvtb+Q==} hasBin: true @@ -1860,6 +2048,14 @@ packages: resolution: {integrity: sha512-4gd7VpWNQNB4UKKCFFVcp1AVv+FMOgs9NKzjHKusc8jTMhd5eL1NqQqOpE0KzMds804/yHlglp3uxgluOqAPLw==} engines: {node: '>= 0.4'} + ip-address@10.2.0: + resolution: {integrity: sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==} + engines: {node: '>= 12'} + + ipaddr.js@1.9.1: + resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} + engines: {node: '>= 0.10'} + is-array-buffer@3.0.5: resolution: {integrity: sha512-DDfANUiiG2wC1qawP66qlTugJeL5HyzMpfr8lLK+jMQirGzNod0B12cFB/9q838Ru27sBwfw78/rdoU7RERz6A==} engines: {node: '>= 0.4'} @@ -2023,6 +2219,9 @@ packages: resolution: {integrity: sha512-H0dkQoCa3b2VEeKQBOxFph+JAbcrQdE7KC0UkqwpLmv2EC4P41QXP+rqo9wYodACiG5/WM5s9oDApTU8utwj9g==} engines: {node: '>= 0.4'} + jose@6.2.3: + resolution: {integrity: sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==} + js-tokens@10.0.0: resolution: {integrity: sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==} @@ -2056,9 +2255,19 @@ packages: json-parse-even-better-errors@2.3.1: resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} + json-schema-to-ts@3.1.1: + resolution: {integrity: sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==} + engines: {node: '>=16'} + json-schema-traverse@0.4.1: resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} + json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + + json-schema-typed@8.0.2: + resolution: {integrity: sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==} + json-stable-stringify-without-jsonify@1.0.1: resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} @@ -2157,6 +2366,10 @@ packages: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} + media-typer@1.1.0: + resolution: {integrity: sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==} + engines: {node: '>= 0.8'} + memfs@4.51.0: resolution: {integrity: sha512-4zngfkVM/GpIhC8YazOsM6E8hoB33NP0BCESPOA6z7qaL6umPJNqkO8CNYaLV2FB2MV6H1O3x2luHHOSqppv+A==} @@ -2164,6 +2377,10 @@ packages: resolution: {integrity: sha512-xcSBHD5Z86zaOc+781KrupuHAzeGXSLtiAOmBsiLDiPSaYSB6hdew2ng9EBAnZ62jagG9MHAOdxpDi/lWBFJ/A==} engines: {node: '>=4'} + merge-descriptors@2.0.0: + resolution: {integrity: sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==} + engines: {node: '>=18'} + merge2@1.4.1: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} @@ -2185,10 +2402,18 @@ packages: resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} engines: {node: '>= 0.6'} + mime-db@1.54.0: + resolution: {integrity: sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==} + engines: {node: '>= 0.6'} + mime-types@2.1.35: resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} engines: {node: '>= 0.6'} + mime-types@3.0.2: + resolution: {integrity: sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==} + engines: {node: '>=18'} + minimatch@10.2.4: resolution: {integrity: sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==} engines: {node: 18 || 20 || >=22} @@ -2227,6 +2452,10 @@ packages: natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + negotiator@1.0.0: + resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==} + engines: {node: '>= 0.6'} + no-case@3.0.4: resolution: {integrity: sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==} @@ -2289,6 +2518,10 @@ packages: obug@2.1.1: resolution: {integrity: sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==} + on-finished@2.4.1: + resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} + engines: {node: '>= 0.8'} + once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} @@ -2357,6 +2590,10 @@ packages: resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} engines: {node: '>=8'} + parseurl@1.3.3: + resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} + engines: {node: '>= 0.8'} + pascal-case@3.1.2: resolution: {integrity: sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g==} @@ -2379,6 +2616,9 @@ packages: path-parse@1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} + path-to-regexp@8.4.2: + resolution: {integrity: sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==} + path-type@3.0.0: resolution: {integrity: sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==} engines: {node: '>=4'} @@ -2413,6 +2653,10 @@ packages: resolution: {integrity: sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==} engines: {node: '>= 6'} + pkce-challenge@5.0.1: + resolution: {integrity: sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==} + engines: {node: '>=16.20.0'} + possible-typed-array-names@1.1.0: resolution: {integrity: sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==} engines: {node: '>= 0.4'} @@ -2442,10 +2686,18 @@ packages: prop-types@15.8.1: resolution: {integrity: sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==} + proxy-addr@2.0.7: + resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} + engines: {node: '>= 0.10'} + punycode@2.1.1: resolution: {integrity: sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==} engines: {node: '>=6'} + qs@6.15.3: + resolution: {integrity: sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==} + engines: {node: '>=0.6'} + quansync@0.2.11: resolution: {integrity: sha512-AifT7QEbW9Nri4tAwR5M/uzpBuqfZf+zwaEM/QkzEjj7NBuFD2rBuy0K3dE+8wltbezDV7JMA0WfnCPYRSYbXA==} @@ -2456,6 +2708,14 @@ packages: resolution: {integrity: sha512-tRS7sTgyxMXtLum8L65daJnHUhfDUgboRdcWW2bR9vBfrj2+O5HSMbQOJfJJjIVSPFqbBCF37FpwWXGitDc5tA==} engines: {node: '>=4'} + range-parser@1.3.0: + resolution: {integrity: sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==} + engines: {node: '>= 0.6'} + + raw-body@3.0.2: + resolution: {integrity: sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==} + engines: {node: '>= 0.10'} + react-is@16.13.1: resolution: {integrity: sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==} @@ -2497,6 +2757,10 @@ packages: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} + require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + require-main-filename@2.0.0: resolution: {integrity: sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==} @@ -2536,6 +2800,10 @@ packages: engines: {node: '>=18.0.0', npm: '>=8.0.0'} hasBin: true + router@2.2.0: + resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==} + engines: {node: '>= 18'} + run-parallel@1.2.0: resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} @@ -2567,6 +2835,14 @@ packages: engines: {node: '>=10'} hasBin: true + send@1.2.1: + resolution: {integrity: sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==} + engines: {node: '>= 18'} + + serve-static@2.2.1: + resolution: {integrity: sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==} + engines: {node: '>= 18'} + set-blocking@2.0.0: resolution: {integrity: sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==} @@ -2582,6 +2858,9 @@ packages: resolution: {integrity: sha512-RJRdvCo6IAnPdsvP/7m6bsQqNnn1FCBX5ZNtFL98MmFF/4xAIJTIg1YbHW5DC2W5SKZanrC6i4HsJqlajw/dZw==} engines: {node: '>= 0.4'} + setprototypeof@1.2.0: + resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + shebang-command@2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} engines: {node: '>=8'} @@ -2594,6 +2873,10 @@ packages: resolution: {integrity: sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==} engines: {node: '>= 0.4'} + side-channel-list@1.0.1: + resolution: {integrity: sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==} + engines: {node: '>= 0.4'} + side-channel-map@1.0.1: resolution: {integrity: sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==} engines: {node: '>= 0.4'} @@ -2606,6 +2889,10 @@ packages: resolution: {integrity: sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==} engines: {node: '>= 0.4'} + side-channel@1.1.1: + resolution: {integrity: sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==} + engines: {node: '>= 0.4'} + siginfo@2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} @@ -2652,6 +2939,13 @@ packages: stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + standardwebhooks@1.0.0: + resolution: {integrity: sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==} + + statuses@2.0.2: + resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} + engines: {node: '>= 0.8'} + std-env@3.10.0: resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} @@ -2765,6 +3059,10 @@ packages: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} engines: {node: '>=8.0'} + toidentifier@1.0.1: + resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} + engines: {node: '>=0.6'} + tr46@0.0.3: resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} @@ -2778,6 +3076,9 @@ packages: resolution: {integrity: sha512-MTBWv3jhVjTU7XR3IQHllbiJs8sc75a80OEhB6or/q7pLTWgQ0bMGQXXYQSrSuXe6WiKWDZ5txXY5P59a/coVA==} engines: {node: '>=4'} + ts-algebra@2.0.0: + resolution: {integrity: sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==} + ts-api-utils@2.5.0: resolution: {integrity: sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==} engines: {node: '>=18.12'} @@ -2814,6 +3115,10 @@ packages: resolution: {integrity: sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==} engines: {node: '>=10'} + type-is@2.1.0: + resolution: {integrity: sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==} + engines: {node: '>= 18'} + typed-array-buffer@1.0.3: resolution: {integrity: sha512-nAYYwfY3qnzX30IkA6AQZjVbtK6duGontcQm1WSG1MD94YLqK0515GNApXkoxKOWMusVssAHWLh9SeaoefYFGw==} engines: {node: '>= 0.4'} @@ -2850,6 +3155,10 @@ packages: resolution: {integrity: sha512-6bc58dPYhCMHHuwxldQxO3RRNZ4eCogZ/st++0+fcC1nr0jiGUtAdBJ2qzmLQWSxbtz42pWt4QQMiZ9HvZf5cg==} engines: {node: '>=0.10.0'} + unpipe@1.0.0: + resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} + engines: {node: '>= 0.8'} + uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} @@ -2866,6 +3175,10 @@ packages: resolution: {integrity: sha512-9r0wQsWD8z/BxPOvnwbPf05ZvFngXyouE9EKB+5GbYix+BYnAwrIChCUyFIinfbf2FL/U71z+CPpbnmTdxrwBg==} engines: {node: '>=12'} + vary@1.1.2: + resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} + engines: {node: '>= 0.8'} + vite@7.2.2: resolution: {integrity: sha512-BxAKBWmIbrDgrokdGZH1IgkIk/5mMHDreLDmCJ0qpyJaAteP8NvMhkwr/ZCQNqNH97bw/dANTE9PDzqwJghfMQ==} engines: {node: ^20.19.0 || >=22.12.0} @@ -3026,8 +3339,62 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} + zod-to-json-schema@3.25.2: + resolution: {integrity: sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==} + peerDependencies: + zod: ^3.25.28 || ^4 + + zod@4.4.3: + resolution: {integrity: sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==} + snapshots: + '@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.205': + optional: true + + '@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.205': + optional: true + + '@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.205': + optional: true + + '@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.205': + optional: true + + '@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.205': + optional: true + + '@anthropic-ai/claude-agent-sdk-linux-x64@0.3.205': + optional: true + + '@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.205': + optional: true + + '@anthropic-ai/claude-agent-sdk-win32-x64@0.3.205': + optional: true + + '@anthropic-ai/claude-agent-sdk@0.3.205(@anthropic-ai/sdk@0.110.0(zod@4.4.3))(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(zod@4.4.3)': + dependencies: + '@anthropic-ai/sdk': 0.110.0(zod@4.4.3) + '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) + zod: 4.4.3 + optionalDependencies: + '@anthropic-ai/claude-agent-sdk-darwin-arm64': 0.3.205 + '@anthropic-ai/claude-agent-sdk-darwin-x64': 0.3.205 + '@anthropic-ai/claude-agent-sdk-linux-arm64': 0.3.205 + '@anthropic-ai/claude-agent-sdk-linux-arm64-musl': 0.3.205 + '@anthropic-ai/claude-agent-sdk-linux-x64': 0.3.205 + '@anthropic-ai/claude-agent-sdk-linux-x64-musl': 0.3.205 + '@anthropic-ai/claude-agent-sdk-win32-arm64': 0.3.205 + '@anthropic-ai/claude-agent-sdk-win32-x64': 0.3.205 + + '@anthropic-ai/sdk@0.110.0(zod@4.4.3)': + dependencies: + json-schema-to-ts: 3.1.1 + standardwebhooks: 1.0.0 + optionalDependencies: + zod: 4.4.3 + '@ardatan/aggregate-error@0.0.6': dependencies: tslib: 2.0.3 @@ -3469,6 +3836,10 @@ snapshots: dependencies: graphql: 15.10.1 + '@hono/node-server@1.19.14(hono@4.12.28)': + dependencies: + hono: 4.12.28 + '@humanwhocodes/config-array@0.9.5': dependencies: '@humanwhocodes/object-schema': 1.2.1 @@ -3572,6 +3943,28 @@ snapshots: '@microsoft/fetch-event-source@2.0.1': {} + '@modelcontextprotocol/sdk@1.29.0(zod@4.4.3)': + dependencies: + '@hono/node-server': 1.19.14(hono@4.12.28) + ajv: 8.20.0 + ajv-formats: 3.0.1(ajv@8.20.0) + content-type: 1.0.5 + cors: 2.8.6 + cross-spawn: 7.0.6 + eventsource: 3.0.7 + eventsource-parser: 3.1.0 + express: 5.2.1 + express-rate-limit: 8.5.2(express@5.2.1) + hono: 4.12.28 + jose: 6.2.3 + json-schema-typed: 8.0.2 + pkce-challenge: 5.0.1 + raw-body: 3.0.2 + zod: 4.4.3 + zod-to-json-schema: 3.25.2(zod@4.4.3) + transitivePeerDependencies: + - supports-color + '@napi-rs/wasm-runtime@1.1.1': dependencies: '@emnapi/core': 1.8.1 @@ -3721,6 +4114,8 @@ snapshots: '@rtsao/scc@1.1.0': {} + '@stablelib/base64@1.0.1': {} + '@standard-schema/spec@1.0.0': {} '@swc-node/core@1.14.1(@swc/core@1.15.18)(@swc/types@0.1.25)': @@ -3950,12 +4345,21 @@ snapshots: dependencies: event-target-shim: 5.0.1 + accepts@2.0.0: + dependencies: + mime-types: 3.0.2 + negotiator: 1.0.0 + acorn-jsx@5.3.2(acorn@8.7.1): dependencies: acorn: 8.7.1 acorn@8.7.1: {} + ajv-formats@3.0.1(ajv@8.20.0): + optionalDependencies: + ajv: 8.20.0 + ajv@6.12.6: dependencies: fast-deep-equal: 3.1.3 @@ -3963,6 +4367,13 @@ snapshots: json-schema-traverse: 0.4.1 uri-js: 4.4.1 + ajv@8.20.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-uri: 3.1.3 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + ansi-colors@4.1.3: {} ansi-regex@5.0.1: {} @@ -4094,6 +4505,20 @@ snapshots: dependencies: is-windows: 1.0.2 + body-parser@2.3.0: + dependencies: + bytes: 3.1.2 + content-type: 2.0.0 + debug: 4.4.3 + http-errors: 2.0.1 + iconv-lite: 0.7.2 + on-finished: 2.4.1 + qs: 6.15.3 + raw-body: 3.0.2 + type-is: 2.1.0 + transitivePeerDependencies: + - supports-color + brace-expansion@1.1.11: dependencies: balanced-match: 1.0.2 @@ -4119,6 +4544,8 @@ snapshots: base64-js: 1.5.1 ieee754: 1.2.1 + bytes@3.1.2: {} + call-bind-apply-helpers@1.0.2: dependencies: es-errors: 1.3.0 @@ -4196,6 +4623,21 @@ snapshots: concat-map@0.0.1: {} + content-disposition@1.1.0: {} + + content-type@1.0.5: {} + + content-type@2.0.0: {} + + cookie-signature@1.2.2: {} + + cookie@0.7.2: {} + + cors@2.8.6: + dependencies: + object-assign: 4.1.1 + vary: 1.1.2 + cosmiconfig-toml-loader@1.0.0: dependencies: '@iarna/toml': 2.2.5 @@ -4287,6 +4729,8 @@ snapshots: delayed-stream@1.0.0: {} + depd@2.0.0: {} + detect-indent@6.1.0: {} diff@4.0.4: {} @@ -4309,10 +4753,14 @@ snapshots: es-errors: 1.3.0 gopd: 1.2.0 + ee-first@1.1.1: {} + emoji-regex@8.0.0: {} emoji-regex@9.2.2: {} + encodeurl@2.0.0: {} + enquirer@2.4.1: dependencies: ansi-colors: 4.1.3 @@ -4458,6 +4906,8 @@ snapshots: '@esbuild/win32-ia32': 0.25.12 '@esbuild/win32-x64': 0.25.12 + escape-html@1.0.3: {} + escape-string-regexp@1.0.5: {} escape-string-regexp@4.0.0: {} @@ -4693,12 +5143,58 @@ snapshots: esutils@2.0.3: {} + etag@1.8.1: {} + event-target-shim@5.0.1: {} eventemitter3@3.1.2: {} + eventsource-parser@3.1.0: {} + + eventsource@3.0.7: + dependencies: + eventsource-parser: 3.1.0 + expect-type@1.2.2: {} + express-rate-limit@8.5.2(express@5.2.1): + dependencies: + express: 5.2.1 + ip-address: 10.2.0 + + express@5.2.1: + dependencies: + accepts: 2.0.0 + body-parser: 2.3.0 + content-disposition: 1.1.0 + content-type: 1.0.5 + cookie: 0.7.2 + cookie-signature: 1.2.2 + debug: 4.4.3 + depd: 2.0.0 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + finalhandler: 2.1.1 + fresh: 2.0.0 + http-errors: 2.0.1 + merge-descriptors: 2.0.0 + mime-types: 3.0.2 + on-finished: 2.4.1 + once: 1.4.0 + parseurl: 1.3.3 + proxy-addr: 2.0.7 + qs: 6.15.3 + range-parser: 1.3.0 + router: 2.2.0 + send: 1.2.1 + serve-static: 2.2.1 + statuses: 2.0.2 + type-is: 2.1.0 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + extendable-error@0.1.7: {} extract-files@9.0.0: {} @@ -4719,6 +5215,10 @@ snapshots: fast-levenshtein@2.0.6: {} + fast-sha256@1.3.0: {} + + fast-uri@3.1.3: {} + fastq@1.13.0: dependencies: reusify: 1.0.4 @@ -4740,6 +5240,17 @@ snapshots: async: 0.9.2 is-directory: 0.2.3 + finalhandler@2.1.1: + dependencies: + debug: 4.4.3 + encodeurl: 2.0.0 + escape-html: 1.0.3 + on-finished: 2.4.1 + parseurl: 1.3.3 + statuses: 2.0.2 + transitivePeerDependencies: + - supports-color + find-up@2.1.0: dependencies: locate-path: 2.0.0 @@ -4766,6 +5277,10 @@ snapshots: combined-stream: 1.0.8 mime-types: 2.1.35 + forwarded@0.2.0: {} + + fresh@2.0.0: {} + fs-extra@7.0.1: dependencies: graceful-fs: 4.2.10 @@ -4940,10 +5455,20 @@ snapshots: dependencies: function-bind: 1.1.2 + hono@4.12.28: {} + hosted-git-info@2.8.9: {} html-escaper@2.0.2: {} + http-errors@2.0.1: + dependencies: + depd: 2.0.0 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 2.0.2 + toidentifier: 1.0.1 + human-id@4.1.3: {} hyperdyperid@1.2.0: {} @@ -4989,6 +5514,10 @@ snapshots: hasown: 2.0.2 side-channel: 1.1.0 + ip-address@10.2.0: {} + + ipaddr.js@1.9.1: {} + is-array-buffer@3.0.5: dependencies: call-bind: 1.0.8 @@ -5153,6 +5682,8 @@ snapshots: has-symbols: 1.1.0 set-function-name: 2.0.2 + jose@6.2.3: {} + js-tokens@10.0.0: {} js-tokens@4.0.0: {} @@ -5180,8 +5711,17 @@ snapshots: json-parse-even-better-errors@2.3.1: {} + json-schema-to-ts@3.1.1: + dependencies: + '@babel/runtime': 7.28.6 + ts-algebra: 2.0.0 + json-schema-traverse@0.4.1: {} + json-schema-traverse@1.0.0: {} + + json-schema-typed@8.0.2: {} + json-stable-stringify-without-jsonify@1.0.1: {} json5@1.0.2: @@ -5277,6 +5817,8 @@ snapshots: math-intrinsics@1.1.0: {} + media-typer@1.1.0: {} + memfs@4.51.0: dependencies: '@jsonjoy.com/json-pack': 1.21.0(tslib@2.8.1) @@ -5298,6 +5840,8 @@ snapshots: redent: 2.0.0 trim-newlines: 2.0.0 + merge-descriptors@2.0.0: {} + merge2@1.4.1: {} meros@1.1.4(@types/node@25.3.3): @@ -5311,10 +5855,16 @@ snapshots: mime-db@1.52.0: {} + mime-db@1.54.0: {} + mime-types@2.1.35: dependencies: mime-db: 1.52.0 + mime-types@3.0.2: + dependencies: + mime-db: 1.54.0 + minimatch@10.2.4: dependencies: brace-expansion: 5.0.5 @@ -5348,6 +5898,8 @@ snapshots: natural-compare@1.4.0: {} + negotiator@1.0.0: {} + no-case@3.0.4: dependencies: lower-case: 2.0.2 @@ -5421,6 +5973,10 @@ snapshots: obug@2.1.1: {} + on-finished@2.4.1: + dependencies: + ee-first: 1.1.1 + once@1.4.0: dependencies: wrappy: 1.0.2 @@ -5515,6 +6071,8 @@ snapshots: json-parse-even-better-errors: 2.3.1 lines-and-columns: 1.2.4 + parseurl@1.3.3: {} + pascal-case@3.1.2: dependencies: no-case: 3.0.4 @@ -5530,6 +6088,8 @@ snapshots: path-parse@1.0.7: {} + path-to-regexp@8.4.2: {} + path-type@3.0.0: dependencies: pify: 3.0.0 @@ -5550,6 +6110,8 @@ snapshots: pirates@4.0.7: {} + pkce-challenge@5.0.1: {} + possible-typed-array-names@1.1.0: {} postcss@8.5.6: @@ -5574,14 +6136,33 @@ snapshots: object-assign: 4.1.1 react-is: 16.13.1 + proxy-addr@2.0.7: + dependencies: + forwarded: 0.2.0 + ipaddr.js: 1.9.1 + punycode@2.1.1: {} + qs@6.15.3: + dependencies: + es-define-property: 1.0.1 + side-channel: 1.1.1 + quansync@0.2.11: {} queue-microtask@1.2.3: {} quick-lru@1.1.0: {} + range-parser@1.3.0: {} + + raw-body@3.0.2: + dependencies: + bytes: 3.1.2 + http-errors: 2.0.1 + iconv-lite: 0.7.2 + unpipe: 1.0.0 + react-is@16.13.1: {} read-pkg-up@3.0.0: @@ -5635,6 +6216,8 @@ snapshots: require-directory@2.1.1: {} + require-from-string@2.0.2: {} + require-main-filename@2.0.0: {} resolve-from@4.0.0: {} @@ -5696,6 +6279,16 @@ snapshots: '@rollup/rollup-win32-x64-msvc': 4.53.2 fsevents: 2.3.3 + router@2.2.0: + dependencies: + debug: 4.4.3 + depd: 2.0.0 + is-promise: 4.0.0 + parseurl: 1.3.3 + path-to-regexp: 8.4.2 + transitivePeerDependencies: + - supports-color + run-parallel@1.2.0: dependencies: queue-microtask: 1.2.3 @@ -5727,6 +6320,31 @@ snapshots: semver@7.7.4: {} + send@1.2.1: + dependencies: + debug: 4.4.3 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + fresh: 2.0.0 + http-errors: 2.0.1 + mime-types: 3.0.2 + ms: 2.1.3 + on-finished: 2.4.1 + range-parser: 1.3.0 + statuses: 2.0.2 + transitivePeerDependencies: + - supports-color + + serve-static@2.2.1: + dependencies: + encodeurl: 2.0.0 + escape-html: 1.0.3 + parseurl: 1.3.3 + send: 1.2.1 + transitivePeerDependencies: + - supports-color + set-blocking@2.0.0: {} set-function-length@1.2.2: @@ -5751,6 +6369,8 @@ snapshots: es-errors: 1.3.0 es-object-atoms: 1.1.1 + setprototypeof@1.2.0: {} + shebang-command@2.0.0: dependencies: shebang-regex: 3.0.0 @@ -5762,6 +6382,11 @@ snapshots: es-errors: 1.3.0 object-inspect: 1.13.4 + side-channel-list@1.0.1: + dependencies: + es-errors: 1.3.0 + object-inspect: 1.13.4 + side-channel-map@1.0.1: dependencies: call-bound: 1.0.4 @@ -5785,6 +6410,14 @@ snapshots: side-channel-map: 1.0.1 side-channel-weakmap: 1.0.2 + side-channel@1.1.1: + dependencies: + es-errors: 1.3.0 + object-inspect: 1.13.4 + side-channel-list: 1.0.1 + side-channel-map: 1.0.1 + side-channel-weakmap: 1.0.2 + siginfo@2.0.0: {} signal-exit@3.0.7: {} @@ -5825,6 +6458,13 @@ snapshots: stackback@0.0.2: {} + standardwebhooks@1.0.0: + dependencies: + '@stablelib/base64': 1.0.1 + fast-sha256: 1.3.0 + + statuses@2.0.2: {} + std-env@3.10.0: {} stop-iteration-iterator@1.1.0: @@ -5956,6 +6596,8 @@ snapshots: dependencies: is-number: 7.0.0 + toidentifier@1.0.1: {} + tr46@0.0.3: {} tree-dump@1.1.0(tslib@2.8.1): @@ -5964,6 +6606,8 @@ snapshots: trim-newlines@2.0.0: {} + ts-algebra@2.0.0: {} + ts-api-utils@2.5.0(typescript@5.9.3): dependencies: typescript: 5.9.3 @@ -5999,6 +6643,12 @@ snapshots: type-fest@0.20.2: {} + type-is@2.1.0: + dependencies: + content-type: 2.0.0 + media-typer: 1.1.0 + mime-types: 3.0.2 + typed-array-buffer@1.0.3: dependencies: call-bound: 1.0.4 @@ -6049,6 +6699,8 @@ snapshots: dependencies: normalize-path: 2.1.1 + unpipe@1.0.0: {} + uri-js@4.4.1: dependencies: punycode: 2.1.1 @@ -6064,6 +6716,8 @@ snapshots: value-or-promise@1.0.6: {} + vary@1.1.2: {} + vite@7.2.2(@types/node@25.3.3)(yaml@2.8.3): dependencies: esbuild: 0.25.12 @@ -6214,3 +6868,9 @@ snapshots: yn@3.1.1: {} yocto-queue@0.1.0: {} + + zod-to-json-schema@3.25.2(zod@4.4.3): + dependencies: + zod: 4.4.3 + + zod@4.4.3: {} diff --git a/workflows/review/eval/agent-extract.test.ts b/workflows/review/eval/agent-extract.test.ts new file mode 100644 index 00000000..56102827 --- /dev/null +++ b/workflows/review/eval/agent-extract.test.ts @@ -0,0 +1,130 @@ +import {readFileSync} from "node:fs"; + +import {describe, it, expect} from "vitest"; + +import {AgentExtractError, extractAgents} from "./agent-extract"; +import {PRODUCTION_REVIEW_DIR} from "./live-stage"; + +/** Build a well-formed agent section. */ +const section = ( + name: string, + body = "Review the diff and return JSON.", + frontmatter?: string, +): string => + [ + `## agent: \`${name}\``, + "---", + frontmatter ?? + [ + `name: ${name}`, + `description: A ${name} test agent.`, + "model: claude-opus-4-8", + "# effort: high (launch default)", + ].join("\n"), + "---", + body, + "", + ].join("\n"); + +describe("extractAgents: fixture markdown", () => { + it("extracts sections in order with frontmatter fields and body", () => { + const md = [ + "# Workflow prose the extractor must skip", + "", + section("alpha", "Alpha prompt body.\n\nWith two paragraphs."), + section("beta"), + ].join("\n"); + const agents = extractAgents(md); + expect([...agents.keys()]).toEqual(["alpha", "beta"]); + const alpha = agents.get("alpha"); + expect(alpha?.model).toBe("claude-opus-4-8"); + expect(alpha?.description).toBe("A alpha test agent."); + expect(alpha?.prompt).toBe( + "Alpha prompt body.\n\nWith two paragraphs.", + ); + }); + + it("ignores comment lines in frontmatter (the effort annotations)", () => { + const agents = extractAgents(section("gamma")); + expect(agents.get("gamma")?.model).toBe("claude-opus-4-8"); + }); + + it("throws listing every problem at once", () => { + const md = [ + section("good"), + // name mismatch + section( + "bad-name", + "Body.", + ["name: other", "description: d", "model: m"].join("\n"), + ), + // missing model + section( + "no-model", + "Body.", + ["name: no-model", "description: d"].join("\n"), + ), + ].join("\n"); + let message = ""; + try { + extractAgents(md); + } catch (error) { + expect(error).toBeInstanceOf(AgentExtractError); + message = (error as Error).message; + } + expect(message).toMatch(/bad-name.*does not match heading/); + expect(message).toMatch(/no-model.*missing frontmatter model/); + }); + + it("throws on an unterminated frontmatter block", () => { + const md = ["## agent: `broken`", "---", "name: broken", ""].join("\n"); + expect(() => extractAgents(md)).toThrow(/unterminated frontmatter/); + }); + + it("throws on duplicate agent names", () => { + expect(() => extractAgents(section("dup") + section("dup"))).toThrow( + /duplicate agent name/, + ); + }); + + it("throws when no agent sections exist", () => { + expect(() => extractAgents("# just prose")).toThrow( + /no `## agent:` sections/, + ); + }); +}); + +describe("extractAgents: the real review.md", () => { + const markdown = readFileSync("workflows/review/review.md", "utf8"); + const agents = extractAgents(markdown); + + it("extracts every `## agent:` section in the file", () => { + const headings = markdown + .split("\n") + .filter((line) => /^## agent: `[^`]+`\s*$/.test(line)); + expect(agents.size).toBe(headings.length); + expect(agents.size).toBeGreaterThanOrEqual(21); + }); + + it("pins a model on every agent", () => { + for (const agent of agents.values()) { + expect(agent.model).toMatch(/^claude-/); + } + }); + + it("the default-roster reviewers reference the production staging root", () => { + // The staging-path rewrite (live-stage.ts) depends on prompts naming + // PRODUCTION_REVIEW_DIR verbatim; if review.md renames the staging + // root, this fails here rather than silently staging nothing. + for (const name of [ + "correctness-reviewer", + "skill-auditor", + "claim-validator", + ]) { + expect( + agents.get(name)?.prompt, + `${name} missing ${PRODUCTION_REVIEW_DIR}`, + ).toContain(PRODUCTION_REVIEW_DIR); + } + }); +}); diff --git a/workflows/review/eval/agent-extract.ts b/workflows/review/eval/agent-extract.ts new file mode 100644 index 00000000..45598b7b --- /dev/null +++ b/workflows/review/eval/agent-extract.ts @@ -0,0 +1,162 @@ +/** + * Sub-agent prompt extraction for the live eval arm (`live-ab-plan.md` + * Phase 2a). + * + * `review.md` defines every reviewer sub-agent as a `## agent: \`name\`` + * section: YAML-ish frontmatter (`name`, `description`, `model`, plus + * comment lines) followed by the prompt body. The live producer needs those + * prompts as data, for BOTH arms of an A/B: the candidate arm reads the + * working tree's `review.md`, the baseline arm reads the merge-base version + * via `git show`. So the core function takes the markdown as a string and + * performs no filesystem access. + * + * Parsing is deliberately strict: a section whose frontmatter is missing, + * unterminated, name-mismatched, or model-less throws rather than degrading, + * because a silently dropped agent would skew an eval arm without failing it. + */ + +/** One extracted sub-agent definition. */ +export type ExtractedAgent = { + /** The agent name (heading and frontmatter must agree). */ + name: string; + /** The frontmatter description line. */ + description: string; + /** The pinned model id (e.g. `claude-opus-4-8`). */ + model: string; + /** The full prompt body (everything after the frontmatter). */ + prompt: string; +}; + +/** Thrown when `review.md`'s agent sections cannot be parsed. */ +export class AgentExtractError extends Error { + constructor(errors: string[]) { + super( + `Malformed agent section(s) in review markdown:\n${errors + .map((e) => ` - ${e}`) + .join("\n")}`, + ); + this.name = "AgentExtractError"; + } +} + +const HEADING = /^## agent: `([^`]+)`\s*$/; + +/** Parse one frontmatter block's `key: value` lines (comments ignored). */ +const parseFrontmatter = (lines: string[]): Map => { + const out = new Map(); + for (const line of lines) { + if (line.trim() === "" || line.trim().startsWith("#")) { + continue; + } + const colon = line.indexOf(":"); + if (colon === -1) { + // A wrapped comment continuation or stray text; skip rather than + // guess. Required keys are checked by the caller. + continue; + } + const key = line.slice(0, colon).trim(); + const value = line.slice(colon + 1).trim(); + if (key.length > 0 && !out.has(key)) { + out.set(key, value); + } + } + return out; +}; + +/** + * Extract every `## agent:` section from review markdown. Returns agents in + * definition order (a Map preserves insertion order). Throws + * {@link AgentExtractError} listing every problem at once. + */ +export const extractAgents = ( + markdown: string, +): Map => { + const lines = markdown.split("\n"); + const agents = new Map(); + const errors: string[] = []; + + /** Indices of every agent heading, plus a sentinel end. */ + const headings: {index: number; name: string}[] = []; + lines.forEach((line, index) => { + const match = HEADING.exec(line); + if (match?.[1] !== undefined) { + headings.push({index, name: match[1]}); + } + }); + + headings.forEach((heading, i) => { + const sectionEnd = headings[i + 1]?.index ?? lines.length; + const at = `agent \`${heading.name}\` (line ${heading.index + 1})`; + + // Frontmatter: first non-blank line after the heading must open it. + let cursor = heading.index + 1; + while (cursor < sectionEnd && (lines[cursor] ?? "").trim() === "") { + cursor++; + } + if ((lines[cursor] ?? "").trim() !== "---") { + errors.push(`${at}: expected frontmatter opening ---`); + return; + } + const fmStart = cursor + 1; + let fmEnd = -1; + for (let j = fmStart; j < sectionEnd; j++) { + if ((lines[j] ?? "").trim() === "---") { + fmEnd = j; + break; + } + } + if (fmEnd === -1) { + errors.push(`${at}: unterminated frontmatter`); + return; + } + + const frontmatter = parseFrontmatter(lines.slice(fmStart, fmEnd)); + const name = frontmatter.get("name"); + const description = frontmatter.get("description"); + const model = frontmatter.get("model"); + if (name !== heading.name) { + errors.push( + `${at}: frontmatter name "${ + name ?? "" + }" does not match heading`, + ); + } + if (description === undefined || description === "") { + errors.push(`${at}: missing frontmatter description`); + } + if (model === undefined || model === "") { + errors.push(`${at}: missing frontmatter model`); + } + if (agents.has(heading.name)) { + errors.push(`${at}: duplicate agent name`); + } + if ( + name !== heading.name || + description === undefined || + description === "" || + model === undefined || + model === "" || + agents.has(heading.name) + ) { + return; + } + + const prompt = lines + .slice(fmEnd + 1, sectionEnd) + .join("\n") + .trim(); + if (prompt === "") { + errors.push(`${at}: empty prompt body`); + return; + } + agents.set(heading.name, {name, description, model, prompt}); + }); + + if (errors.length > 0) { + throw new AgentExtractError(errors); + } + if (agents.size === 0) { + throw new AgentExtractError(["no `## agent:` sections found"]); + } + return agents; +}; diff --git a/workflows/review/eval/live-producer.test.ts b/workflows/review/eval/live-producer.test.ts new file mode 100644 index 00000000..0b918e2c --- /dev/null +++ b/workflows/review/eval/live-producer.test.ts @@ -0,0 +1,410 @@ +import {describe, it, expect} from "vitest"; +import {Volume} from "memfs"; + +import {parseCase} from "./corpus/loader"; +import { + produceLive, + resolveRuntimeImports, + type LiveAgentRequest, + type LiveAgentRunner, +} from "./live-producer"; +import type {ExtractedAgent} from "./agent-extract"; +import type {StageFs} from "./live-stage"; + +/** Adapt a memfs volume to the staging fs seam. */ +const volFs = (vol: InstanceType): StageFs => ({ + existsSync: (p) => vol.existsSync(p), + mkdirSync: (p, opts) => { + vol.mkdirSync(p, opts); + }, + readdirSync: (p, opts) => + vol.readdirSync(p, opts) as unknown as ReturnType< + StageFs["readdirSync"] + >, + readFileSync: (p, enc) => vol.readFileSync(p, enc) as string, + writeFileSync: (p, data) => { + vol.writeFileSync(p, data); + }, +}); + +const DIFF = [ + "diff --git a/src/a.ts b/src/a.ts", + "--- a/src/a.ts", + "+++ b/src/a.ts", + "@@ -1,2 +1,2 @@", + "-const a = 1;", + "+const a = 2;", + " export {a};", + "", +].join("\n"); + +const CASE = parseCase( + { + id: "produce-case", + tags: ["live"], + category: "incident-repro", + description: "a producible case", + changedFiles: [{path: "src/a.ts", status: "modified"}], + expected: {verdict: "APPROVE"}, + diff: DIFF, + routerConfig: { + lensRules: [{pattern: "src/**", lenses: ["money-payments"]}], + }, + live: { + prContext: { + title: "t", + description: "", + author: "a", + baseBranch: "main", + }, + }, + }, + "/corpus/incidents/produce-case/case.json", +); + +const caseVol = () => + Volume.fromJSON({ + "/corpus/incidents/produce-case/tree/src/a.ts": + "const a = 2;\nexport {a};\n", + }); + +const agent = (name: string, prompt = `${name} prompt`): ExtractedAgent => ({ + name, + description: `${name} description`, + model: "claude-opus-4-8", + prompt, +}); + +const AGENTS = new Map( + [ + "correctness-reviewer", + "skill-auditor", + "money-payments", + "claim-validator", + ].map((name) => [name, agent(name)]), +); + +const LABEL_FINDING = { + path: "src/a.ts", + line: 1, + label: "issue (blocking)", + failure_scenario: "with input X the constant is wrong and Y crashes.", + subject: "Constant changed incorrectly", + discussion: "The new value breaks the Y invariant.", +}; + +const SCHEMA_FINDING = { + schema_version: 2, + id: "lens-money-1", + lens: "money-payments", + anchor: {type: "line", path: "src/a.ts", line: 1, side: "RIGHT"}, + severity: "advisory", + confidence: 0.6, + evidence_trace: ["read src/a.ts line 1"], + failure_scenario: "amounts drift by a cent on large values.", + producing_hunt: "money:rounding", + model_authored_prose: "Money should stay in integer cents.", +}; + +/** A scripted runner: outputs queued per agent name, requests recorded. */ +const scriptedRunner = ( + scripts: Record, +): {runner: LiveAgentRunner; requests: LiveAgentRequest[]} => { + const requests: LiveAgentRequest[] = []; + const cursors: Record = {}; + const runner: LiveAgentRunner = async (request) => { + requests.push(request); + const queue = scripts[request.name] ?? []; + const cursor = cursors[request.name] ?? 0; + cursors[request.name] = cursor + 1; + const output = queue[Math.min(cursor, queue.length - 1)] ?? "{}"; + return {output, usd: 0.25, turns: 3, wallMs: 1000}; + }; + return {runner, requests}; +}; + +const validatorOutput = ( + entries: {id: string; verification: string; confidence?: number}[], +): string => + JSON.stringify({ + claims: entries.map((entry) => ({...entry, reason: "checked"})), + }); + +describe("produceLive", () => { + it("runs the default finders plus routed lenses and the validator", async () => { + const {runner, requests} = scriptedRunner({ + "correctness-reviewer": [ + JSON.stringify({files: [], findings: [LABEL_FINDING]}), + ], + "skill-auditor": [JSON.stringify({findings: []})], + "money-payments": [ + JSON.stringify({findings: [SCHEMA_FINDING], hunts: []}), + ], + "claim-validator": [ + validatorOutput([ + { + id: "produce-case:live-correctness-reviewer-1", + verification: "confirmed", + confidence: 0.9, + }, + { + id: "produce-case:lens-money-1", + verification: "plausible", + confidence: 0.3, + }, + ]), + ], + }); + const vol = caseVol(); + const result = await produceLive(CASE, AGENTS, { + runner, + stageDir: "/stage", + fs: volFs(vol), + }); + + expect(requests.map((r) => r.name).sort()).toEqual([ + "claim-validator", + "correctness-reviewer", + "money-payments", + "skill-auditor", + ]); + // Every dispatch runs in the staged checkout. + expect(new Set(requests.map((r) => r.cwd))).toEqual( + new Set(["/stage/checkout"]), + ); + + // The label-shape finding is mapped into the schema. + const correctness = result.findings.find( + (f) => f.source === "correctness", + ); + expect(correctness?.finding.id).toBe( + "produce-case:live-correctness-reviewer-1", + ); + expect(correctness?.finding.severity).toBe("blocking"); + expect(correctness?.finding.lens).toBe("correctness"); + expect(correctness?.finding.confidence).toBe(0.7); + // The lens finding passes through as-is. + const lens = result.findings.find((f) => f.source === "money-payments"); + expect(lens?.finding.id).toBe("produce-case:lens-money-1"); + + // claims.json staged for the validator with code-owned labels. + const claims = JSON.parse( + vol.readFileSync("/stage/context/claims.json", "utf8") as string, + ); + expect(claims.map((c: {id: string}) => c.id).sort()).toEqual( + [ + "produce-case:lens-money-1", + "produce-case:live-correctness-reviewer-1", + ].sort(), + ); + expect( + claims.find( + (c: {id: string}) => + c.id === "produce-case:live-correctness-reviewer-1", + ).label, + ).toBe("issue (blocking)"); + + // Verifications parsed into the corpus validation shape. + expect(result.validation).toEqual([ + { + id: "produce-case:live-correctness-reviewer-1", + verification: "confirmed", + confidence: 0.9, + }, + { + id: "produce-case:lens-money-1", + verification: "plausible", + confidence: 0.3, + }, + ]); + + // Cost accounting: one entry per dispatched agent. + expect(result.perAgent.length).toBe(4); + expect(result.perAgent.every((a) => a.usd === 0.25)).toBe(true); + }); + + it("retries once on malformed output and keeps the second answer", async () => { + const {runner, requests} = scriptedRunner({ + "correctness-reviewer": [ + "sorry, here is prose instead of JSON", + JSON.stringify({findings: [LABEL_FINDING]}), + ], + "skill-auditor": [JSON.stringify({findings: []})], + "money-payments": [JSON.stringify({findings: []})], + "claim-validator": [ + validatorOutput([ + { + id: "produce-case:live-correctness-reviewer-1", + verification: "confirmed", + }, + ]), + ], + }); + const result = await produceLive(CASE, AGENTS, { + runner, + stageDir: "/stage", + fs: volFs(caseVol()), + }); + const report = result.perAgent.find( + (a) => a.name === "correctness-reviewer", + ); + expect(report?.retried).toBe(true); + expect(report?.failed).toBeUndefined(); + expect(report?.usd).toBeCloseTo(0.5, 10); // both attempts billed + expect(result.findings.length).toBe(1); + // The retry prompt carries the rejection reason. + const retryPrompt = requests.filter( + (r) => r.name === "correctness-reviewer", + )[1]?.prompt; + expect(retryPrompt).toMatch(/previous output was rejected/); + }); + + it("marks a twice-failed agent failed and keeps everyone else", async () => { + const {runner} = scriptedRunner({ + "correctness-reviewer": ["not json", "still not json"], + "skill-auditor": [ + JSON.stringify({ + findings: [ + { + ...LABEL_FINDING, + skill: "error-handling", + label: "issue (blocking, best-practice)", + }, + ], + }), + ], + "money-payments": [JSON.stringify({findings: []})], + "claim-validator": [ + validatorOutput([ + { + id: "produce-case:live-skill-auditor-1", + verification: "refuted", + }, + ]), + ], + }); + const vol = caseVol(); + const result = await produceLive(CASE, AGENTS, { + runner, + stageDir: "/stage", + fs: volFs(vol), + }); + expect( + result.perAgent.find((a) => a.name === "correctness-reviewer") + ?.failed, + ).toMatch(/malformed output/); + const skill = result.findings.find((f) => f.source === "skill"); + expect(skill?.finding.lens).toBe("conventions"); + // The skill name rides into the claims for the validator. + const claims = JSON.parse( + vol.readFileSync("/stage/context/claims.json", "utf8") as string, + ); + expect(claims[0].skill).toBe("error-handling"); + expect(result.validation[0]?.verification).toBe("refuted"); + }); + + it("skips the validator entirely when no findings were produced", async () => { + const {runner, requests} = scriptedRunner({ + "correctness-reviewer": [JSON.stringify({findings: []})], + "skill-auditor": [JSON.stringify({findings: []})], + "money-payments": [JSON.stringify({findings: []})], + }); + const result = await produceLive(CASE, AGENTS, { + runner, + stageDir: "/stage", + fs: volFs(caseVol()), + }); + expect(result.findings).toEqual([]); + expect(result.validation).toEqual([]); + expect(requests.some((r) => r.name === "claim-validator")).toBe(false); + }); + + it("prefixes colliding finding ids with the producing agent", async () => { + const {runner} = scriptedRunner({ + "correctness-reviewer": [JSON.stringify({findings: []})], + "skill-auditor": [JSON.stringify({findings: []})], + "money-payments": [ + JSON.stringify({ + findings: [ + SCHEMA_FINDING, + {...SCHEMA_FINDING, id: "lens-money-1"}, + ], + }), + ], + "claim-validator": [validatorOutput([])], + }); + const result = await produceLive(CASE, AGENTS, { + runner, + stageDir: "/stage", + fs: volFs(caseVol()), + }); + expect(result.findings.map((f) => f.finding.id).sort()).toEqual([ + "money-payments:produce-case:lens-money-1", + "produce-case:lens-money-1", + ]); + }); + + it("throws when the roster names an agent review.md does not define", async () => { + const {runner} = scriptedRunner({}); + const agents = new Map([ + ["correctness-reviewer", agent("correctness-reviewer")], + ]); + await expect( + produceLive(CASE, agents, { + runner, + stageDir: "/stage", + fs: volFs(caseVol()), + }), + ).rejects.toThrow(/"skill-auditor" is not defined/); + }); +}); + +describe("resolveRuntimeImports", () => { + it("inlines imports present in the checkout and notes absent ones", () => { + const vol = Volume.fromJSON({ + "/checkout/.github/aw/review/skills.md": "## skills index", + }); + const fs = volFs(vol); + const prompt = [ + "Skills:\n{{#runtime-import .github/aw/review/skills.md}}", + "CI:\n{{#runtime-import? .github/aw/review/ci-tooling.md}}", + ].join("\n"); + const resolved = resolveRuntimeImports(prompt, "/checkout", fs); + expect(resolved).toContain("## skills index"); + expect(resolved).toContain("(not configured for this eval case)"); + expect(resolved).not.toMatch(/runtime-import/); + }); + + it("reaches the dispatched prompts through produceLive", async () => { + const {runner, requests} = scriptedRunner({ + "correctness-reviewer": [JSON.stringify({findings: []})], + "skill-auditor": [JSON.stringify({findings: []})], + "money-payments": [JSON.stringify({findings: []})], + }); + const vol = caseVol(); + vol.mkdirSync("/corpus/incidents/produce-case/tree/.github/aw/review", { + recursive: true, + }); + vol.writeFileSync( + "/corpus/incidents/produce-case/tree/.github/aw/review/skills.md", + "## the case skills index", + ); + const agents = new Map(AGENTS); + agents.set( + "skill-auditor", + agent( + "skill-auditor", + "Audit skills:\n{{#runtime-import .github/aw/review/skills.md}}", + ), + ); + await produceLive(CASE, agents, { + runner, + stageDir: "/stage", + fs: volFs(vol), + }); + const skillPrompt = requests.find( + (r) => r.name === "skill-auditor", + )?.prompt; + expect(skillPrompt).toContain("## the case skills index"); + }); +}); diff --git a/workflows/review/eval/live-producer.ts b/workflows/review/eval/live-producer.ts new file mode 100644 index 00000000..e5bb1283 --- /dev/null +++ b/workflows/review/eval/live-producer.ts @@ -0,0 +1,601 @@ +/** + * The live finding producer (`live-ab-plan.md` Phase 2c): run the REAL model + * sub-agents from a `review.md` over one live-enabled corpus case and return + * findings + claim-validator verifications in exactly the shapes the + * deterministic runner consumes (`RunOptions.produceFindings` + + * `applyValidation`). Every downstream stage (provenance gate, scope filter, + * verdict, rendering, metrics) is then identical between a recorded replay + * and a live arm. + * + * The model seam is an injected {@link LiveAgentRunner} (mirroring how + * `judge.ts` takes a `JudgeModel`): this module performs no model or network + * call itself, so its logic is unit-testable with a stub. The one production + * implementation (Agent SDK) lives in `live-runner.ts`. + * + * Deliberate deviations from production, documented here once: + * - No `pattern-triage` pass and no `thread-reconciler` (no threads exist in + * eval); the roster is the two default whole-change reviewers plus the + * router's `lensesToSpawn`. + * - `{{#runtime-import }}` directives are compile-time inlines of + * consumer-repo files. Here they resolve against the case's checkout tree + * when the file exists there, else to a fixed "not configured" note, so a + * case can opt into a skills index by carrying the file in its tree. + * - The investigation-cap CLI the prompts invoke is not staged; sub-agents + * run with read-only tools and treat the unavailable cap as a denied + * budget (the prompt's own fallback: stop investigating, report what you + * have). + */ + +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + writeFileSync, +} from "node:fs"; + +import {isBlockingLabel, labelForFinding} from "../lib/render-comment"; +import {route, type RouterConfig} from "../lib/router"; +import {validateFinding, type Finding, type Lens} from "../lib/finding-schema"; +import { + VERIFICATION_STATES, + type CaseVerification, + type CorpusCase, + type RecordedFinding, + type VerificationState, +} from "./corpus/loader"; +import type {ExtractedAgent} from "./agent-extract"; +import { + rewriteAgentPrompt, + stageCase, + type StageFs, + type StagedCase, +} from "./live-stage"; + +/* -------------------------------------------------------------------------- */ +/* The model seam */ +/* -------------------------------------------------------------------------- */ + +/** One sub-agent dispatch request. */ +export type LiveAgentRequest = { + /** Agent name (for labeling/telemetry). */ + name: string; + /** Pinned model id from the agent's frontmatter. */ + model: string; + /** The fully-resolved prompt (imports inlined, staging paths rewritten). */ + prompt: string; + /** The staged checkout the agent investigates (its cwd). */ + cwd: string; + /** Hard turn cap. */ + maxTurns: number; + /** Hard wall-clock cap, enforced by the runner. */ + timeoutMs: number; +}; + +/** What a dispatch returned, with its measured cost. */ +export type LiveAgentResult = { + /** The agent's final text (expected to be the JSON contract). */ + output: string; + /** Billed cost in USD (0 when the runner cannot price it). */ + usd: number; + /** Turns consumed. */ + turns: number; + /** Wall-clock milliseconds. */ + wallMs: number; +}; + +/** The injected model runner; the ONLY place a real model is invoked. */ +export type LiveAgentRunner = ( + request: LiveAgentRequest, +) => Promise; + +/* -------------------------------------------------------------------------- */ +/* Results */ +/* -------------------------------------------------------------------------- */ + +/** Per-agent accounting for the cost report. */ +export type PerAgentReport = { + name: string; + model: string; + usd: number; + turns: number; + wallMs: number; + /** Whether the malformed-output retry fired. */ + retried: boolean; + /** Fixed-format failure note; the agent contributed nothing when set. */ + failed?: string; +}; + +export type ProduceLiveResult = { + /** Schema-valid findings, in the corpus `RecordedFinding` shape. */ + findings: RecordedFinding[]; + /** Claim-validator verifications, in the corpus `validation` shape. */ + validation: CaseVerification[]; + perAgent: PerAgentReport[]; + staged: StagedCase; +}; + +export type ProduceLiveOptions = { + runner: LiveAgentRunner; + /** Directory to stage the case under (one case per directory). */ + stageDir: string; + fs?: StageFs; + maxTurns?: number; + timeoutMs?: number; + /** Concurrent sub-agent dispatches within the case. */ + concurrency?: number; +}; + +const DEFAULT_MAX_TURNS = 30; +const DEFAULT_TIMEOUT_MS = 5 * 60 * 1000; +const DEFAULT_CONCURRENCY = 4; + +/** The real filesystem, in the staging seam's shape (mirrors live-stage). */ +const NODE_FS: StageFs = { + existsSync, + mkdirSync: (p, opts) => { + mkdirSync(p, opts); + }, + readdirSync: (p, opts) => + readdirSync(p, opts) as unknown as ReturnType, + readFileSync: (p, enc) => readFileSync(p, enc), + writeFileSync: (p, data) => { + writeFileSync(p, data); + }, +}; + +/** Production's confidence default for label-shape reviewers (review.md). */ +const LABEL_SHAPE_CONFIDENCE = 0.7; + +/** The always-on finders (pattern-triage and thread-reconciler excluded). */ +const DEFAULT_FINDERS = ["correctness-reviewer", "skill-auditor"] as const; + +const VALIDATOR = "claim-validator"; + +/* -------------------------------------------------------------------------- */ +/* Prompt resolution */ +/* -------------------------------------------------------------------------- */ + +const RUNTIME_IMPORT = /\{\{#runtime-import\??\s+([^}\s]+)\s*\}\}/g; + +const IMPORT_FALLBACK = "(not configured for this eval case)"; + +/** + * Inline `{{#runtime-import }}` directives from the case's checkout + * tree, falling back to a fixed note when the tree does not carry the file. + * Exported for the A/B runner's reporting (which imports resolved per case). + */ +export const resolveRuntimeImports = ( + prompt: string, + checkoutDir: string, + fs: Pick, +): string => + prompt.replace(RUNTIME_IMPORT, (_match, importPath: string) => { + const full = `${checkoutDir}/${importPath}`; + return fs.existsSync(full) + ? fs.readFileSync(full, "utf8") + : IMPORT_FALLBACK; + }); + +/* -------------------------------------------------------------------------- */ +/* Output parsing: the three sub-agent contracts -> RecordedFinding */ +/* -------------------------------------------------------------------------- */ + +/** A produced finding plus the claims-path extras the validator reads. */ +type LiveFinding = RecordedFinding & {skill?: string}; + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +/** Extract the JSON object from an agent's final text (live-judge's rule). */ +const parseJsonObject = (output: string): Record => { + const match = output.match(/\{[\s\S]*\}/); + if (!match) { + throw new Error("output carries no JSON object"); + } + const parsed: unknown = JSON.parse(match[0]); + if (!isRecord(parsed)) { + throw new Error("output JSON is not an object"); + } + return parsed; +}; + +/** + * Map one label-shape finding (correctness-reviewer / skill-auditor contract) + * into a schema finding. The lens is code-assigned: `correctness` for the + * correctness reviewer, `conventions` for the skill auditor (the one + * best-practice lens, so `labelForFinding` reproduces the `, best-practice` + * label variants the auditor emits). + */ +const fromLabelShape = ( + agentName: string, + lens: Lens, + source: string, + raw: unknown, + index: number, +): LiveFinding => { + if (!isRecord(raw)) { + throw new Error(`findings[${index}] is not an object`); + } + const label = typeof raw["label"] === "string" ? raw["label"] : ""; + const subject = typeof raw["subject"] === "string" ? raw["subject"] : ""; + const discussion = + typeof raw["discussion"] === "string" ? raw["discussion"] : ""; + const candidate: Record = { + schema_version: 2, + id: `live-${agentName}-${index + 1}`, + lens, + anchor: { + type: "line", + path: raw["path"], + line: raw["line"], + side: "RIGHT", + }, + severity: isBlockingLabel(label) ? "blocking" : "advisory", + confidence: LABEL_SHAPE_CONFIDENCE, + evidence_trace: [ + `${agentName} label: ${label}`, + ...(discussion === "" ? [] : [discussion]), + ], + failure_scenario: raw["failure_scenario"], + producing_hunt: `live:${agentName}`, + model_authored_prose: + discussion === "" ? subject : `${subject} ${discussion}`.trim(), + ...(typeof raw["suggestion"] === "string" && raw["suggestion"] !== "" + ? {suggested_patch: raw["suggestion"]} + : {}), + }; + const result = validateFinding(candidate); + if (!result.ok) { + throw new Error(`findings[${index}]: ${result.errors.join("; ")}`); + } + return { + source, + finding: result.finding, + ...(typeof raw["skill"] === "string" && raw["skill"] !== "" + ? {skill: raw["skill"]} + : {}), + }; +}; + +/** + * Parse one agent's output into live findings, per its contract. Every id is + * namespaced with the case id (`:`): live agents choose their own + * ids, so without the namespace two cases produce colliding ids (every case's + * first correctness finding would be `live-correctness-reviewer-1`), and the + * judge's score join requires ids unique across the whole arm. + */ +const parseAgentFindings = ( + agent: ExtractedAgent, + output: string, + usedIds: Set, + caseId: string, +): LiveFinding[] => { + const parsed = parseJsonObject(output); + const rawFindings = parsed["findings"]; + if (!Array.isArray(rawFindings)) { + throw new Error("output JSON has no findings array"); + } + + const labelLens: Record = { + "correctness-reviewer": {lens: "correctness", source: "correctness"}, + "skill-auditor": {lens: "conventions", source: "skill"}, + }; + + const findings = rawFindings.map((raw, index): LiveFinding => { + const label = labelLens[agent.name]; + if (label !== undefined) { + return fromLabelShape( + agent.name, + label.lens, + label.source, + raw, + index, + ); + } + // Specialist lens: already the structured finding schema. + const result = validateFinding(raw); + if (!result.ok) { + throw new Error(`findings[${index}]: ${result.errors.join("; ")}`); + } + return {source: agent.name, finding: result.finding}; + }); + + // Namespace with the case id (see the function doc), then dedupe within + // the case: prefix a collision with the producing agent's name rather + // than dropping a real finding. + for (const live of findings) { + live.finding = {...live.finding, id: `${caseId}:${live.finding.id}`}; + if (usedIds.has(live.finding.id)) { + live.finding = { + ...live.finding, + id: `${agent.name}:${live.finding.id}`, + }; + } + usedIds.add(live.finding.id); + } + return findings; +}; + +/* -------------------------------------------------------------------------- */ +/* The claims path */ +/* -------------------------------------------------------------------------- */ + +/** Build the claims.json entries the validator's contract names. */ +const buildClaims = (findings: LiveFinding[]): Record[] => + findings.map((live) => { + const {finding} = live; + return { + id: finding.id, + source: live.source, + ...(finding.anchor.type !== "pr" + ? {path: finding.anchor.path} + : {}), + ...(finding.anchor.type === "line" + ? {line: finding.anchor.line} + : {}), + label: labelForFinding(finding), + subject: finding.model_authored_prose, + discussion: finding.evidence_trace.join(" | "), + failure_scenario: finding.failure_scenario, + confidence: finding.confidence, + ...(finding.suggested_patch !== undefined + ? {suggestion: finding.suggested_patch} + : {}), + ...(live.skill !== undefined ? {skill: live.skill} : {}), + }; + }); + +/** Parse the validator's `{"claims": [...]}` output into verifications. */ +const parseVerifications = ( + output: string, + knownIds: Set, +): CaseVerification[] => { + const parsed = parseJsonObject(output); + const rawClaims = parsed["claims"]; + if (!Array.isArray(rawClaims)) { + throw new Error("validator output has no claims array"); + } + const verifications: CaseVerification[] = []; + rawClaims.forEach((raw, index) => { + if (!isRecord(raw)) { + throw new Error(`claims[${index}] is not an object`); + } + const id = raw["id"]; + const verification = raw["verification"]; + if (typeof id !== "string" || !knownIds.has(id)) { + throw new Error( + `claims[${index}].id does not match a produced finding`, + ); + } + if ( + typeof verification !== "string" || + !VERIFICATION_STATES.includes(verification as VerificationState) + ) { + throw new Error(`claims[${index}].verification is invalid`); + } + const out: CaseVerification = { + id, + verification: verification as VerificationState, + }; + const confidence = raw["confidence"]; + if ( + typeof confidence === "number" && + confidence >= 0 && + confidence <= 1 + ) { + out.confidence = confidence; + } + verifications.push(out); + }); + return verifications; +}; + +/* -------------------------------------------------------------------------- */ +/* Dispatch */ +/* -------------------------------------------------------------------------- */ + +/** A bounded-concurrency map that preserves input order in its results. */ +const mapWithConcurrency = async ( + items: T[], + limit: number, + fn: (item: T) => Promise, +): Promise => { + const results: R[] = new Array(items.length); + let next = 0; + const workers = Array.from( + {length: Math.min(limit, items.length)}, + async () => { + for (;;) { + const index = next++; + if (index >= items.length) { + return; + } + results[index] = await fn(items[index] as T); + } + }, + ); + await Promise.all(workers); + return results; +}; + +/** + * Dispatch one agent with the malformed-output retry: a first failure is fed + * back verbatim and the agent gets exactly one more attempt; a second failure + * marks the agent failed and the run continues without it. + */ +const dispatchWithRetry = async ( + agent: ExtractedAgent, + prompt: string, + request: Omit, + runner: LiveAgentRunner, + parse: (output: string) => R, +): Promise<{report: PerAgentReport; parsed?: R}> => { + const report: PerAgentReport = { + name: agent.name, + model: agent.model, + usd: 0, + turns: 0, + wallMs: 0, + retried: false, + }; + let attemptPrompt = prompt; + for (let attempt = 0; attempt < 2; attempt++) { + let failure: string; + try { + const result = await runner({...request, prompt: attemptPrompt}); + report.usd += result.usd; + report.turns += result.turns; + report.wallMs += result.wallMs; + try { + return {report, parsed: parse(result.output)}; + } catch (parseError) { + failure = `malformed output: ${String( + parseError instanceof Error + ? parseError.message + : parseError, + )}`; + } + attemptPrompt = + `${prompt}\n\n` + + `Your previous output was rejected: ${failure}\n` + + `Return ONLY the corrected JSON object.`; + } catch (runError) { + failure = `dispatch failed: ${String( + runError instanceof Error ? runError.message : runError, + )}`; + } + if (attempt === 0) { + report.retried = true; + } else { + report.failed = failure; + } + } + return {report}; +}; + +/* -------------------------------------------------------------------------- */ +/* The producer */ +/* -------------------------------------------------------------------------- */ + +/** + * Run the live sub-agent roster over one live-enabled corpus case: stage it, + * dispatch the default finders plus the routed lenses, parse and + * schema-validate their findings, then dispatch the claim-validator over the + * assembled claims. Partial results are kept: a failed agent is reported in + * `perAgent` and contributes nothing; a failed validator yields an empty + * `validation` list (the deterministic replay then posts unvalidated + * candidates, exactly production's fallback). + */ +export const produceLive = async ( + corpusCase: CorpusCase, + agents: Map, + options: ProduceLiveOptions, +): Promise => { + const {runner} = options; + const maxTurns = options.maxTurns ?? DEFAULT_MAX_TURNS; + const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY; + + const fs = options.fs ?? NODE_FS; + const staged = stageCase(corpusCase, options.stageDir, fs); + + // Roster: default finders + routed specialist lenses. + const routerConfig: RouterConfig = { + generatedPatterns: [], + ...(corpusCase.routerConfig as Partial), + }; + const routing = route({files: corpusCase.changedFiles}, routerConfig); + const rosterNames = [...DEFAULT_FINDERS, ...routing.lensesToSpawn]; + const roster = rosterNames.map((name) => { + const agent = agents.get(name); + if (agent === undefined) { + throw new Error( + `sub-agent "${name}" is not defined in the extracted review.md`, + ); + } + return agent; + }); + + const resolvePrompt = (agent: ExtractedAgent): string => + rewriteAgentPrompt( + resolveRuntimeImports(agent.prompt, staged.checkoutDir, fs), + staged, + ); + + const usedIds = new Set(); + const findings: LiveFinding[] = []; + const perAgent: PerAgentReport[] = []; + + const finderResults = await mapWithConcurrency( + roster, + concurrency, + async (agent) => + dispatchWithRetry( + agent, + resolvePrompt(agent), + { + name: agent.name, + model: agent.model, + cwd: staged.checkoutDir, + maxTurns, + timeoutMs, + }, + runner, + (output) => + parseAgentFindings(agent, output, usedIds, corpusCase.id), + ), + ); + for (const {report, parsed} of finderResults) { + perAgent.push(report); + if (parsed !== undefined) { + findings.push(...parsed); + } + } + + // The claims path: skip entirely when nothing was found (production + // skips Phase 3 on an empty candidate set). + let validation: CaseVerification[] = []; + if (findings.length > 0) { + const validator = agents.get(VALIDATOR); + if (validator === undefined) { + throw new Error( + `sub-agent "${VALIDATOR}" is not defined in the extracted review.md`, + ); + } + const claims = buildClaims(findings); + fs.writeFileSync( + `${staged.contextDir}/claims.json`, + JSON.stringify(claims, null, 2), + ); + const knownIds = new Set(findings.map((live) => live.finding.id)); + const {report, parsed} = await dispatchWithRetry( + validator, + resolvePrompt(validator), + { + name: validator.name, + model: validator.model, + cwd: staged.checkoutDir, + maxTurns, + timeoutMs, + }, + runner, + (output) => parseVerifications(output, knownIds), + ); + perAgent.push(report); + validation = parsed ?? []; + } + + return { + findings: findings.map( + ({source, finding}): RecordedFinding => ({source, finding}), + ), + validation, + perAgent, + staged, + }; +}; + +/** Re-exported so the A/B runner types its recorded outputs without reaching + * into internals. */ +export type {Finding}; diff --git a/workflows/review/eval/live-runner.ts b/workflows/review/eval/live-runner.ts new file mode 100644 index 00000000..5a0bdc2b --- /dev/null +++ b/workflows/review/eval/live-runner.ts @@ -0,0 +1,158 @@ +/** + * The production {@link LiveAgentRunner}: dispatch one sub-agent as a bounded + * agentic loop via the Claude Agent SDK, plus a CLI smoke entry point + * (`live-ab-plan.md` Phase 2c). + * + * This is the ONLY module in the eval suite that talks to a real model + * runtime. `live-producer.ts` stays SDK-free behind its runner seam, so unit + * tests never load this file. + * + * Tool policy: read-only investigation (Read/Grep/Glob), cwd pinned to the + * staged checkout, no network. The investigation-cap CLI the prompts mention + * is not runnable under this policy; the prompts' own fallback applies (a + * denied budget request stops investigation, findings still report). + * + * Run one case end to end (requires ANTHROPIC_API_KEY): + * + * pnpm dlx tsx workflows/review/eval/live-runner.ts --case + * [--review-md workflows/review/review.md] [--stage-root /tmp/review-live] + */ + +/* eslint-disable no-console -- CLI entry point; console IS the interface. */ + +import {mkdtempSync, readFileSync} from "node:fs"; +import {tmpdir} from "node:os"; + +import {query} from "@anthropic-ai/claude-agent-sdk"; + +import {extractAgents} from "./agent-extract"; +import {loadLiveCorpus} from "./corpus/loader"; +import {produceLive, type LiveAgentRunner} from "./live-producer"; + +/** Read-only investigation tools; see the module doc for the rationale. */ +const ALLOWED_TOOLS = ["Read", "Grep", "Glob"]; + +/** + * Build the SDK-backed runner. Each request becomes one `query()` run: the + * agent's prompt, its pinned model, the staged checkout as cwd, hard turn and + * wall-clock caps, and cost/turn accounting read off the result message. + */ +export const sdkRunner = (): LiveAgentRunner => async (request) => { + const started = Date.now(); + const abort = new AbortController(); + const timer = setTimeout(() => { + abort.abort( + new Error(`sub-agent timed out after ${request.timeoutMs}ms`), + ); + }, request.timeoutMs); + try { + const run = query({ + prompt: request.prompt, + options: { + cwd: request.cwd, + model: request.model, + maxTurns: request.maxTurns, + allowedTools: ALLOWED_TOOLS, + permissionMode: "bypassPermissions", + abortController: abort, + }, + }); + let output = ""; + let usd = 0; + let turns = 0; + for await (const message of run) { + if (message.type !== "result") { + continue; + } + const result = message as unknown as { + subtype: string; + result?: string; + total_cost_usd?: number; + num_turns?: number; + }; + if (result.subtype !== "success") { + throw new Error( + `sub-agent run ended without success: ${result.subtype}`, + ); + } + output = result.result ?? ""; + usd = result.total_cost_usd ?? 0; + turns = result.num_turns ?? 0; + } + return {output, usd, turns, wallMs: Date.now() - started}; + } finally { + clearTimeout(timer); + } +}; + +/* -------------------------------------------------------------------------- */ +/* CLI */ +/* -------------------------------------------------------------------------- */ + +const argValue = (flag: string): string | undefined => { + const index = process.argv.indexOf(flag); + return index === -1 ? undefined : process.argv[index + 1]; +}; + +const main = async (): Promise => { + if (!process.env["ANTHROPIC_API_KEY"]) { + throw new Error("ANTHROPIC_API_KEY is required for a live run."); + } + const caseId = argValue("--case"); + if (caseId === undefined) { + throw new Error("usage: live-runner.ts --case "); + } + const reviewMdPath = + argValue("--review-md") ?? "workflows/review/review.md"; + const stageRoot = + argValue("--stage-root") ?? mkdtempSync(`${tmpdir()}/review-live-`); + + const cases = loadLiveCorpus(); + const corpusCase = cases.find((c) => c.id === caseId); + if (corpusCase === undefined) { + throw new Error( + `no live case "${caseId}"; available: ${cases + .map((c) => c.id) + .join(", ")}`, + ); + } + + const agents = extractAgents(readFileSync(reviewMdPath, "utf8")); + console.error( + `running case ${caseId} (${agents.size} agents extracted) ` + + `staged under ${stageRoot}`, + ); + + const result = await produceLive(corpusCase, agents, { + runner: sdkRunner(), + stageDir: `${stageRoot}/${caseId}`, + }); + + const totalUsd = result.perAgent.reduce((sum, a) => sum + a.usd, 0); + console.log( + JSON.stringify( + { + caseId, + findings: result.findings, + validation: result.validation, + perAgent: result.perAgent, + totalUsd, + }, + null, + 2, + ), + ); + console.error( + `done: ${result.findings.length} finding(s), ` + + `${result.validation.length} verification(s), ` + + `$${totalUsd.toFixed(2)}`, + ); +}; + +// CLI entry point (mirrors live-judge.ts): run when executed, not imported. +if (process.argv[1]?.endsWith("live-runner.ts")) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +} diff --git a/workflows/review/eval/live-stage.test.ts b/workflows/review/eval/live-stage.test.ts new file mode 100644 index 00000000..e72e1648 --- /dev/null +++ b/workflows/review/eval/live-stage.test.ts @@ -0,0 +1,157 @@ +import {describe, it, expect} from "vitest"; +import {Volume} from "memfs"; + +import {parseCase} from "./corpus/loader"; +import { + PRODUCTION_REVIEW_DIR, + rewriteAgentPrompt, + stageCase, + type StageFs, +} from "./live-stage"; + +/** Adapt a memfs volume to the staging fs seam. */ +const volFs = (vol: InstanceType): StageFs => ({ + existsSync: (p) => vol.existsSync(p), + mkdirSync: (p, opts) => { + vol.mkdirSync(p, opts); + }, + readdirSync: (p, opts) => + vol.readdirSync(p, opts) as unknown as ReturnType< + StageFs["readdirSync"] + >, + readFileSync: (p, enc) => vol.readFileSync(p, enc) as string, + writeFileSync: (p, data) => { + vol.writeFileSync(p, data); + }, +}); + +const DIFF = [ + "diff --git a/src/a.ts b/src/a.ts", + "--- a/src/a.ts", + "+++ b/src/a.ts", + "@@ -1,2 +1,2 @@", + "-const a = 1;", + "+const a = 2;", + " export {a};", + "", +].join("\n"); + +/** A live case parsed through the real loader path. */ +const liveCase = (over: Record = {}) => + parseCase( + { + id: "stage-case", + tags: ["live"], + category: "clean", + description: "a stageable case", + changedFiles: [ + {path: "src/a.ts", status: "modified"}, + // Listed but absent from the diff: hasPatch must be false. + {path: "assets/logo.png", status: "modified"}, + ], + expected: {verdict: "APPROVE"}, + diff: DIFF, + live: { + prContext: { + title: "A staged change", + description: "body text", + author: "octocat", + baseBranch: "main", + }, + }, + ...over, + }, + "/corpus/clean/stage-case/case.json", + ); + +const treeVol = () => + Volume.fromJSON({ + "/corpus/clean/stage-case/tree/src/a.ts": "const a = 2;\nexport {a};\n", + "/corpus/clean/stage-case/tree/assets/logo.png": "binaryish", + }); + +describe("stageCase", () => { + it("materializes the production staging layout", () => { + const vol = treeVol(); + const staged = stageCase(liveCase(), "/stage", volFs(vol)); + expect(staged.contextDir).toBe("/stage/context"); + expect(staged.checkoutDir).toBe("/stage/checkout"); + + const read = (p: string) => vol.readFileSync(p, "utf8") as string; + expect(read("/stage/context/full.diff")).toBe(DIFF); + expect(read("/stage/context/pr.diff")).toBe(DIFF); + expect(read("/stage/context/full-stripped.diff")).toBe(DIFF); + + const files = JSON.parse(read("/stage/context/files.json")); + expect(files).toEqual([ + {path: "src/a.ts", status: "modified", hasPatch: true}, + {path: "assets/logo.png", status: "modified", hasPatch: false}, + ]); + expect(read("/stage/context/review-files.json")).toBe( + read("/stage/context/files.json"), + ); + + const prContext = JSON.parse(read("/stage/context/pr-context.json")); + expect(prContext.title).toBe("A staged change"); + expect(prContext.author).toBe("octocat"); + expect(prContext.baseBranch).toBe("main"); + expect(prContext.isDraft).toBe(false); + expect(prContext.diffPath).toBe("/stage/context/full.diff"); + expect(prContext.filesPath).toBe("/stage/context/files.json"); + + const provenance = JSON.parse(read("/stage/context/provenance.json")); + expect(provenance.warnings).toEqual([]); + expect(provenance.files["src/a.ts"].added).toContain(1); + + expect(JSON.parse(read("/stage/context/routing.json"))).toHaveProperty( + "lensesToSpawn", + ); + expect(vol.existsSync("/stage/context/out")).toBe(true); + }); + + it("copies the tree recursively into the checkout", () => { + const vol = treeVol(); + stageCase(liveCase(), "/stage", volFs(vol)); + expect(vol.readFileSync("/stage/checkout/src/a.ts", "utf8")).toBe( + "const a = 2;\nexport {a};\n", + ); + expect(vol.existsSync("/stage/checkout/assets/logo.png")).toBe(true); + }); + + it("throws on a non-live case and on a missing tree", () => { + const recorded = parseCase( + { + id: "recorded-only", + tags: ["smoke"], + category: "clean", + description: "no live block", + changedFiles: [{path: "src/a.ts", status: "modified"}], + expected: {verdict: "APPROVE"}, + }, + "/corpus/clean/recorded-only.json", + ); + expect(() => + stageCase(recorded, "/stage", volFs(new Volume())), + ).toThrow(/not live-enabled/); + expect(() => + stageCase(liveCase(), "/stage", volFs(new Volume())), + ).toThrow(/tree .* does not exist/); + }); +}); + +describe("rewriteAgentPrompt", () => { + it("rewrites every production staging path to the case context dir", () => { + const vol = treeVol(); + const staged = stageCase(liveCase(), "/stage", volFs(vol)); + const prompt = [ + `Read ${PRODUCTION_REVIEW_DIR}/pr-context.json first.`, + `The diff: ${PRODUCTION_REVIEW_DIR}/pr.diff.`, + `Write output under ${PRODUCTION_REVIEW_DIR}/out/.`, + ].join("\n"); + const rewritten = rewriteAgentPrompt(prompt, staged); + expect(rewritten).not.toContain(PRODUCTION_REVIEW_DIR); + expect(rewritten).toContain("/stage/context/pr-context.json"); + expect(rewritten).toContain("/stage/context/pr.diff"); + expect(rewritten).toContain("/stage/context/out/"); + }); +}); diff --git a/workflows/review/eval/live-stage.ts b/workflows/review/eval/live-stage.ts new file mode 100644 index 00000000..6f4f4534 --- /dev/null +++ b/workflows/review/eval/live-stage.ts @@ -0,0 +1,197 @@ +/** + * Case staging for the live eval arm (`live-ab-plan.md` Phase 2b). + * + * Production sub-agents have no GitHub access: `review.md` Step 1 stages the + * PR on disk under `/tmp/gh-aw/review/` and every sub-agent prompt names + * those paths. This module materializes the SAME layout for a live-enabled + * corpus case, so the extracted prompts run against a case exactly as they + * run against a real PR: + * + * /context/pr-context.json PR metadata (from the case's live block) + * /context/full.diff the case diff (git-style unified diff) + * /context/full-stripped.diff = full.diff (corpus diffs carry no + * generated files to strip) + * /context/pr.diff = full.diff (no pattern-triage pass: + * every changed file is a review file) + * /context/files.json path/status/hasPatch per changed file + * /context/review-files.json = files.json entries (see pr.diff) + * /context/provenance.json the diff's changed-line map + * /context/routing.json deterministic router output + * /context/out/ sub-agent output directory + * /checkout/ the case's post-change file tree + * + * Prompts are then rewritten with {@link rewriteAgentPrompt}, which swaps the + * production staging root for the case's context directory. + */ + +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + writeFileSync, +} from "node:fs"; + +import {computeDiffProvenance} from "../lib/provenance"; +import {route, type RouterConfig} from "../lib/router"; +import type {CorpusCase} from "./corpus/loader"; + +/** The staging root production prompts reference (review.md Step 1). */ +export const PRODUCTION_REVIEW_DIR = "/tmp/gh-aw/review"; + +/** Filesystem seam so staging is testable against memfs. */ +export type StageFs = { + existsSync: (p: string) => boolean; + mkdirSync: (p: string, opts: {recursive: true}) => void; + readdirSync: ( + p: string, + opts: {withFileTypes: true}, + ) => {name: string; isDirectory: () => boolean; isFile: () => boolean}[]; + readFileSync: (p: string, enc: "utf8") => string; + writeFileSync: (p: string, data: string) => void; +}; + +const DEFAULT_FS: StageFs = { + existsSync, + mkdirSync: (p, opts) => { + mkdirSync(p, opts); + }, + readdirSync: (p, opts) => + readdirSync(p, opts) as unknown as ReturnType, + readFileSync: (p, enc) => readFileSync(p, enc), + writeFileSync: (p, data) => { + writeFileSync(p, data); + }, +}; + +/** A staged case: the directories a sub-agent dispatch needs. */ +export type StagedCase = { + caseId: string; + /** The staging root (`destDir` as given). */ + rootDir: string; + /** The context directory prompts are rewritten to point at. */ + contextDir: string; + /** The post-change checkout the sub-agent runs in (its cwd). */ + checkoutDir: string; +}; + +/** Recursively copy a directory through the fs seam. */ +const copyDir = (src: string, dest: string, fs: StageFs): void => { + fs.mkdirSync(dest, {recursive: true}); + for (const entry of fs.readdirSync(src, {withFileTypes: true})) { + const from = `${src}/${entry.name}`; + const to = `${dest}/${entry.name}`; + if (entry.isDirectory()) { + copyDir(from, to, fs); + } else if (entry.isFile()) { + fs.writeFileSync(to, fs.readFileSync(from, "utf8")); + } + } +}; + +/** + * Stage one live-enabled corpus case under `destDir`. Throws when the case is + * not live-enabled (no `live` block / no diff) or its tree is missing: the + * caller selects cases via `loadLiveCorpus()`, so a miss here is a bug, not + * an input to tolerate. + */ +export const stageCase = ( + corpusCase: CorpusCase, + destDir: string, + fs: StageFs = DEFAULT_FS, +): StagedCase => { + const live = corpusCase.live; + const diff = corpusCase.diff; + if (live === undefined || diff === undefined) { + throw new Error( + `stageCase: case "${corpusCase.id}" is not live-enabled`, + ); + } + + const contextDir = `${destDir}/context`; + const checkoutDir = `${destDir}/checkout`; + fs.mkdirSync(`${contextDir}/out`, {recursive: true}); + fs.mkdirSync(checkoutDir, {recursive: true}); + + // The diff surfaces. Corpus diffs carry no generated files, so the + // stripped diff equals the full one; with no pattern-triage pass, the + // review diff does too. + fs.writeFileSync(`${contextDir}/full.diff`, diff); + fs.writeFileSync(`${contextDir}/full-stripped.diff`, diff); + fs.writeFileSync(`${contextDir}/pr.diff`, diff); + + // files.json + review-files.json: path/status/hasPatch. `hasPatch` is + // whether the diff carries a section for the path (the completeness + // cross-check the provenance gate reads). + const provenance = computeDiffProvenance(diff); + const files = corpusCase.changedFiles.map((file) => ({ + path: file.path, + status: file.status, + hasPatch: provenance.files[file.path] !== undefined, + })); + const filesJson = JSON.stringify(files, null, 2); + fs.writeFileSync(`${contextDir}/files.json`, filesJson); + fs.writeFileSync(`${contextDir}/review-files.json`, filesJson); + fs.writeFileSync( + `${contextDir}/provenance.json`, + JSON.stringify(provenance, null, 2), + ); + + // Deterministic routing, exactly as the no-post runner computes it. + const routerConfig: RouterConfig = { + generatedPatterns: [], + ...(corpusCase.routerConfig as Partial), + }; + const routing = route({files: corpusCase.changedFiles}, routerConfig); + fs.writeFileSync( + `${contextDir}/routing.json`, + JSON.stringify(routing, null, 2), + ); + + // PR context (review.md Step 1's shape). Synthetic identity fields are + // fixed values: nothing downstream may key on them. + fs.writeFileSync( + `${contextDir}/pr-context.json`, + JSON.stringify( + { + number: 0, + title: live.prContext.title, + description: live.prContext.description, + author: live.prContext.author, + baseBranch: live.prContext.baseBranch, + headSha: "0000000000000000000000000000000000000000", + isDraft: false, + repo: "eval/corpus", + diffPath: `${contextDir}/full.diff`, + filesPath: `${contextDir}/files.json`, + }, + null, + 2, + ), + ); + + // The post-change checkout the sub-agents read and investigate. + const lastSlash = corpusCase.sourcePath.lastIndexOf("/"); + const caseDir = + lastSlash === -1 ? "." : corpusCase.sourcePath.slice(0, lastSlash); + const treeDir = `${caseDir}/${live.tree}`; + if (!fs.existsSync(treeDir)) { + throw new Error( + `stageCase: case "${corpusCase.id}" tree "${treeDir}" does not exist`, + ); + } + copyDir(treeDir, checkoutDir, fs); + + return {caseId: corpusCase.id, rootDir: destDir, contextDir, checkoutDir}; +}; + +/** + * Rewrite an extracted agent prompt to read the staged case instead of the + * production staging root. Every occurrence of {@link PRODUCTION_REVIEW_DIR} + * is replaced, so any staged filename a prompt names (pr.diff, files.json, + * full-stripped.diff, out/...) resolves inside the case's context directory. + */ +export const rewriteAgentPrompt = ( + prompt: string, + staged: StagedCase, +): string => prompt.split(PRODUCTION_REVIEW_DIR).join(staged.contextDir);