diff --git a/.github/agents/registry.yml b/.github/agents/registry.yml new file mode 100644 index 00000000..2b6def67 --- /dev/null +++ b/.github/agents/registry.yml @@ -0,0 +1,16 @@ +version: 1 + +default_agent: codex + +agents: + codex: + runner_workflow: .github/workflows/reusable-codex-run.yml + required_secrets: + - CODEX_AUTH_JSON + branch_prefix: codex/issue- + ui_mentions_allowed: false + capabilities: + pr_keepalive: true + pr_autofix: true + belt: true + verifier_checkbox: true diff --git a/.github/scripts/agent_registry.js b/.github/scripts/agent_registry.js new file mode 100644 index 00000000..2aaeb3c8 --- /dev/null +++ b/.github/scripts/agent_registry.js @@ -0,0 +1,216 @@ +'use strict'; + +const fs = require('node:fs'); + +function stripTrailingComment(rawLine) { + const line = String(rawLine ?? ''); + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) { + return ''; + } + + // Keep this intentionally simple: our registry YAML should not rely on inline comments. + const match = line.match(/^(.*?)(\s+#.*)?$/); + return (match?.[1] ?? line).replace(/\s+$/, ''); +} + +function parseScalar(value) { + const raw = String(value ?? '').trim(); + if (!raw) { + return ''; + } + + if (raw === 'true') { + return true; + } + if (raw === 'false') { + return false; + } + + if (/^-?\d+$/.test(raw)) { + return Number(raw); + } + + const quoted = raw.match(/^(['"])(.*)\1$/); + if (quoted) { + return quoted[2]; + } + + return raw; +} + +function countIndent(line) { + // Match all leading horizontal whitespace (spaces and tabs). + const match = String(line).match(/^([ \t]*)/); + const indentPrefix = match?.[1] ?? ''; + if (indentPrefix.includes('\t')) { + throw new Error('Registry YAML must use spaces only (tabs are not allowed)'); + } + if (indentPrefix.length % 2 !== 0) { + throw new Error( + `Registry YAML indentation must be multiples of 2 spaces (got ${indentPrefix.length})`, + ); + } + return indentPrefix.length; +} + +function findNextMeaningfulLine(lines, startIndex) { + for (let index = startIndex; index < lines.length; index += 1) { + const stripped = stripTrailingComment(lines[index]); + if (!stripped.trim()) { + continue; + } + return { + index, + indent: countIndent(stripped), + trimmed: stripped.trim(), + }; + } + return null; +} + +// Minimal YAML parser for the registry file. +// Supported features: +// - nested mappings via indentation (2 spaces) +// - scalar values (strings, booleans, integers) +// - sequences using "- item" lines (scalar items only) +// Unsupported (intentionally): anchors, multiline strings, flow maps, complex quoting. +function parseRegistryYaml(text) { + const rawLines = String(text ?? '').split(/\r?\n/); + const lines = rawLines.map(stripTrailingComment); + + const root = {}; + const stack = [{ indent: -1, container: root }]; + + for (let lineIndex = 0; lineIndex < lines.length; lineIndex += 1) { + const rawLine = lines[lineIndex]; + if (!rawLine.trim()) { + continue; + } + + const indent = countIndent(rawLine); + const trimmed = rawLine.trim(); + + while (stack.length > 1 && indent <= stack[stack.length - 1].indent) { + stack.pop(); + } + + const parent = stack[stack.length - 1].container; + + if (trimmed.startsWith('- ')) { + if (!Array.isArray(parent)) { + throw new Error(`Unexpected list item at line ${lineIndex + 1}; parent is not a list`); + } + parent.push(parseScalar(trimmed.slice(2))); + continue; + } + + const sepIndex = trimmed.indexOf(':'); + if (sepIndex <= 0) { + throw new Error(`Invalid registry YAML line ${lineIndex + 1}: expected "key: value"`); + } + + const key = trimmed.slice(0, sepIndex).trim(); + const rest = trimmed.slice(sepIndex + 1).trim(); + + if (!key) { + throw new Error(`Invalid registry YAML line ${lineIndex + 1}: empty key`); + } + if (typeof parent !== 'object' || parent === null || Array.isArray(parent)) { + throw new Error(`Invalid registry YAML line ${lineIndex + 1}: cannot assign key under a list`); + } + + if (rest) { + parent[key] = parseScalar(rest); + continue; + } + + const next = findNextMeaningfulLine(lines, lineIndex + 1); + const shouldBeList = Boolean(next && next.indent > indent && next.trimmed.startsWith('- ')); + const child = shouldBeList ? [] : {}; + parent[key] = child; + stack.push({ indent, container: child }); + } + + return root; +} + +function loadAgentRegistry({ registryPath } = {}) { + const path = registryPath || '.github/agents/registry.yml'; + const raw = fs.readFileSync(path, 'utf8'); + const registry = parseRegistryYaml(raw); + if (!registry || typeof registry !== 'object') { + throw new Error('Agent registry did not parse into an object'); + } + if (!registry.agents || typeof registry.agents !== 'object') { + throw new Error('Agent registry missing required "agents" mapping'); + } + if (!registry.default_agent || typeof registry.default_agent !== 'string') { + throw new Error('Agent registry missing required "default_agent" string'); + } + return registry; +} + +function normalizeLabel(label) { + if (!label) { + return ''; + } + if (typeof label === 'string') { + return label.trim(); + } + if (typeof label === 'object' && typeof label.name === 'string') { + return label.name.trim(); + } + return ''; +} + +function resolveAgentFromLabels(labels, { registryPath } = {}) { + const registry = loadAgentRegistry({ registryPath }); + const labelList = Array.isArray(labels) ? labels : []; + const agentLabels = labelList + .map(normalizeLabel) + .filter(Boolean) + .filter((value) => value.startsWith('agent:')); + + const uniqueAgents = new Set(agentLabels.map((value) => value.slice('agent:'.length))); + + if (uniqueAgents.size > 1) { + throw new Error(`Multiple agent labels present: ${Array.from(uniqueAgents).join(', ')}`); + } + + const explicit = Array.from(uniqueAgents)[0]; + const agentKey = explicit || registry.default_agent; + if (!registry.agents[agentKey]) { + const known = Object.keys(registry.agents).sort(); + throw new Error(`Unknown agent key: ${agentKey}. Known agents: ${known.join(', ') || '(none)'}`); + } + return agentKey; +} + +function getAgentConfig(agentKey, { registryPath } = {}) { + const registry = loadAgentRegistry({ registryPath }); + const key = String(agentKey || '').trim() || registry.default_agent; + const config = registry.agents[key]; + if (!config) { + const known = Object.keys(registry.agents).sort(); + throw new Error(`Unknown agent key: ${key}. Known agents: ${known.join(', ') || '(none)'}`); + } + return config; +} + +function getRunnerWorkflow(agentKey, { registryPath } = {}) { + const config = getAgentConfig(agentKey, { registryPath }); + const workflow = String(config.runner_workflow || '').trim(); + if (!workflow) { + throw new Error(`Agent config missing runner_workflow for agent: ${agentKey}`); + } + return workflow; +} + +module.exports = { + getAgentConfig, + getRunnerWorkflow, + loadAgentRegistry, + parseRegistryYaml, + resolveAgentFromLabels, +}; diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index 42fe2842..475fb5ca 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -3,6 +3,7 @@ # # Triggers: # - Gate workflow completes with failure (lint/format issues detected) +# - Lint job fails early (workflow_job completed) # - PR labeled with 'autofix' or 'autofix:clean' (manual trigger) # # Copy this file to: .github/workflows/autofix.yml @@ -15,6 +16,8 @@ on: workflow_run: workflows: ["Gate", "CI", "Python CI"] types: [completed] + workflow_job: + types: [completed] pull_request_target: types: - labeled @@ -30,7 +33,8 @@ concurrency: group: >- autofix-${{ github.event.pull_request.number - || github.event.workflow_run.pull_requests[0].number + || (github.event.workflow_run.pull_requests && github.event.workflow_run.pull_requests[0] && github.event.workflow_run.pull_requests[0].number) + || (github.event.workflow_job.pull_requests && github.event.workflow_job.pull_requests[0] && github.event.workflow_job.pull_requests[0].number) || github.run_id }} cancel-in-progress: true @@ -149,6 +153,43 @@ jobs: return true; }; + const listFilesOrNullOnRateLimit = async ({ owner, repo, prNumber }) => { + try { + return await paginateWithRetry( + github.rest.pulls.listFiles, + { + owner, + repo, + pull_number: prNumber, + per_page: 100, + }, + { maxRetries: 3 } + ); + } catch (error) { + const message = String(error?.message || error || ''); + const status = Number(error?.status || error?.response?.status || 0); + if (status === 403 && message.toLowerCase().includes('rate limit')) { + core.warning( + 'Rate limited listing PR files; proceeding without file filter.' + ); + return null; + } + throw error; + } + }; + + const setOutputs = ({ pr, sameRepo, callerActor }) => { + const labels = (pr.labels || []).map((l) => l.name); + core.setOutput('should_run', 'true'); + core.setOutput('pr_number', pr.number); + core.setOutput('head_ref', pr.head.ref); + core.setOutput('pr_title', pr.title); + core.setOutput('pr_is_draft', pr.draft ? 'true' : 'false'); + core.setOutput('pr_labels_json', JSON.stringify(labels)); + core.setOutput('same_repo', sameRepo ? 'true' : 'false'); + core.setOutput('caller_actor', callerActor); + }; + // --- workflow_run trigger (after Gate/CI completes) --- if (context.eventName === 'workflow_run') { const run = context.payload.workflow_run; @@ -261,28 +302,11 @@ jobs: } // Only run autofix when Python files are present. - let files = []; - try { - files = await paginateWithRetry( - github.rest.pulls.listFiles, - { - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber, - per_page: 100, - }, - { maxRetries: 3 } - ); - } catch (error) { - const message = String(error?.message || error || ''); - const status = Number(error?.status || error?.response?.status || 0); - if (status === 403 && message.toLowerCase().includes('rate limit')) { - core.warning('Rate limited listing PR files; proceeding without file filter.'); - files = null; - } else { - throw error; - } - } + const files = await listFilesOrNullOnRateLimit({ + owner: context.repo.owner, + repo: context.repo.repo, + prNumber, + }); const hasPython = files === null @@ -298,29 +322,123 @@ jobs: return; } - const labels = - (pr.labels || []).map(l => l.name); + setOutputs({ + pr, + sameRepo, + callerActor: run.actor?.login || context.actor, + }); + return; + } - core.setOutput('should_run', 'true'); - core.setOutput('pr_number', pr.number); - core.setOutput('head_ref', pr.head.ref); - core.setOutput('pr_title', pr.title); - core.setOutput( - 'pr_is_draft', - pr.draft ? 'true' : 'false', - ); - core.setOutput( - 'pr_labels_json', - JSON.stringify(labels), - ); - core.setOutput( - 'same_repo', - sameRepo ? 'true' : 'false', - ); - core.setOutput( - 'caller_actor', - run.actor?.login || context.actor, + // --- workflow_job trigger (early lint failure) --- + if (context.eventName === 'workflow_job') { + const workflowJob = context.payload.workflow_job; + if (!workflowJob) { + core.setOutput('should_run', 'false'); + return; + } + + const workflowName = String(workflowJob.workflow_name || '').trim(); + const jobName = String(workflowJob.name || '').toLowerCase(); + const conclusion = String(workflowJob.conclusion || '').toLowerCase(); + + // Only respond to failing lint jobs in the workflows we care about. + const relevantJob = + jobName.includes('lint-format') || jobName.includes('lint-ruff'); + if (!relevantJob) { + core.info( + `workflow_job '${workflowJob.name}' is not a lint-format/lint-ruff job.` + ); + core.setOutput('should_run', 'false'); + return; + } + + const allowedWorkflows = new Set(['Gate', 'CI', 'Python CI']); + if (workflowName && !allowedWorkflows.has(workflowName)) { + core.info( + `workflow_job is from workflow '${workflowName}', not Gate/CI/Python CI; skipping.` + ); + core.setOutput('should_run', 'false'); + return; + } + + if (conclusion !== 'failure') { + core.info( + `workflow_job '${workflowJob.name}' concluded '${workflowJob.conclusion}' — no autofix needed` + ); + core.setOutput('should_run', 'false'); + return; + } + + const prs = workflowJob.pull_requests || []; + if (!prs.length) { + core.info('workflow_job event has no associated PR; skipping.'); + core.setOutput('should_run', 'false'); + return; + } + + const prNumber = prs[0].number; + const triggerHeadSha = String(workflowJob.head_sha || ''); + const { owner, repo } = context.repo; + const { data: pr } = await withRetry((client) => + client.rest.pulls.get({ owner, repo, pull_number: prNumber }) ); + + if (pr.state !== 'open') { + core.info(`PR #${prNumber} is ${pr.state}`); + core.setOutput('should_run', 'false'); + return; + } + + if (pr.draft) { + core.info('PR is draft.'); + core.setOutput('should_run', 'false'); + return; + } + + const sameRepo = + pr.head.repo !== null && + pr.head.repo.full_name === pr.base.repo?.full_name; + if (!sameRepo) { + core.info('Fork PR — not supported.'); + core.setOutput('should_run', 'false'); + return; + } + + const headSha = pr.head?.sha; + if (!headSha) { + core.info('PR head SHA missing; skipping autofix.'); + core.setOutput('should_run', 'false'); + return; + } + + if (triggerHeadSha && triggerHeadSha !== headSha) { + core.info( + `workflow_job head_sha ${triggerHeadSha} does not match PR head ${headSha}; skipping stale event.` + ); + core.setOutput('should_run', 'false'); + return; + } + + const files = await listFilesOrNullOnRateLimit({ owner, repo, prNumber }); + const hasPython = + files === null + ? true + : files.some( + (file) => + file.filename.endsWith('.py') || file.filename.endsWith('.pyi') + ); + if (!hasPython) { + core.info('No Python files changed.'); + core.setOutput('should_run', 'false'); + return; + } + + setOutputs({ + pr, + sameRepo, + callerActor: context.payload.sender?.login || context.actor, + }); return; } @@ -384,27 +502,11 @@ jobs: (pr.labels || []).map(l => l.name); let files = []; - try { - files = await paginateWithRetry( - github.rest.pulls.listFiles, - { - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: pr.number, - per_page: 100, - }, - { maxRetries: 3 } - ); - } catch (error) { - const message = String(error?.message || error || ''); - const status = Number(error?.status || error?.response?.status || 0); - if (status === 403 && message.toLowerCase().includes('rate limit')) { - core.warning('Rate limited listing PR files; proceeding without file filter.'); - files = null; - } else { - throw error; - } - } + files = await listFilesOrNullOnRateLimit({ + owner: context.repo.owner, + repo: context.repo.repo, + prNumber: pr.number, + }); const hasPython = files === null @@ -420,23 +522,11 @@ jobs: return; } - core.setOutput('should_run', 'true'); - core.setOutput('pr_number', pr.number); - core.setOutput('head_ref', pr.head.ref); - core.setOutput('pr_title', pr.title); - core.setOutput( - 'pr_is_draft', - pr.draft ? 'true' : 'false', - ); - core.setOutput( - 'pr_labels_json', - JSON.stringify(labels), - ); - core.setOutput( - 'same_repo', - sameRepo ? 'true' : 'false', - ); - core.setOutput('caller_actor', context.actor); + setOutputs({ + pr, + sameRepo, + callerActor: context.actor, + }); # Call reusable autofix workflow autofix: diff --git a/CLAUDE.md b/CLAUDE.md index ba489457..685aac40 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -49,53 +49,6 @@ When an issue is labeled `agent:codex`: ## Common Issues -## CI Test Policy (PR Gate vs CI vs Release E2E) - -This repo intentionally does **not** run the full test surface area on every PR Gate run. - -### PR Gate (`.github/workflows/pr-00-gate.yml`) - -Goal: fast feedback for most PRs. - -- Runs pytest **in parallel** (xdist): `-n auto --dist loadscope` -- Runs pytest **without coverage** (`coverage: false`) -- Skips integration directories: - - `tests/integration/` - - `tests/integrations/` -- Skips **release/packaging** tests via marker: `pytest_markers: "not release"` - -These suites will NOT run on PR Gate unless you run them manually (see below). - -### Main-branch CI (`.github/workflows/ci.yml`) - -Goal: enforce full quality gates on `main`. - -- Runs pytest **with coverage** (`coverage: true`, `coverage-min` enforced) -- Runs pytest **in parallel** (xdist): `-n auto --dist loadscope` -- Runs the full test suite (including integration dirs and `release` tests) - -### Release/Packaging E2E (`.github/workflows/release-e2e.yml`) - -Goal: keep slow PyInstaller + packaged-executable checks out of PR Gate. - -- Runs nightly on `main` -- Runs on PRs when the PR is labeled: `run-release` -- Executes only the tests marked `release`: `pytest -m release` - -### How to run skipped suites locally - -```bash -# Fast PR-gate-like run (parallel, no coverage, skip release + integration dirs) -pytest -q -n auto --dist loadscope -m "not release" \ - --ignore=tests/integration --ignore=tests/integrations - -# Release / packaging validation (PyInstaller + packaged executable) -pytest -q -m release - -# Integration directories (if you need them on a PR) -pytest -q tests/integration tests/integrations -``` - ### Workflow fails with "workflow file issue" - A reusable workflow is being called that doesn't exist - Check Workflows repo has the required `reusable-*.yml` file diff --git a/scripts/langchain/progress_reviewer.py b/scripts/langchain/progress_reviewer.py index c17bfdea..b3953ff9 100755 --- a/scripts/langchain/progress_reviewer.py +++ b/scripts/langchain/progress_reviewer.py @@ -354,7 +354,7 @@ def review_progress_with_llm( ), feedback_for_agent="Review your recent work against the acceptance criteria.", summary=( - f"Heuristic review: {len(aligned)}/{len(recent_commits)} commits appear aligned" + f"Heuristic review: {len(aligned)}/" f"{len(recent_commits)} commits appear aligned" ), used_llm=False, error="LLM unavailable, using heuristic fallback", @@ -454,7 +454,7 @@ def review_progress( ), feedback_for_agent="Work appears aligned. Continue toward task completion.", summary=( - f"Heuristic: {len(aligned)}/{len(recent_commits)} commits aligned with criteria" + f"Heuristic: {len(aligned)}/" f"{len(recent_commits)} commits aligned with criteria" ), used_llm=False, ) diff --git a/scripts/sync_dev_dependencies.py b/scripts/sync_dev_dependencies.py index b38259c6..90a5dd48 100755 --- a/scripts/sync_dev_dependencies.py +++ b/scripts/sync_dev_dependencies.py @@ -57,10 +57,6 @@ ) -def _is_black_drift(change: str) -> bool: - return change.strip().lower().startswith("black:") - - def parse_env_file(path: Path) -> dict[str, str]: """Parse the autofix-versions.env file into a dict of key=value pairs.""" if not path.exists(): @@ -437,12 +433,6 @@ def main(argv: list[str] | None = None) -> int: return 2 if changes: - if args.check and any(_is_black_drift(change) for change in changes): - print( - "Error: Black formatting pin drift detected (version mismatch/out of sync).", - file=sys.stderr, - ) - print(f"{'Applied' if args.apply else 'Found'} {len(changes)} version updates:") for change in changes: print(f" - {change}") @@ -450,9 +440,9 @@ def main(argv: list[str] | None = None) -> int: if args.check: print("\nRun with --apply to update dependency files") return 1 - - print("\n✓ Dependency files updated") - return 0 + else: + print("\n✓ Dependency files updated") + return 0 else: print("✓ All dev dependency versions are in sync") return 0 diff --git a/tests/test_historical_update.py b/tests/test_historical_update.py index fc3bac58..a1bccb73 100644 --- a/tests/test_historical_update.py +++ b/tests/test_historical_update.py @@ -640,9 +640,7 @@ def test_append_wal_row_preserves_existing_formulas_and_formatting(tmp_path: Pat sheet.cell(row=3, column=2).value = "=2.10" sheet.cell(row=3, column=2).number_format = "0.00" sheet.cell(row=3, column=2).font = styles.Font(bold=True) - sheet.cell(row=3, column=2).fill = styles.PatternFill( - patternType="solid", fgColor="FFFF00" - ) + sheet.cell(row=3, column=2).fill = styles.PatternFill(patternType="solid", fgColor="FFFF00") sheet.cell(row=3, column=2).border = styles.Border( left=styles.Side(style="thin"), right=styles.Side(style="thin"), diff --git a/tools/requirements-llm.txt b/tools/requirements-llm.txt index 1cfa17ca..d97f23c8 100644 --- a/tools/requirements-llm.txt +++ b/tools/requirements-llm.txt @@ -3,10 +3,10 @@ # - These are standalone runtime pins for workflow LLM steps, not app deps. # - When updating, coordinate with requirements.lock and pyproject.toml. # - Use strict X.Y.Z pins to keep workflow installs reproducible. -langchain==1.2.9 -langchain-core==1.2.11 +langchain==1.2.10 +langchain-core==1.2.13 langchain-community==0.4.1 -langchain-openai==1.1.7 -langchain-anthropic==1.3.2 +langchain-openai==1.1.9 +langchain-anthropic==1.3.3 pydantic==2.12.5 requests==2.32.5