diff --git a/.github/scripts/__tests__/keepalive-loop.test.js b/.github/scripts/__tests__/keepalive-loop.test.js index 299bf5d92..e0856e17f 100644 --- a/.github/scripts/__tests__/keepalive-loop.test.js +++ b/.github/scripts/__tests__/keepalive-loop.test.js @@ -704,6 +704,62 @@ test('updateKeepaliveLoopSummary increments iteration and clears failures on suc assert.match(github.actions[0].body, /"failure":\{\}/); }); +test('updateKeepaliveLoopSummary ignores status-only checklist metrics for reconciliation', async () => { + const pr = { + number: 1234, + labels: [{ name: 'agent:codex' }], + body: [ + '## Tasks', + '- [ ] Updated: 2026-04-26T12:33:27.204Z', + '- [ ] Repos checked: 11/11', + '- [ ] Open sync PRs: 439', + '', + '## Acceptance Criteria', + '- [ ] Acceptance criteria section missing from source issue.', + ].join('\n'), + }; + const existingState = formatStateComment({ + trace: 'status-only-trace', + iteration: 2, + max_iterations: 5, + tasks: { total: 13, unchecked: 13 }, + }); + const github = buildGithubStub({ + pr, + comments: [{ id: 91, body: existingState, html_url: 'https://example.com/91' }], + }); + + await updateKeepaliveLoopSummary({ + github, + context: buildContext(pr.number), + core: buildCore(), + inputs: { + prNumber: pr.number, + action: 'run', + runResult: 'success', + gateConclusion: 'success', + tasksTotal: 13, + tasksUnchecked: 13, + keepaliveEnabled: true, + autofixEnabled: false, + iteration: 2, + maxIterations: 5, + failureThreshold: 3, + trace: 'status-only-trace', + codex_changes_made: 'true', + codex_files_changed: 2, + codex_commit_sha: 'deadbeef', + codex_summary: 'Changed keepalive parser behavior for status metrics.', + }, + }); + + assert.equal(github.actions.length, 2); + const parsedState = parseStateComment(github.actions[0].body); + assert.ok(parsedState); + assert.deepEqual(parsedState.data.tasks, { total: 0, unchecked: 0 }); + assert.equal(parsedState.data.needs_task_reconciliation, false); +}); + test('updateKeepaliveLoopSummary reuses cached PR data for labels and body', async () => { const pr = { number: 321, @@ -2419,6 +2475,51 @@ test('buildTaskAppendix highlights attempted tasks and suggests next task', () = assert.ok(appendix.includes('- Task B')); }); +test('buildTaskAppendix does not suggest status-metric checklist items as next task', () => { + const { buildTaskAppendix } = require('../keepalive_loop.js'); + const sections = { + tasks: [ + '- [ ] Updated: 2026-04-26T12:33:27.204Z', + '- [ ] Repos checked: 11/11', + '- [ ] Open sync PRs: 439', + ].join('\n'), + acceptance: '- [ ] Acceptance criteria section missing from source issue.', + }; + const checkboxCounts = { total: 0, checked: 0, unchecked: 0 }; + + const appendix = buildTaskAppendix(sections, checkboxCounts, {}); + assert.ok(!appendix.includes('### Suggested Next Task')); +}); + +test('evaluateKeepaliveLoop stops with no-checklists when only status metrics/placeholders exist', async () => { + const pr = { + number: 113, + head: { ref: 'feature/status-metrics-only', sha: 'sha-13' }, + labels: [{ name: 'agent:codex' }], + body: [ + '## Tasks', + '- [ ] Updated: 2026-04-26T12:33:27.204Z', + '- [ ] Repos checked: 11/11', + '- [ ] Open sync PRs: 439', + '', + '## Acceptance Criteria', + '- [ ] Acceptance criteria section missing from source issue.', + ].join('\n'), + }; + const github = buildGithubStub({ + pr, + workflowRuns: [{ head_sha: 'sha-13', conclusion: 'success' }], + }); + const result = await evaluateKeepaliveLoop({ + github, + context: buildContext(pr.number), + core: buildCore(), + }); + assert.equal(result.action, 'stop'); + assert.equal(result.reason, 'no-checklists'); + assert.deepEqual(result.checkboxCounts, { total: 0, checked: 0, unchecked: 0 }); +}); + test('markAgentRunning updates summary comment with running status', async () => { // Use formatStateComment to create proper state marker const existingStateBody = formatStateComment({ diff --git a/.github/scripts/__tests__/terminal-disposition-coverage.test.js b/.github/scripts/__tests__/terminal-disposition-coverage.test.js index f1f822798..9e48e7354 100644 --- a/.github/scripts/__tests__/terminal-disposition-coverage.test.js +++ b/.github/scripts/__tests__/terminal-disposition-coverage.test.js @@ -253,8 +253,9 @@ test('summarizes verifier model compatibility with configurable unsupported mode ]); assert.equal( normalizeVerifierModelMetadataContract().required_after, - '2026-04-26T04:25:00Z' + '' ); + assert.equal(normalizeVerifierModelMetadataContract().model_metadata_required, false); const summary = summarizeVerifierModelCompatibility( [ @@ -288,27 +289,32 @@ test('summarizes verifier model compatibility with configurable unsupported mode }); test('warns when Codex verifier terminal records omit model metadata', () => { - const report = summarizeTerminalDispositionCoverage([ - { - schema: 'workflows-terminal-disposition/v1', - artifact_family: 'verifier-terminal-disposition', - source_type: 'pull-request', - source_id: '1872', - pr_number: 1872, - run_id: '24948023778', - disposition: 'verifier-error', - verifier_mode: 'compare', - }, + const report = summarizeTerminalDispositionCoverage( + [ + { + schema: 'workflows-terminal-disposition/v1', + artifact_family: 'verifier-terminal-disposition', + source_type: 'pull-request', + source_id: '1872', + pr_number: 1872, + run_id: '24948023778', + disposition: 'verifier-error', + verifier_mode: 'compare', + }, + { + schema: 'workflows-terminal-disposition/v1', + artifact_family: 'verifier-terminal-disposition', + source_type: 'pull-request', + source_id: '1873', + pr_number: 1873, + disposition: 'verified-pass', + verifier_mode: 'evaluate', + }, + ], { - schema: 'workflows-terminal-disposition/v1', - artifact_family: 'verifier-terminal-disposition', - source_type: 'pull-request', - source_id: '1873', - pr_number: 1873, - disposition: 'verified-pass', - verifier_mode: 'evaluate', - }, - ]); + model_metadata_required_after: '2026-04-26T04:25:00Z', + } + ); const markdown = formatTerminalDispositionCoverageMarkdown(report); assert.equal(report.status, 'warning'); @@ -317,13 +323,13 @@ test('warns when Codex verifier terminal records omit model metadata', () => { report.verifier_model_compatibility.missing_model_records.map((record) => record.source_key), ['pull-request:1872'] ); - assert.deepEqual(report.enforcement.blockers, ['unsupported-verifier-model']); + assert.deepEqual(report.enforcement.blockers, ['missing-verifier-model-metadata']); assert.match(markdown, /Missing verifier model metadata records: 1/); assert.match(markdown, /pull-request:1872/); assert.doesNotMatch(markdown, /\| pull-request:1873 \| verified-pass \| evaluate/); }); -test('warns when verifier terminal model metadata is missing with unknown mode', () => { +test('does not require verifier model metadata when mode is unknown', () => { const report = summarizeTerminalDispositionCoverage([ { schema: 'workflows-terminal-disposition/v1', @@ -337,11 +343,10 @@ test('warns when verifier terminal model metadata is missing with unknown mode', }, ]); - assert.equal(report.status, 'warning'); - assert.equal(report.verifier_model_compatibility.status, 'warning'); - assert.equal(report.verifier_model_compatibility.missing_model_record_count, 1); - assert.equal(report.verifier_model_compatibility.missing_model_records[0].verifier_mode, 'unknown'); - assert.deepEqual(report.enforcement.blockers, ['unsupported-verifier-model']); + assert.equal(report.status, 'pass'); + assert.equal(report.verifier_model_compatibility.status, 'pass'); + assert.equal(report.verifier_model_compatibility.missing_model_record_count, 0); + assert.deepEqual(report.enforcement.blockers, []); }); test('suppresses pre-contract verifier terminal records missing model metadata', () => { @@ -359,6 +364,7 @@ test('suppresses pre-contract verifier terminal records missing model metadata', }, ], { + model_metadata_required_after: '2026-04-26T04:25:00Z', input_file_count: 1, artifact_selection_report: { schema: 'workflows-weekly-metrics-artifact-selection/v1', @@ -400,6 +406,7 @@ test('still warns for post-contract verifier terminal records missing model meta }, ], { + model_metadata_required_after: '2026-04-26T04:25:00Z', input_file_count: 1, artifact_selection_report: { schema: 'workflows-weekly-metrics-artifact-selection/v1', @@ -420,7 +427,27 @@ test('still warns for post-contract verifier terminal records missing model meta assert.equal(report.verifier_model_compatibility.status, 'warning'); assert.equal(report.verifier_model_compatibility.missing_model_record_count, 1); assert.equal(report.verifier_model_compatibility.legacy_missing_model_record_count, 0); - assert.deepEqual(report.enforcement.blockers, ['unsupported-verifier-model']); + assert.deepEqual(report.enforcement.blockers, ['missing-verifier-model-metadata']); +}); + +test('leaves verifier model metadata checks disabled unless explicitly configured', () => { + const report = summarizeTerminalDispositionCoverage([ + { + schema: 'workflows-terminal-disposition/v1', + artifact_family: 'verifier-terminal-disposition', + source_type: 'pull-request', + source_id: '1877', + pr_number: 1877, + disposition: 'verifier-error', + verifier_mode: 'compare', + }, + ]); + + assert.equal(report.status, 'pass'); + assert.equal(report.verifier_model_compatibility.status, 'pass'); + assert.equal(report.verifier_model_compatibility.model_metadata_contract.model_metadata_required, false); + assert.equal(report.verifier_model_compatibility.missing_model_record_count, 0); + assert.deepEqual(report.enforcement.blockers, []); }); test('reads ndjson files and counts parse errors', () => { diff --git a/.github/scripts/__tests__/weekly-metrics-download-manifest.test.js b/.github/scripts/__tests__/weekly-metrics-download-manifest.test.js index 9fe5696f0..d778672cd 100644 --- a/.github/scripts/__tests__/weekly-metrics-download-manifest.test.js +++ b/.github/scripts/__tests__/weekly-metrics-download-manifest.test.js @@ -6,6 +6,7 @@ const { buildInitialManifest, finalizeManifest, formatMarkdown, + safeArtifactPathSegment, updateArtifactResult, } = require('../weekly_metrics_download_manifest.js'); @@ -125,6 +126,24 @@ test('does not fall back to name when artifact id mismatches', () => { ); }); +test('sanitizes artifact names used in extraction paths', () => { + const manifest = buildInitialManifest({ + ...selection, + selected_artifacts: [ + { + id: 44, + name: '../bad/name with spaces', + family: 'keepalive-metrics', + }, + ], + }); + + assert.equal(safeArtifactPathSegment('../bad/name with spaces'), '__bad_name_with_spaces'); + assert.equal(manifest.artifacts[0].name, '../bad/name with spaces'); + assert.equal(manifest.artifacts[0].artifact_dir, 'artifacts/__bad_name_with_spaces/44'); + assert.equal(manifest.artifacts[0].zip_path, 'artifacts/__bad_name_with_spaces/44/44.zip'); +}); + test('formats human-visible markdown without replacing the JSON contract', () => { const manifest = buildInitialManifest(selection); updateArtifactResult(manifest, { diff --git a/.github/scripts/keepalive_loop.js b/.github/scripts/keepalive_loop.js index 478c6a12b..2c876846e 100644 --- a/.github/scripts/keepalive_loop.js +++ b/.github/scripts/keepalive_loop.js @@ -1178,6 +1178,79 @@ function extractChecklistItems(markdown) { return items; } +const STATUS_METRIC_LABELS = new Set([ + 'updated', + 'repos checked', + 'open sync prs', + 'open dependabot prs', + 'active review threads queued', + 'items needing local codex', + 'actionable local codex items', + 'claimable local codex items', + 'source-fixed candidates', + 'superseded sync candidates', + 'finished local results without published source changes', + 'claimed local codex items', + 'next claim lease expires', +]); + +function normaliseChecklistText(value) { + return normaliseTaskText(value) + .replace(/`([^`]+)`/g, '$1') + .replace(/[*_~]/g, '') + .trim() + .toLowerCase(); +} + +function isStatusMetricChecklistItem(text) { + const normalized = normaliseChecklistText(text); + if (!normalized || !normalized.includes(':')) { + return false; + } + const [labelRaw, ...valueParts] = normalized.split(':'); + const label = labelRaw.trim(); + const value = valueParts.join(':').trim(); + if (!label || !value) { + return false; + } + if (!STATUS_METRIC_LABELS.has(label)) { + return false; + } + return true; +} + +function isPlaceholderChecklistItem(text) { + const normalized = normaliseChecklistText(text); + if (!normalized) { + return true; + } + if (normalized.includes('section missing from source issue')) { + return true; + } + if (/^no tasks defined\.?$/.test(normalized)) { + return true; + } + if (/^no acceptance criteria defined\.?$/.test(normalized)) { + return true; + } + return false; +} + +function isActionableChecklistItemText(text) { + return !isStatusMetricChecklistItem(text) && !isPlaceholderChecklistItem(text); +} + +function toActionableChecklistCounts(markdown) { + const actionable = extractChecklistItems(markdown).filter((item) => isActionableChecklistItemText(item.text)); + const checked = actionable.filter((item) => item.checked).length; + const total = actionable.length; + return { + total, + checked, + unchecked: Math.max(0, total - checked), + }; +} + /** * Extract file-path glob patterns from the scope section text. * Looks for patterns like `runtime/**`, `src/foo.py`, or backtick-quoted paths. @@ -1345,7 +1418,9 @@ function buildTaskAppendix(sections, checkboxCounts, state = {}, options = {}) { const attemptedTasks = normaliseAttemptedTasks(state?.attempted_tasks); const candidateSource = sections?.tasks || sections?.acceptance || ''; - const taskItems = extractChecklistItems(candidateSource); + const taskItems = extractChecklistItems(candidateSource).filter((item) => + isActionableChecklistItemText(item.text) + ); const unchecked = taskItems.filter((item) => !item.checked); const attemptedKeys = new Set(attemptedTasks.map((entry) => entry.key)); const suggested = unchecked.find((item) => !attemptedKeys.has(normaliseTaskKey(item.text))) || unchecked[0]; @@ -2283,7 +2358,7 @@ async function evaluateKeepaliveLoop({ github: rawGithub, context, core, payload const combinedChecklist = [normalisedSections?.tasks, normalisedSections?.acceptance] .filter(Boolean) .join('\n'); - const checkboxCounts = countCheckboxes(combinedChecklist); + const checkboxCounts = toActionableChecklistCounts(combinedChecklist); const tasksPresent = checkboxCounts.total > 0; const tasksRemaining = checkboxCounts.unchecked > 0; @@ -2291,8 +2366,8 @@ async function evaluateKeepaliveLoop({ github: rawGithub, context, core, payload // Tasks = what the agent works on (checkable by agent and auto-reconciliation). // Acceptance criteria = what must be independently verified (only verifier or agent). // The stop decision requires BOTH to be satisfied. - const taskCounts = countCheckboxes(normalisedSections?.tasks || ''); - const acceptanceCounts = countCheckboxes(normalisedSections?.acceptance || ''); + const taskCounts = toActionableChecklistCounts(normalisedSections?.tasks || ''); + const acceptanceCounts = toActionableChecklistCounts(normalisedSections?.acceptance || ''); const allTasksDone = taskCounts.total === 0 || taskCounts.unchecked === 0; const allCriteriaMet = acceptanceCounts.total === 0 || acceptanceCounts.unchecked === 0; const allComplete = tasksPresent && !tasksRemaining && allTasksDone && allCriteriaMet; @@ -2946,18 +3021,16 @@ async function updateKeepaliveLoopSummary({ github: rawGithub, context, core, in const liveCombined = [focusSections.tasks, focusSections.acceptance] .filter(Boolean) .join('\n'); - const liveCounts = countCheckboxes(liveCombined); - if (liveCounts.total > 0) { - const staleTotal = tasksTotal; - const staleUnchecked = tasksUnchecked; - tasksTotal = liveCounts.total; - tasksUnchecked = liveCounts.unchecked; - if (staleTotal !== tasksTotal || staleUnchecked !== tasksUnchecked) { - core?.info?.( - `[summary] Re-counted checkboxes from live PR body: ` + - `total ${staleTotal}→${tasksTotal}, unchecked ${staleUnchecked}→${tasksUnchecked}`, - ); - } + const liveCounts = toActionableChecklistCounts(liveCombined); + const staleTotal = tasksTotal; + const staleUnchecked = tasksUnchecked; + tasksTotal = liveCounts.total; + tasksUnchecked = liveCounts.unchecked; + if (staleTotal !== tasksTotal || staleUnchecked !== tasksUnchecked) { + core?.info?.( + `[summary] Re-counted actionable checkboxes from live PR body: ` + + `total ${staleTotal}→${tasksTotal}, unchecked ${staleUnchecked}→${tasksUnchecked}`, + ); } } @@ -3052,6 +3125,7 @@ async function updateKeepaliveLoopSummary({ github: rawGithub, context, core, in runResult === 'success' && agentChangesMade === 'true' && agentFilesChanged > 0 && + tasksTotal > 0 && tasksCompletedThisRound <= 0; if (action === 'run' || action === 'fix') { diff --git a/.github/scripts/terminal_disposition_coverage.js b/.github/scripts/terminal_disposition_coverage.js index 94550ced8..7706dd5cd 100644 --- a/.github/scripts/terminal_disposition_coverage.js +++ b/.github/scripts/terminal_disposition_coverage.js @@ -13,7 +13,7 @@ const TERMINAL_ARTIFACT_FAMILIES = new Set([ 'review-thread-terminal-disposition', ]); const DEFAULT_UNSUPPORTED_CODEX_MODELS = ['gpt-5.2-codex']; -const DEFAULT_VERIFIER_MODEL_METADATA_REQUIRED_AFTER = '2026-04-26T04:25:00Z'; +const DEFAULT_VERIFIER_MODEL_METADATA_REQUIRED_AFTER = ''; const DEFAULT_ENFORCEMENT_MODE = 'warning-only'; const HARD_BLOCK_MODE = 'hard-block'; @@ -97,6 +97,7 @@ function normalizeVerifierModelMetadataContract(value) { return { required_after: '', required_after_epoch_ms: null, + model_metadata_required: false, suppress_pre_contract_missing_metadata: false, }; } @@ -104,6 +105,7 @@ function normalizeVerifierModelMetadataContract(value) { return { required_after: text, required_after_epoch_ms: epochMs, + model_metadata_required: true, suppress_pre_contract_missing_metadata: epochMs !== null, }; } @@ -172,10 +174,10 @@ function summarizeVerifierModelCompatibility(records = [], options = {}) { const model = cleanString(record.llm_model ?? record.model).toLowerCase(); const reason = cleanString(record.model_selection_reason); const verifierMode = cleanString(record.verifier_mode).toLowerCase(); - const requiresCodexModel = verifierMode !== 'evaluate'; + const requiresCodexModel = Boolean(verifierMode) && verifierMode !== 'evaluate'; if (model) selectedModels[model] = (selectedModels[model] || 0) + 1; if (reason) modelSelectionReasons[reason] = (modelSelectionReasons[reason] || 0) + 1; - if (!model && requiresCodexModel) { + if (!model && requiresCodexModel && modelMetadataContract.model_metadata_required) { const runId = cleanString(record.run_id); const metadata = artifactMetadata.get(runId); const missingRecord = { @@ -357,9 +359,12 @@ function summarizeTerminalDispositionCoverage(records = [], options = {}) { if (missing.length > 0) enforcementBlockers.push('missing-review-thread-sources'); if (parseErrors > 0) enforcementBlockers.push('parse-errors'); if (artifactSelectionWarning) enforcementBlockers.push('artifact-selection-warning'); - if (verifierModelCompatibility.status !== 'pass') { + if (verifierModelCompatibility.unsupported_record_count > 0) { enforcementBlockers.push('unsupported-verifier-model'); } + if (verifierModelCompatibility.missing_model_record_count > 0) { + enforcementBlockers.push('missing-verifier-model-metadata'); + } const hardBlockEligible = enforcementBlockers.length === 0; const hardBlockActive = policy.effective_mode === HARD_BLOCK_MODE; diff --git a/.github/scripts/weekly_metrics_download_manifest.js b/.github/scripts/weekly_metrics_download_manifest.js index bcb61bb23..746e62881 100644 --- a/.github/scripts/weekly_metrics_download_manifest.js +++ b/.github/scripts/weekly_metrics_download_manifest.js @@ -26,12 +26,21 @@ function writeJsonFile(filePath, payload) { fs.writeFileSync(filePath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8'); } +function safeArtifactPathSegment(value) { + const sanitized = cleanString(value) + .replace(/[\\/]+/g, '_') + .replace(/[^A-Za-z0-9._-]/g, '_') + .replace(/\.\.+/g, '_') + .replace(/^\.{1,2}$/, '_'); + return sanitized || 'unknown'; +} + function selectedArtifactsFromSelection(selection = {}) { return Array.isArray(selection.selected_artifacts) ? selection.selected_artifacts : []; } function defaultArtifactDir(root, artifact) { - return path.posix.join(root, cleanString(artifact.name) || 'unknown', String(artifact.id || '')); + return path.posix.join(root, safeArtifactPathSegment(artifact.name), String(artifact.id || '')); } function defaultZipPath(root, artifact) { @@ -306,5 +315,6 @@ module.exports = { buildInitialManifest, finalizeManifest, formatMarkdown, + safeArtifactPathSegment, updateArtifactResult, }; diff --git a/.github/workflows/agents-weekly-metrics.yml b/.github/workflows/agents-weekly-metrics.yml index df0960f6f..19f10d70f 100644 --- a/.github/workflows/agents-weekly-metrics.yml +++ b/.github/workflows/agents-weekly-metrics.yml @@ -88,7 +88,8 @@ jobs: fi while IFS=$'\t' read -r id name; do echo "Downloading artifact $id ($name)" - artifact_dir="artifacts/$name/$id" + safe_name="$(node -e 'const name = process.argv[1] || ""; const safe = name.trim().replace(/[\\\\/]+/g, "_").replace(/[^A-Za-z0-9._-]/g, "_").replace(/\.\.+/g, "_").replace(/^\.{1,2}$/, "_") || "unknown"; process.stdout.write(safe);' "$name")" + artifact_dir="artifacts/$safe_name/$id" mkdir -p "$artifact_dir" # shellcheck disable=SC1009,SC1073,SC1039,SC1072 export ARTIFACT_ID="$id" diff --git a/scripts/aggregate_agent_metrics.py b/scripts/aggregate_agent_metrics.py index ff8f48c20..b9e086c78 100755 --- a/scripts/aggregate_agent_metrics.py +++ b/scripts/aggregate_agent_metrics.py @@ -19,7 +19,7 @@ _DEFAULT_JSON_OUTPUT = "agent-metrics-summary.json" _DEFAULT_DOWNLOAD_MANIFEST_PATH = "artifacts/metric-artifact-download-manifest.json" _DEFAULT_UNSUPPORTED_VERIFIER_MODELS = {"gpt-5.2-codex"} -_DEFAULT_VERIFIER_MODEL_METADATA_REQUIRED_AFTER = "2026-04-26T04:25:00Z" +_DEFAULT_VERIFIER_MODEL_METADATA_REQUIRED_AFTER = "" _EXACT_ARTIFACT_FAMILIES = { "keepalive-metrics", "agents-autofix-metrics", @@ -172,7 +172,6 @@ def _read_ndjson(files: Iterable[Path]) -> tuple[list[dict[str, Any]], list[Pars entries: list[dict[str, Any]] = [] errors: list[ParseErrorDetail] = [] for path in files: - source = _metric_source(path) try: handle = path.open("r", encoding="utf-8") except OSError: @@ -194,13 +193,7 @@ def _read_ndjson(files: Iterable[Path]) -> tuple[list[dict[str, Any]], list[Pars file_errors.append(_parse_error_detail(path, line_number, "invalid-json")) continue if isinstance(parsed, dict): - enriched = dict(parsed) - enriched.setdefault("artifact_name", source.artifact) - enriched.setdefault("artifact_family", source.artifact_family) - enriched.setdefault("metric_artifact", source.artifact) - enriched.setdefault("metric_artifact_family", source.artifact_family) - enriched.setdefault("metric_path", source.path) - file_entries.append(enriched) + file_entries.append(_attach_metric_source(parsed, path)) raw_lines_for_fallback = [] else: file_errors.append(_parse_error_detail(path, line_number, "non-object-json")) @@ -271,8 +264,16 @@ def _safe_int(value: Any) -> int | None: def _unsupported_verifier_models() -> set[str]: - raw = os.environ.get("UNSUPPORTED_VERIFIER_MODELS", "") - if not raw.strip(): + raw = "" + for env_name in ( + "UNSUPPORTED_VERIFIER_MODELS", + "TERMINAL_DISPOSITION_UNSUPPORTED_CODEX_MODELS", + ): + candidate = os.environ.get(env_name, "") + if candidate.strip(): + raw = candidate + break + if not raw: return set(_DEFAULT_UNSUPPORTED_VERIFIER_MODELS) return {item.strip().lower() for item in raw.split(",") if item.strip()} @@ -288,6 +289,15 @@ def _verifier_model_metadata_required_after() -> _dt.datetime | None: return _parse_timestamp(raw) +def _verifier_model_metadata_required() -> bool: + raw = ( + os.environ.get("TERMINAL_DISPOSITION_VERIFIER_MODEL_METADATA_REQUIRED_AFTER") + or os.environ.get("VERIFIER_MODEL_METADATA_REQUIRED_AFTER") + or _DEFAULT_VERIFIER_MODEL_METADATA_REQUIRED_AFTER + ).strip() + return raw.lower() not in {"", "0", "false", "none", "off", "disabled"} + + def _is_pre_contract_verifier_model_record( entry: dict[str, Any], required_after: _dt.datetime | None ) -> bool: @@ -394,6 +404,7 @@ def _summarise_verifier(entries: list[dict[str, Any]]) -> dict[str, Any]: unsupported_model_dispositions = Counter() missing_verifier_model_metadata = Counter() unsupported_models = _unsupported_verifier_models() + model_metadata_required = _verifier_model_metadata_required() model_metadata_required_after = _verifier_model_metadata_required_after() legacy_missing_verifier_model_metadata = Counter() verifier_modes = Counter() @@ -435,9 +446,9 @@ def _summarise_verifier(entries: list[dict[str, Any]]) -> dict[str, Any]: unsupported_verifier_models[normalized_model_text] += 1 disposition = entry.get("disposition") or entry.get("terminal_state") or "unknown" unsupported_model_dispositions[str(disposition)] += 1 - elif is_verifier_terminal: + elif is_verifier_terminal and model_metadata_required: verifier_mode = str(entry.get("verifier_mode") or "").strip().lower() - if verifier_mode != "evaluate": + if verifier_mode and verifier_mode != "evaluate": disposition = entry.get("disposition") or entry.get("terminal_state") or "unknown" if _is_pre_contract_verifier_model_record(entry, model_metadata_required_after): legacy_missing_verifier_model_metadata[str(disposition)] += 1 diff --git a/templates/consumer-repo/.github/scripts/keepalive_loop.js b/templates/consumer-repo/.github/scripts/keepalive_loop.js index 478c6a12b..2c876846e 100644 --- a/templates/consumer-repo/.github/scripts/keepalive_loop.js +++ b/templates/consumer-repo/.github/scripts/keepalive_loop.js @@ -1178,6 +1178,79 @@ function extractChecklistItems(markdown) { return items; } +const STATUS_METRIC_LABELS = new Set([ + 'updated', + 'repos checked', + 'open sync prs', + 'open dependabot prs', + 'active review threads queued', + 'items needing local codex', + 'actionable local codex items', + 'claimable local codex items', + 'source-fixed candidates', + 'superseded sync candidates', + 'finished local results without published source changes', + 'claimed local codex items', + 'next claim lease expires', +]); + +function normaliseChecklistText(value) { + return normaliseTaskText(value) + .replace(/`([^`]+)`/g, '$1') + .replace(/[*_~]/g, '') + .trim() + .toLowerCase(); +} + +function isStatusMetricChecklistItem(text) { + const normalized = normaliseChecklistText(text); + if (!normalized || !normalized.includes(':')) { + return false; + } + const [labelRaw, ...valueParts] = normalized.split(':'); + const label = labelRaw.trim(); + const value = valueParts.join(':').trim(); + if (!label || !value) { + return false; + } + if (!STATUS_METRIC_LABELS.has(label)) { + return false; + } + return true; +} + +function isPlaceholderChecklistItem(text) { + const normalized = normaliseChecklistText(text); + if (!normalized) { + return true; + } + if (normalized.includes('section missing from source issue')) { + return true; + } + if (/^no tasks defined\.?$/.test(normalized)) { + return true; + } + if (/^no acceptance criteria defined\.?$/.test(normalized)) { + return true; + } + return false; +} + +function isActionableChecklistItemText(text) { + return !isStatusMetricChecklistItem(text) && !isPlaceholderChecklistItem(text); +} + +function toActionableChecklistCounts(markdown) { + const actionable = extractChecklistItems(markdown).filter((item) => isActionableChecklistItemText(item.text)); + const checked = actionable.filter((item) => item.checked).length; + const total = actionable.length; + return { + total, + checked, + unchecked: Math.max(0, total - checked), + }; +} + /** * Extract file-path glob patterns from the scope section text. * Looks for patterns like `runtime/**`, `src/foo.py`, or backtick-quoted paths. @@ -1345,7 +1418,9 @@ function buildTaskAppendix(sections, checkboxCounts, state = {}, options = {}) { const attemptedTasks = normaliseAttemptedTasks(state?.attempted_tasks); const candidateSource = sections?.tasks || sections?.acceptance || ''; - const taskItems = extractChecklistItems(candidateSource); + const taskItems = extractChecklistItems(candidateSource).filter((item) => + isActionableChecklistItemText(item.text) + ); const unchecked = taskItems.filter((item) => !item.checked); const attemptedKeys = new Set(attemptedTasks.map((entry) => entry.key)); const suggested = unchecked.find((item) => !attemptedKeys.has(normaliseTaskKey(item.text))) || unchecked[0]; @@ -2283,7 +2358,7 @@ async function evaluateKeepaliveLoop({ github: rawGithub, context, core, payload const combinedChecklist = [normalisedSections?.tasks, normalisedSections?.acceptance] .filter(Boolean) .join('\n'); - const checkboxCounts = countCheckboxes(combinedChecklist); + const checkboxCounts = toActionableChecklistCounts(combinedChecklist); const tasksPresent = checkboxCounts.total > 0; const tasksRemaining = checkboxCounts.unchecked > 0; @@ -2291,8 +2366,8 @@ async function evaluateKeepaliveLoop({ github: rawGithub, context, core, payload // Tasks = what the agent works on (checkable by agent and auto-reconciliation). // Acceptance criteria = what must be independently verified (only verifier or agent). // The stop decision requires BOTH to be satisfied. - const taskCounts = countCheckboxes(normalisedSections?.tasks || ''); - const acceptanceCounts = countCheckboxes(normalisedSections?.acceptance || ''); + const taskCounts = toActionableChecklistCounts(normalisedSections?.tasks || ''); + const acceptanceCounts = toActionableChecklistCounts(normalisedSections?.acceptance || ''); const allTasksDone = taskCounts.total === 0 || taskCounts.unchecked === 0; const allCriteriaMet = acceptanceCounts.total === 0 || acceptanceCounts.unchecked === 0; const allComplete = tasksPresent && !tasksRemaining && allTasksDone && allCriteriaMet; @@ -2946,18 +3021,16 @@ async function updateKeepaliveLoopSummary({ github: rawGithub, context, core, in const liveCombined = [focusSections.tasks, focusSections.acceptance] .filter(Boolean) .join('\n'); - const liveCounts = countCheckboxes(liveCombined); - if (liveCounts.total > 0) { - const staleTotal = tasksTotal; - const staleUnchecked = tasksUnchecked; - tasksTotal = liveCounts.total; - tasksUnchecked = liveCounts.unchecked; - if (staleTotal !== tasksTotal || staleUnchecked !== tasksUnchecked) { - core?.info?.( - `[summary] Re-counted checkboxes from live PR body: ` + - `total ${staleTotal}→${tasksTotal}, unchecked ${staleUnchecked}→${tasksUnchecked}`, - ); - } + const liveCounts = toActionableChecklistCounts(liveCombined); + const staleTotal = tasksTotal; + const staleUnchecked = tasksUnchecked; + tasksTotal = liveCounts.total; + tasksUnchecked = liveCounts.unchecked; + if (staleTotal !== tasksTotal || staleUnchecked !== tasksUnchecked) { + core?.info?.( + `[summary] Re-counted actionable checkboxes from live PR body: ` + + `total ${staleTotal}→${tasksTotal}, unchecked ${staleUnchecked}→${tasksUnchecked}`, + ); } } @@ -3052,6 +3125,7 @@ async function updateKeepaliveLoopSummary({ github: rawGithub, context, core, in runResult === 'success' && agentChangesMade === 'true' && agentFilesChanged > 0 && + tasksTotal > 0 && tasksCompletedThisRound <= 0; if (action === 'run' || action === 'fix') { diff --git a/templates/consumer-repo/.github/scripts/terminal_disposition_coverage.js b/templates/consumer-repo/.github/scripts/terminal_disposition_coverage.js index 94550ced8..7706dd5cd 100644 --- a/templates/consumer-repo/.github/scripts/terminal_disposition_coverage.js +++ b/templates/consumer-repo/.github/scripts/terminal_disposition_coverage.js @@ -13,7 +13,7 @@ const TERMINAL_ARTIFACT_FAMILIES = new Set([ 'review-thread-terminal-disposition', ]); const DEFAULT_UNSUPPORTED_CODEX_MODELS = ['gpt-5.2-codex']; -const DEFAULT_VERIFIER_MODEL_METADATA_REQUIRED_AFTER = '2026-04-26T04:25:00Z'; +const DEFAULT_VERIFIER_MODEL_METADATA_REQUIRED_AFTER = ''; const DEFAULT_ENFORCEMENT_MODE = 'warning-only'; const HARD_BLOCK_MODE = 'hard-block'; @@ -97,6 +97,7 @@ function normalizeVerifierModelMetadataContract(value) { return { required_after: '', required_after_epoch_ms: null, + model_metadata_required: false, suppress_pre_contract_missing_metadata: false, }; } @@ -104,6 +105,7 @@ function normalizeVerifierModelMetadataContract(value) { return { required_after: text, required_after_epoch_ms: epochMs, + model_metadata_required: true, suppress_pre_contract_missing_metadata: epochMs !== null, }; } @@ -172,10 +174,10 @@ function summarizeVerifierModelCompatibility(records = [], options = {}) { const model = cleanString(record.llm_model ?? record.model).toLowerCase(); const reason = cleanString(record.model_selection_reason); const verifierMode = cleanString(record.verifier_mode).toLowerCase(); - const requiresCodexModel = verifierMode !== 'evaluate'; + const requiresCodexModel = Boolean(verifierMode) && verifierMode !== 'evaluate'; if (model) selectedModels[model] = (selectedModels[model] || 0) + 1; if (reason) modelSelectionReasons[reason] = (modelSelectionReasons[reason] || 0) + 1; - if (!model && requiresCodexModel) { + if (!model && requiresCodexModel && modelMetadataContract.model_metadata_required) { const runId = cleanString(record.run_id); const metadata = artifactMetadata.get(runId); const missingRecord = { @@ -357,9 +359,12 @@ function summarizeTerminalDispositionCoverage(records = [], options = {}) { if (missing.length > 0) enforcementBlockers.push('missing-review-thread-sources'); if (parseErrors > 0) enforcementBlockers.push('parse-errors'); if (artifactSelectionWarning) enforcementBlockers.push('artifact-selection-warning'); - if (verifierModelCompatibility.status !== 'pass') { + if (verifierModelCompatibility.unsupported_record_count > 0) { enforcementBlockers.push('unsupported-verifier-model'); } + if (verifierModelCompatibility.missing_model_record_count > 0) { + enforcementBlockers.push('missing-verifier-model-metadata'); + } const hardBlockEligible = enforcementBlockers.length === 0; const hardBlockActive = policy.effective_mode === HARD_BLOCK_MODE; diff --git a/templates/consumer-repo/.github/scripts/weekly_metrics_download_manifest.js b/templates/consumer-repo/.github/scripts/weekly_metrics_download_manifest.js index bcb61bb23..746e62881 100644 --- a/templates/consumer-repo/.github/scripts/weekly_metrics_download_manifest.js +++ b/templates/consumer-repo/.github/scripts/weekly_metrics_download_manifest.js @@ -26,12 +26,21 @@ function writeJsonFile(filePath, payload) { fs.writeFileSync(filePath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8'); } +function safeArtifactPathSegment(value) { + const sanitized = cleanString(value) + .replace(/[\\/]+/g, '_') + .replace(/[^A-Za-z0-9._-]/g, '_') + .replace(/\.\.+/g, '_') + .replace(/^\.{1,2}$/, '_'); + return sanitized || 'unknown'; +} + function selectedArtifactsFromSelection(selection = {}) { return Array.isArray(selection.selected_artifacts) ? selection.selected_artifacts : []; } function defaultArtifactDir(root, artifact) { - return path.posix.join(root, cleanString(artifact.name) || 'unknown', String(artifact.id || '')); + return path.posix.join(root, safeArtifactPathSegment(artifact.name), String(artifact.id || '')); } function defaultZipPath(root, artifact) { @@ -306,5 +315,6 @@ module.exports = { buildInitialManifest, finalizeManifest, formatMarkdown, + safeArtifactPathSegment, updateArtifactResult, }; diff --git a/templates/consumer-repo/.github/workflows/agents-weekly-metrics.yml b/templates/consumer-repo/.github/workflows/agents-weekly-metrics.yml index fce9ea89d..827056e83 100644 --- a/templates/consumer-repo/.github/workflows/agents-weekly-metrics.yml +++ b/templates/consumer-repo/.github/workflows/agents-weekly-metrics.yml @@ -99,7 +99,8 @@ jobs: fi while IFS=$'\t' read -r id name; do echo "Downloading artifact $id ($name)" - artifact_dir="artifacts/$name/$id" + safe_name="$(node -e 'const name = process.argv[1] || ""; const safe = name.trim().replace(/[\\\\/]+/g, "_").replace(/[^A-Za-z0-9._-]/g, "_").replace(/\.\.+/g, "_").replace(/^\.{1,2}$/, "_") || "unknown"; process.stdout.write(safe);' "$name")" + artifact_dir="artifacts/$safe_name/$id" mkdir -p "$artifact_dir" # shellcheck disable=SC1009,SC1073,SC1039,SC1072 export ARTIFACT_ID="$id" diff --git a/templates/consumer-repo/scripts/aggregate_agent_metrics.py b/templates/consumer-repo/scripts/aggregate_agent_metrics.py index ff8f48c20..b9e086c78 100755 --- a/templates/consumer-repo/scripts/aggregate_agent_metrics.py +++ b/templates/consumer-repo/scripts/aggregate_agent_metrics.py @@ -19,7 +19,7 @@ _DEFAULT_JSON_OUTPUT = "agent-metrics-summary.json" _DEFAULT_DOWNLOAD_MANIFEST_PATH = "artifacts/metric-artifact-download-manifest.json" _DEFAULT_UNSUPPORTED_VERIFIER_MODELS = {"gpt-5.2-codex"} -_DEFAULT_VERIFIER_MODEL_METADATA_REQUIRED_AFTER = "2026-04-26T04:25:00Z" +_DEFAULT_VERIFIER_MODEL_METADATA_REQUIRED_AFTER = "" _EXACT_ARTIFACT_FAMILIES = { "keepalive-metrics", "agents-autofix-metrics", @@ -172,7 +172,6 @@ def _read_ndjson(files: Iterable[Path]) -> tuple[list[dict[str, Any]], list[Pars entries: list[dict[str, Any]] = [] errors: list[ParseErrorDetail] = [] for path in files: - source = _metric_source(path) try: handle = path.open("r", encoding="utf-8") except OSError: @@ -194,13 +193,7 @@ def _read_ndjson(files: Iterable[Path]) -> tuple[list[dict[str, Any]], list[Pars file_errors.append(_parse_error_detail(path, line_number, "invalid-json")) continue if isinstance(parsed, dict): - enriched = dict(parsed) - enriched.setdefault("artifact_name", source.artifact) - enriched.setdefault("artifact_family", source.artifact_family) - enriched.setdefault("metric_artifact", source.artifact) - enriched.setdefault("metric_artifact_family", source.artifact_family) - enriched.setdefault("metric_path", source.path) - file_entries.append(enriched) + file_entries.append(_attach_metric_source(parsed, path)) raw_lines_for_fallback = [] else: file_errors.append(_parse_error_detail(path, line_number, "non-object-json")) @@ -271,8 +264,16 @@ def _safe_int(value: Any) -> int | None: def _unsupported_verifier_models() -> set[str]: - raw = os.environ.get("UNSUPPORTED_VERIFIER_MODELS", "") - if not raw.strip(): + raw = "" + for env_name in ( + "UNSUPPORTED_VERIFIER_MODELS", + "TERMINAL_DISPOSITION_UNSUPPORTED_CODEX_MODELS", + ): + candidate = os.environ.get(env_name, "") + if candidate.strip(): + raw = candidate + break + if not raw: return set(_DEFAULT_UNSUPPORTED_VERIFIER_MODELS) return {item.strip().lower() for item in raw.split(",") if item.strip()} @@ -288,6 +289,15 @@ def _verifier_model_metadata_required_after() -> _dt.datetime | None: return _parse_timestamp(raw) +def _verifier_model_metadata_required() -> bool: + raw = ( + os.environ.get("TERMINAL_DISPOSITION_VERIFIER_MODEL_METADATA_REQUIRED_AFTER") + or os.environ.get("VERIFIER_MODEL_METADATA_REQUIRED_AFTER") + or _DEFAULT_VERIFIER_MODEL_METADATA_REQUIRED_AFTER + ).strip() + return raw.lower() not in {"", "0", "false", "none", "off", "disabled"} + + def _is_pre_contract_verifier_model_record( entry: dict[str, Any], required_after: _dt.datetime | None ) -> bool: @@ -394,6 +404,7 @@ def _summarise_verifier(entries: list[dict[str, Any]]) -> dict[str, Any]: unsupported_model_dispositions = Counter() missing_verifier_model_metadata = Counter() unsupported_models = _unsupported_verifier_models() + model_metadata_required = _verifier_model_metadata_required() model_metadata_required_after = _verifier_model_metadata_required_after() legacy_missing_verifier_model_metadata = Counter() verifier_modes = Counter() @@ -435,9 +446,9 @@ def _summarise_verifier(entries: list[dict[str, Any]]) -> dict[str, Any]: unsupported_verifier_models[normalized_model_text] += 1 disposition = entry.get("disposition") or entry.get("terminal_state") or "unknown" unsupported_model_dispositions[str(disposition)] += 1 - elif is_verifier_terminal: + elif is_verifier_terminal and model_metadata_required: verifier_mode = str(entry.get("verifier_mode") or "").strip().lower() - if verifier_mode != "evaluate": + if verifier_mode and verifier_mode != "evaluate": disposition = entry.get("disposition") or entry.get("terminal_state") or "unknown" if _is_pre_contract_verifier_model_record(entry, model_metadata_required_after): legacy_missing_verifier_model_metadata[str(disposition)] += 1 diff --git a/tests/scripts/test_aggregate_agent_metrics.py b/tests/scripts/test_aggregate_agent_metrics.py index 303da413b..14997a540 100644 --- a/tests/scripts/test_aggregate_agent_metrics.py +++ b/tests/scripts/test_aggregate_agent_metrics.py @@ -563,7 +563,66 @@ def test_verifier_summary_counts_unsupported_models( assert "Unsupported model dispositions: verifier-error (1)" in summary -def test_verifier_summary_counts_missing_model_metadata() -> None: +def test_verifier_summary_accepts_terminal_disposition_unsupported_model_alias( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("UNSUPPORTED_VERIFIER_MODELS", raising=False) + monkeypatch.setenv("TERMINAL_DISPOSITION_UNSUPPORTED_CODEX_MODELS", "alias-bad") + + verifier = aggregate_agent_metrics._summarise_verifier( + [ + { + "schema": "workflows-terminal-disposition/v1", + "artifact_family": "verifier-terminal-disposition", + "run_id": "24948023778", + "pr_number": 1872, + "disposition": "verifier-error", + "llm_model": "Alias-Bad", + }, + ] + ) + + assert verifier["unsupported_verifier_models"]["alias-bad"] == 1 + + +def test_verifier_summary_prefers_specific_unsupported_model_env( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("UNSUPPORTED_VERIFIER_MODELS", "primary-bad") + monkeypatch.setenv("TERMINAL_DISPOSITION_UNSUPPORTED_CODEX_MODELS", "alias-bad") + + verifier = aggregate_agent_metrics._summarise_verifier( + [ + { + "schema": "workflows-terminal-disposition/v1", + "artifact_family": "verifier-terminal-disposition", + "run_id": "24948023778", + "pr_number": 1872, + "disposition": "verifier-error", + "llm_model": "primary-bad", + }, + { + "schema": "workflows-terminal-disposition/v1", + "artifact_family": "verifier-terminal-disposition", + "run_id": "24948023779", + "pr_number": 1873, + "disposition": "verifier-error", + "llm_model": "alias-bad", + }, + ] + ) + + assert verifier["unsupported_verifier_models"]["primary-bad"] == 1 + assert verifier["unsupported_verifier_models"]["alias-bad"] == 0 + + +def test_verifier_summary_counts_missing_model_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv( + "TERMINAL_DISPOSITION_VERIFIER_MODEL_METADATA_REQUIRED_AFTER", + "2026-04-26T04:25:00Z", + ) summary = aggregate_agent_metrics.build_summary( [ { @@ -572,6 +631,7 @@ def test_verifier_summary_counts_missing_model_metadata() -> None: "run_id": "24948023778", "pr_number": 1872, "disposition": "verifier-error", + "verifier_mode": "compare", }, { "schema": "workflows-terminal-disposition/v1", @@ -587,7 +647,13 @@ def test_verifier_summary_counts_missing_model_metadata() -> None: assert "Missing verifier model metadata: verifier-error (1)" in summary -def test_verifier_summary_suppresses_pre_contract_missing_model_metadata() -> None: +def test_verifier_summary_suppresses_pre_contract_missing_model_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv( + "TERMINAL_DISPOSITION_VERIFIER_MODEL_METADATA_REQUIRED_AFTER", + "2026-04-26T04:25:00Z", + ) summary = aggregate_agent_metrics.build_summary( [ { @@ -597,6 +663,7 @@ def test_verifier_summary_suppresses_pre_contract_missing_model_metadata() -> No "run_id": "24948023778", "pr_number": 1872, "disposition": "verifier-error", + "verifier_mode": "compare", }, { "schema": "workflows-terminal-disposition/v1", @@ -605,6 +672,7 @@ def test_verifier_summary_suppresses_pre_contract_missing_model_metadata() -> No "run_id": "24950000000", "pr_number": 1877, "disposition": "needs-human", + "verifier_mode": "compare", }, ], errors=0, @@ -614,7 +682,13 @@ def test_verifier_summary_suppresses_pre_contract_missing_model_metadata() -> No assert "Legacy missing verifier model metadata: verifier-error (1)" in summary -def test_verifier_summary_ignores_review_thread_terminal_model_metadata() -> None: +def test_verifier_summary_ignores_review_thread_terminal_model_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv( + "TERMINAL_DISPOSITION_VERIFIER_MODEL_METADATA_REQUIRED_AFTER", + "2026-04-26T04:25:00Z", + ) summary = aggregate_agent_metrics.build_summary( [ { @@ -632,6 +706,7 @@ def test_verifier_summary_ignores_review_thread_terminal_model_metadata() -> Non "source_id": "1872", "pr_number": 1872, "disposition": "verifier-error", + "verifier_mode": "compare", }, ], errors=0, @@ -644,6 +719,24 @@ def test_verifier_summary_ignores_review_thread_terminal_model_metadata() -> Non assert "wrapper-skipped" not in summary.split("Missing verifier model metadata: ", 1)[1] +def test_verifier_summary_does_not_require_model_metadata_by_default() -> None: + summary = aggregate_agent_metrics.build_summary( + [ + { + "schema": "workflows-terminal-disposition/v1", + "artifact_family": "verifier-terminal-disposition", + "run_id": "24948023778", + "pr_number": 1872, + "disposition": "verifier-error", + "verifier_mode": "compare", + } + ], + errors=0, + ) + + assert "Missing verifier model metadata: n/a" in summary + + def test_format_helpers_and_summary_range() -> None: assert aggregate_agent_metrics._format_counter(Counter()) == "n/a" assert aggregate_agent_metrics._format_rate(1, 0) == "n/a" diff --git a/tests/workflows/test_workflow_agents_consolidation.py b/tests/workflows/test_workflow_agents_consolidation.py index 8d152b4af..80ddc8f3c 100644 --- a/tests/workflows/test_workflow_agents_consolidation.py +++ b/tests/workflows/test_workflow_agents_consolidation.py @@ -295,8 +295,8 @@ def test_weekly_metrics_uploads_selector_report_on_failure(): and "agent-weekly-metrics.json" in text ), "Weekly metrics must upload a machine-readable aggregate summary" assert ( - 'artifact_dir="artifacts/$name/$id"' in text - ), "Weekly metrics must isolate same-name artifact downloads by artifact ID" + 'artifact_dir="artifacts/$safe_name/$id"' in text and "safe_name=" in text + ), "Weekly metrics must sanitize artifact paths and isolate downloads by artifact ID" assert ( 'export ARTIFACT_ZIP="$artifact_dir/$id.zip"' in text and 'unzip -o "$ARTIFACT_ZIP" -d "$ARTIFACT_DIR"' in text