diff --git a/.github/workflows/agents-verify-to-new-pr.yml b/.github/workflows/agents-verify-to-new-pr.yml index e921bd8d0..16b5bd441 100644 --- a/.github/workflows/agents-verify-to-new-pr.yml +++ b/.github/workflows/agents-verify-to-new-pr.yml @@ -32,7 +32,7 @@ jobs: steps: - name: Check PR is merged id: check-merged - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 with: github-token: ${{ github.token }} script: | @@ -66,7 +66,7 @@ jobs: - name: Checkout repository if: steps.check-merged.outputs.merged == 'true' - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: repository: stranske/Workflows token: ${{ steps.select-token.outputs.token }} @@ -91,7 +91,7 @@ jobs: - name: Set up Python if: steps.check-merged.outputs.merged == 'true' - uses: actions/setup-python@v6 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: ${{ env.PYTHON_VERSION }} @@ -103,7 +103,7 @@ jobs: - name: Collect verification and original issue data id: collect if: steps.check-merged.outputs.merged == 'true' - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 with: github-token: ${{ steps.select-token.outputs.token }} script: | @@ -248,7 +248,7 @@ jobs: - name: Check chain depth limit id: chain-check if: steps.check-merged.outputs.merged == 'true' - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: CHAIN_DEPTH: ${{ steps.collect.outputs.chain_depth }} MAX_CHAIN_DEPTH: '2' @@ -389,7 +389,7 @@ jobs: steps.check-merged.outputs.merged == 'true' && steps.chain-check.outputs.exceeded != 'true' && steps.extract-verdict.outputs.needs_human == 'true' - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: LINKED_ISSUE: ${{ steps.collect.outputs.original_issue_number }} NEEDS_HUMAN_REASON: ${{ steps.extract-verdict.outputs.needs_human_reason }} @@ -505,7 +505,7 @@ jobs: steps.check-merged.outputs.merged == 'true' && steps.chain-check.outputs.exceeded != 'true' && steps.extract-verdict.outputs.needs_human != 'true' - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: FOLLOW_UP_DEPTH: ${{ steps.chain-check.outputs.next_depth }} EXTRACTED_VERDICT: ${{ steps.extract-verdict.outputs.verdict }} @@ -677,7 +677,7 @@ jobs: steps.check-merged.outputs.merged == 'true' && steps.chain-check.outputs.exceeded != 'true' && steps.extract-verdict.outputs.needs_human != 'true' - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: ISSUE_TITLE: >- ${{ steps.generate.outputs.issue_title || @@ -740,7 +740,8 @@ jobs: let _defaultAgent = 'codex'; try { const { loadAgentRegistry } = require('./.github/scripts/agent_registry.js'); - _defaultAgent = loadAgentRegistry('./.github/agents/registry.yml').default_agent || 'codex'; + const registry = loadAgentRegistry('./.github/agents/registry.yml'); + _defaultAgent = registry.default_agent || 'codex'; } catch (_) {} let agentKey = _defaultAgent; try { @@ -752,11 +753,13 @@ jobs: }); } catch (err) { core.warning( - `Failed to resolve agent from PR labels; defaulting to ${_defaultAgent}: ${err.message}` + 'Failed to resolve agent from PR labels; ' + + `defaulting to ${_defaultAgent}: ${err.message}` ); agentKey = _defaultAgent; } - const normalized = String(agentKey || _defaultAgent).trim().toLowerCase() || _defaultAgent; + const normalized = + String(agentKey || _defaultAgent).trim().toLowerCase() || _defaultAgent; const agentLabel = `agent:${normalized}`; const fromLabel = `from:${normalized}`; @@ -765,7 +768,13 @@ jobs: repo: context.repo.repo, title: title, body: body, - labels: ['follow-up', agentLabel, fromLabel, `runner:${normalized}`, 'agents:auto-pilot'] + labels: [ + 'follow-up', + agentLabel, + fromLabel, + `runner:${normalized}`, + 'agents:auto-pilot' + ] })); core.info(`Created issue #${issue.data.number}`); @@ -779,7 +788,7 @@ jobs: steps.chain-check.outputs.exceeded != 'true' && steps.extract-verdict.outputs.needs_human != 'true' && steps.create-issue.outputs.issue_number != '' - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: ISSUE_NUMBER: ${{ steps.create-issue.outputs.issue_number }} DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} @@ -816,7 +825,7 @@ jobs: steps.check-merged.outputs.merged == 'true' && steps.chain-check.outputs.exceeded != 'true' && steps.extract-verdict.outputs.needs_human != 'true' - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: ISSUE_NUMBER: ${{ steps.create-issue.outputs.issue_number }} ISSUE_URL: ${{ steps.create-issue.outputs.issue_url }} @@ -870,7 +879,7 @@ jobs: FOLLOWUP_ISSUE_NUMBER: ${{ steps.create-issue.outputs.issue_number }} FOLLOWUP_ISSUE_URL: ${{ steps.create-issue.outputs.issue_url }} DISPATCH_AUTOPILOT_OUTCOME: ${{ steps.dispatch-autopilot.outcome }} - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 with: github-token: ${{ steps.select-token.outputs.token }} script: | @@ -923,13 +932,17 @@ jobs: reason = 'Verification follow-up issue was created and auto-pilot was dispatched.'; policyReason = reason; } else if (dispatchOutcome === 'failure' || dispatchOutcome === 'cancelled') { - reason = `Verification follow-up issue was created, but auto-pilot dispatch ${dispatchOutcome}.`; + reason = + 'Verification follow-up issue was created, but ' + + `auto-pilot dispatch ${dispatchOutcome}.`; policyReason = reason; } else if (dispatchOutcome === 'skipped') { reason = 'Verification follow-up issue was created; auto-pilot dispatch was skipped.'; policyReason = reason; } else if (followupIssue) { - reason = 'Verification follow-up issue was created; auto-pilot dispatch status was not reported.'; + reason = + 'Verification follow-up issue was created; ' + + 'auto-pilot dispatch status was not reported.'; policyReason = reason; } @@ -1006,7 +1019,7 @@ jobs: - name: Upload terminal disposition artifact if: always() && steps.check-merged.outputs.merged == 'true' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@65ecb0ca2d3e252f7b82842cd0489c883189f7d0 # v7 with: name: verifier-terminal-disposition-${{ github.run_id }} path: | @@ -1017,7 +1030,7 @@ jobs: - name: Remove trigger label if: steps.check-merged.outputs.merged == 'true' - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 continue-on-error: true with: github-token: ${{ steps.select-token.outputs.token }} diff --git a/scripts/aggregate_agent_metrics.py b/scripts/aggregate_agent_metrics.py index 8dcd793a1..be3081034 100755 --- a/scripts/aggregate_agent_metrics.py +++ b/scripts/aggregate_agent_metrics.py @@ -47,6 +47,8 @@ _MAX_STORED_PARSE_ERROR_DETAILS = 250 _MAX_LEGACY_JSON_FALLBACK_LINES = 5000 _MAX_LEGACY_JSON_FALLBACK_BYTES = 1024 * 1024 +_PARSE_ERROR_DETAIL_LIMIT_REASON = "detail-limit-exceeded" +_PARSE_ERROR_DETAIL_LIMIT_VALUE = "__multiple__" @dataclass(frozen=True) @@ -179,6 +181,8 @@ def _append_parse_error_detail( *, detail_limit: int = _MAX_STORED_PARSE_ERROR_DETAILS, ) -> None: + if detail_limit <= 0: + return if len(details) < detail_limit: details.append(detail) return @@ -194,7 +198,20 @@ def _append_parse_error_detail( details[index] = replace(existing, count=existing.count + detail.count) return - details.append(replace(detail, line=None)) + for index, existing in enumerate(details): + if existing.reason == _PARSE_ERROR_DETAIL_LIMIT_REASON and existing.line is None: + details[index] = replace(existing, count=existing.count + detail.count) + return + + evicted = details[-1] + details[-1] = ParseErrorDetail( + path=_PARSE_ERROR_DETAIL_LIMIT_VALUE, + artifact=_PARSE_ERROR_DETAIL_LIMIT_VALUE, + artifact_family=_PARSE_ERROR_DETAIL_LIMIT_VALUE, + line=None, + reason=_PARSE_ERROR_DETAIL_LIMIT_REASON, + count=evicted.count + detail.count, + ) def _parse_error_count(parse_error_details: list[ParseErrorDetail]) -> int: diff --git a/templates/consumer-repo/.github/workflows/agents-verify-to-new-pr.yml b/templates/consumer-repo/.github/workflows/agents-verify-to-new-pr.yml index e921bd8d0..16b5bd441 100644 --- a/templates/consumer-repo/.github/workflows/agents-verify-to-new-pr.yml +++ b/templates/consumer-repo/.github/workflows/agents-verify-to-new-pr.yml @@ -32,7 +32,7 @@ jobs: steps: - name: Check PR is merged id: check-merged - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 with: github-token: ${{ github.token }} script: | @@ -66,7 +66,7 @@ jobs: - name: Checkout repository if: steps.check-merged.outputs.merged == 'true' - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: repository: stranske/Workflows token: ${{ steps.select-token.outputs.token }} @@ -91,7 +91,7 @@ jobs: - name: Set up Python if: steps.check-merged.outputs.merged == 'true' - uses: actions/setup-python@v6 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: ${{ env.PYTHON_VERSION }} @@ -103,7 +103,7 @@ jobs: - name: Collect verification and original issue data id: collect if: steps.check-merged.outputs.merged == 'true' - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 with: github-token: ${{ steps.select-token.outputs.token }} script: | @@ -248,7 +248,7 @@ jobs: - name: Check chain depth limit id: chain-check if: steps.check-merged.outputs.merged == 'true' - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: CHAIN_DEPTH: ${{ steps.collect.outputs.chain_depth }} MAX_CHAIN_DEPTH: '2' @@ -389,7 +389,7 @@ jobs: steps.check-merged.outputs.merged == 'true' && steps.chain-check.outputs.exceeded != 'true' && steps.extract-verdict.outputs.needs_human == 'true' - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: LINKED_ISSUE: ${{ steps.collect.outputs.original_issue_number }} NEEDS_HUMAN_REASON: ${{ steps.extract-verdict.outputs.needs_human_reason }} @@ -505,7 +505,7 @@ jobs: steps.check-merged.outputs.merged == 'true' && steps.chain-check.outputs.exceeded != 'true' && steps.extract-verdict.outputs.needs_human != 'true' - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: FOLLOW_UP_DEPTH: ${{ steps.chain-check.outputs.next_depth }} EXTRACTED_VERDICT: ${{ steps.extract-verdict.outputs.verdict }} @@ -677,7 +677,7 @@ jobs: steps.check-merged.outputs.merged == 'true' && steps.chain-check.outputs.exceeded != 'true' && steps.extract-verdict.outputs.needs_human != 'true' - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: ISSUE_TITLE: >- ${{ steps.generate.outputs.issue_title || @@ -740,7 +740,8 @@ jobs: let _defaultAgent = 'codex'; try { const { loadAgentRegistry } = require('./.github/scripts/agent_registry.js'); - _defaultAgent = loadAgentRegistry('./.github/agents/registry.yml').default_agent || 'codex'; + const registry = loadAgentRegistry('./.github/agents/registry.yml'); + _defaultAgent = registry.default_agent || 'codex'; } catch (_) {} let agentKey = _defaultAgent; try { @@ -752,11 +753,13 @@ jobs: }); } catch (err) { core.warning( - `Failed to resolve agent from PR labels; defaulting to ${_defaultAgent}: ${err.message}` + 'Failed to resolve agent from PR labels; ' + + `defaulting to ${_defaultAgent}: ${err.message}` ); agentKey = _defaultAgent; } - const normalized = String(agentKey || _defaultAgent).trim().toLowerCase() || _defaultAgent; + const normalized = + String(agentKey || _defaultAgent).trim().toLowerCase() || _defaultAgent; const agentLabel = `agent:${normalized}`; const fromLabel = `from:${normalized}`; @@ -765,7 +768,13 @@ jobs: repo: context.repo.repo, title: title, body: body, - labels: ['follow-up', agentLabel, fromLabel, `runner:${normalized}`, 'agents:auto-pilot'] + labels: [ + 'follow-up', + agentLabel, + fromLabel, + `runner:${normalized}`, + 'agents:auto-pilot' + ] })); core.info(`Created issue #${issue.data.number}`); @@ -779,7 +788,7 @@ jobs: steps.chain-check.outputs.exceeded != 'true' && steps.extract-verdict.outputs.needs_human != 'true' && steps.create-issue.outputs.issue_number != '' - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: ISSUE_NUMBER: ${{ steps.create-issue.outputs.issue_number }} DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} @@ -816,7 +825,7 @@ jobs: steps.check-merged.outputs.merged == 'true' && steps.chain-check.outputs.exceeded != 'true' && steps.extract-verdict.outputs.needs_human != 'true' - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: ISSUE_NUMBER: ${{ steps.create-issue.outputs.issue_number }} ISSUE_URL: ${{ steps.create-issue.outputs.issue_url }} @@ -870,7 +879,7 @@ jobs: FOLLOWUP_ISSUE_NUMBER: ${{ steps.create-issue.outputs.issue_number }} FOLLOWUP_ISSUE_URL: ${{ steps.create-issue.outputs.issue_url }} DISPATCH_AUTOPILOT_OUTCOME: ${{ steps.dispatch-autopilot.outcome }} - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 with: github-token: ${{ steps.select-token.outputs.token }} script: | @@ -923,13 +932,17 @@ jobs: reason = 'Verification follow-up issue was created and auto-pilot was dispatched.'; policyReason = reason; } else if (dispatchOutcome === 'failure' || dispatchOutcome === 'cancelled') { - reason = `Verification follow-up issue was created, but auto-pilot dispatch ${dispatchOutcome}.`; + reason = + 'Verification follow-up issue was created, but ' + + `auto-pilot dispatch ${dispatchOutcome}.`; policyReason = reason; } else if (dispatchOutcome === 'skipped') { reason = 'Verification follow-up issue was created; auto-pilot dispatch was skipped.'; policyReason = reason; } else if (followupIssue) { - reason = 'Verification follow-up issue was created; auto-pilot dispatch status was not reported.'; + reason = + 'Verification follow-up issue was created; ' + + 'auto-pilot dispatch status was not reported.'; policyReason = reason; } @@ -1006,7 +1019,7 @@ jobs: - name: Upload terminal disposition artifact if: always() && steps.check-merged.outputs.merged == 'true' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@65ecb0ca2d3e252f7b82842cd0489c883189f7d0 # v7 with: name: verifier-terminal-disposition-${{ github.run_id }} path: | @@ -1017,7 +1030,7 @@ jobs: - name: Remove trigger label if: steps.check-merged.outputs.merged == 'true' - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 continue-on-error: true with: github-token: ${{ steps.select-token.outputs.token }} diff --git a/templates/consumer-repo/scripts/aggregate_agent_metrics.py b/templates/consumer-repo/scripts/aggregate_agent_metrics.py index 8dcd793a1..be3081034 100755 --- a/templates/consumer-repo/scripts/aggregate_agent_metrics.py +++ b/templates/consumer-repo/scripts/aggregate_agent_metrics.py @@ -47,6 +47,8 @@ _MAX_STORED_PARSE_ERROR_DETAILS = 250 _MAX_LEGACY_JSON_FALLBACK_LINES = 5000 _MAX_LEGACY_JSON_FALLBACK_BYTES = 1024 * 1024 +_PARSE_ERROR_DETAIL_LIMIT_REASON = "detail-limit-exceeded" +_PARSE_ERROR_DETAIL_LIMIT_VALUE = "__multiple__" @dataclass(frozen=True) @@ -179,6 +181,8 @@ def _append_parse_error_detail( *, detail_limit: int = _MAX_STORED_PARSE_ERROR_DETAILS, ) -> None: + if detail_limit <= 0: + return if len(details) < detail_limit: details.append(detail) return @@ -194,7 +198,20 @@ def _append_parse_error_detail( details[index] = replace(existing, count=existing.count + detail.count) return - details.append(replace(detail, line=None)) + for index, existing in enumerate(details): + if existing.reason == _PARSE_ERROR_DETAIL_LIMIT_REASON and existing.line is None: + details[index] = replace(existing, count=existing.count + detail.count) + return + + evicted = details[-1] + details[-1] = ParseErrorDetail( + path=_PARSE_ERROR_DETAIL_LIMIT_VALUE, + artifact=_PARSE_ERROR_DETAIL_LIMIT_VALUE, + artifact_family=_PARSE_ERROR_DETAIL_LIMIT_VALUE, + line=None, + reason=_PARSE_ERROR_DETAIL_LIMIT_REASON, + count=evicted.count + detail.count, + ) def _parse_error_count(parse_error_details: list[ParseErrorDetail]) -> int: diff --git a/tests/scripts/test_aggregate_agent_metrics.py b/tests/scripts/test_aggregate_agent_metrics.py index 833e242ca..ece8594e6 100644 --- a/tests/scripts/test_aggregate_agent_metrics.py +++ b/tests/scripts/test_aggregate_agent_metrics.py @@ -475,19 +475,41 @@ def test_read_ndjson_bounds_legacy_json_fallback_buffer(tmp_path: Path) -> None: entries, errors = aggregate_agent_metrics._read_ndjson([path]) assert entries == [] - assert len(errors) <= aggregate_agent_metrics._MAX_STORED_PARSE_ERROR_DETAILS + 2 + assert len(errors) <= aggregate_agent_metrics._MAX_STORED_PARSE_ERROR_DETAILS assert sum(error.count for error in errors) == ( aggregate_agent_metrics._MAX_LEGACY_JSON_FALLBACK_LINES + 2 ) - assert errors[-2].line is None - assert errors[-2].reason == "invalid-json" - assert errors[-2].count == ( + assert errors[-1].line is None + assert errors[-1].reason == "detail-limit-exceeded" + assert errors[-1].count == ( aggregate_agent_metrics._MAX_LEGACY_JSON_FALLBACK_LINES - + 1 + + 3 - aggregate_agent_metrics._MAX_STORED_PARSE_ERROR_DETAILS ) - assert errors[-1].line is None - assert errors[-1].reason == "legacy-json-fallback-buffer-limit" + + +def test_append_parse_error_detail_caps_distinct_overflow() -> None: + details: list[aggregate_agent_metrics.ParseErrorDetail] = [] + limit = 3 + + for index in range(8): + aggregate_agent_metrics._append_parse_error_detail( + details, + aggregate_agent_metrics.ParseErrorDetail( + path=f"metrics-{index}.ndjson", + artifact=f"artifact-{index}", + artifact_family=f"family-{index}", + line=index, + reason=f"invalid-json-{index}", + ), + detail_limit=limit, + ) + + assert len(details) == limit + assert sum(detail.count for detail in details) == 8 + assert details[-1].line is None + assert details[-1].reason == "detail-limit-exceeded" + assert details[-1].count == 6 def test_classify_entry_prefers_explicit_type() -> None: