diff --git a/.github/scripts/__tests__/weekly-metrics-download-manifest.test.js b/.github/scripts/__tests__/weekly-metrics-download-manifest.test.js index d778672cd..21cc9a9e4 100644 --- a/.github/scripts/__tests__/weekly-metrics-download-manifest.test.js +++ b/.github/scripts/__tests__/weekly-metrics-download-manifest.test.js @@ -4,6 +4,7 @@ const assert = require('node:assert/strict'); const { DOWNLOAD_MANIFEST_SCHEMA, buildInitialManifest, + compactSelectionDetails, finalizeManifest, formatMarkdown, safeArtifactPathSegment, @@ -13,6 +14,40 @@ const { const selection = { schema: 'workflows-weekly-metrics-artifact-selection/v1', status: 'pass', + candidate_count: 3, + candidate_family_counts: { + 'codex-cli-freshness': 1, + 'keepalive-metrics': 1, + 'review-thread-terminal-disposition': 1, + }, + selected_family_counts: { + 'keepalive-metrics': 1, + 'review-thread-terminal-disposition': 1, + }, + missing_priority_families: ['bot-comment-auth-coverage-reusable'], + latest_candidate_by_family: { + 'codex-cli-freshness': { + id: 44, + name: 'codex-cli-freshness-24965031474', + created_at: '2026-04-26T01:01:00Z', + updated_at: '2026-04-26T01:01:00Z', + }, + }, + priority_family_statuses: [ + { + family: 'codex-cli-freshness', + status: 'available', + candidate_count: 1, + selected_count: 0, + latest_candidate: { + id: 44, + name: 'codex-cli-freshness-24965031474', + created_at: '2026-04-26T01:01:00Z', + updated_at: '2026-04-26T01:01:00Z', + }, + selected_artifact: null, + }, + ], selected_artifacts: [ { id: 42, @@ -45,6 +80,40 @@ test('builds initial download manifest from selected artifacts', () => { schema: 'workflows-weekly-metrics-artifact-selection/v1', status: 'pass', selected_count: 2, + candidate_count: 3, + candidate_family_counts: { + 'codex-cli-freshness': 1, + 'keepalive-metrics': 1, + 'review-thread-terminal-disposition': 1, + }, + selected_family_counts: { + 'keepalive-metrics': 1, + 'review-thread-terminal-disposition': 1, + }, + missing_priority_families: ['bot-comment-auth-coverage-reusable'], + priority_family_statuses: [ + { + family: 'codex-cli-freshness', + status: 'available', + candidate_count: 1, + selected_count: 0, + latest_candidate: { + id: 44, + name: 'codex-cli-freshness-24965031474', + created_at: '2026-04-26T01:01:00Z', + updated_at: '2026-04-26T01:01:00Z', + }, + selected_artifact: null, + }, + ], + latest_candidate_by_family: { + 'codex-cli-freshness': { + id: 44, + name: 'codex-cli-freshness-24965031474', + created_at: '2026-04-26T01:01:00Z', + updated_at: '2026-04-26T01:01:00Z', + }, + }, }); assert.deepEqual(manifest.stats, { selected_count: 2, @@ -60,6 +129,24 @@ test('builds initial download manifest from selected artifacts', () => { assert.equal(manifest.artifacts[0].unzip.status, 'pending'); }); +test('preserves priority family state from the selector contract', () => { + const details = compactSelectionDetails(selection, 'artifacts/metric-artifacts-selection.json'); + + assert.equal(details.selected_count, 2); + assert.equal(details.candidate_count, 3); + assert.deepEqual(details.selected_family_counts, { + 'keepalive-metrics': 1, + 'review-thread-terminal-disposition': 1, + }); + assert.deepEqual(details.latest_candidate_by_family['codex-cli-freshness'], { + id: 44, + name: 'codex-cli-freshness-24965031474', + created_at: '2026-04-26T01:01:00Z', + updated_at: '2026-04-26T01:01:00Z', + }); + assert.equal(details.priority_family_statuses[0].status, 'available'); +}); + test('records download and unzip outcomes and finalizes warning status', () => { const manifest = buildInitialManifest(selection); diff --git a/.github/scripts/weekly_metrics_download_manifest.js b/.github/scripts/weekly_metrics_download_manifest.js index 746e62881..298c4c7ca 100644 --- a/.github/scripts/weekly_metrics_download_manifest.js +++ b/.github/scripts/weekly_metrics_download_manifest.js @@ -39,6 +39,27 @@ function selectedArtifactsFromSelection(selection = {}) { return Array.isArray(selection.selected_artifacts) ? selection.selected_artifacts : []; } +function compactSelectionDetails(selection = {}, selectionPath = '') { + return { + path: selectionPath, + schema: cleanString(selection.schema), + status: cleanString(selection.status || 'pass'), + selected_count: selectedArtifactsFromSelection(selection).length, + candidate_count: Number.isFinite(Number(selection.candidate_count)) + ? Number(selection.candidate_count) + : 0, + candidate_family_counts: selection.candidate_family_counts || {}, + selected_family_counts: selection.selected_family_counts || {}, + missing_priority_families: Array.isArray(selection.missing_priority_families) + ? selection.missing_priority_families + : [], + priority_family_statuses: Array.isArray(selection.priority_family_statuses) + ? selection.priority_family_statuses + : [], + latest_candidate_by_family: selection.latest_candidate_by_family || {}, + }; +} + function defaultArtifactDir(root, artifact) { return path.posix.join(root, safeArtifactPathSegment(artifact.name), String(artifact.id || '')); } @@ -56,12 +77,7 @@ function buildInitialManifest(selection = {}, options = {}) { schema: DOWNLOAD_MANIFEST_SCHEMA, status: selection.status === 'error' ? 'error' : 'pending', generated_at: generatedAt, - selection: { - path: selectionPath, - schema: cleanString(selection.schema), - status: cleanString(selection.status || 'pass'), - selected_count: selected.length, - }, + selection: compactSelectionDetails(selection, selectionPath), stats: { selected_count: selected.length, download_pass_count: 0, @@ -313,6 +329,7 @@ if (require.main === module) { module.exports = { DOWNLOAD_MANIFEST_SCHEMA, buildInitialManifest, + compactSelectionDetails, finalizeManifest, formatMarkdown, safeArtifactPathSegment, diff --git a/scripts/check_codex_cli_freshness.py b/scripts/check_codex_cli_freshness.py index c70b3c221..cda9f46dc 100644 --- a/scripts/check_codex_cli_freshness.py +++ b/scripts/check_codex_cli_freshness.py @@ -52,7 +52,7 @@ def query_latest_npm_version(package: str = DEFAULT_PACKAGE, timeout: int = 30) try: with tempfile.TemporaryDirectory(prefix="codex-cli-freshness-npm-") as cache_dir: env = os.environ.copy() - env.setdefault("NPM_CONFIG_CACHE", cache_dir) + env["NPM_CONFIG_CACHE"] = cache_dir completed = subprocess.run( command, check=True, diff --git a/templates/consumer-repo/.github/scripts/weekly_metrics_download_manifest.js b/templates/consumer-repo/.github/scripts/weekly_metrics_download_manifest.js index 746e62881..298c4c7ca 100644 --- a/templates/consumer-repo/.github/scripts/weekly_metrics_download_manifest.js +++ b/templates/consumer-repo/.github/scripts/weekly_metrics_download_manifest.js @@ -39,6 +39,27 @@ function selectedArtifactsFromSelection(selection = {}) { return Array.isArray(selection.selected_artifacts) ? selection.selected_artifacts : []; } +function compactSelectionDetails(selection = {}, selectionPath = '') { + return { + path: selectionPath, + schema: cleanString(selection.schema), + status: cleanString(selection.status || 'pass'), + selected_count: selectedArtifactsFromSelection(selection).length, + candidate_count: Number.isFinite(Number(selection.candidate_count)) + ? Number(selection.candidate_count) + : 0, + candidate_family_counts: selection.candidate_family_counts || {}, + selected_family_counts: selection.selected_family_counts || {}, + missing_priority_families: Array.isArray(selection.missing_priority_families) + ? selection.missing_priority_families + : [], + priority_family_statuses: Array.isArray(selection.priority_family_statuses) + ? selection.priority_family_statuses + : [], + latest_candidate_by_family: selection.latest_candidate_by_family || {}, + }; +} + function defaultArtifactDir(root, artifact) { return path.posix.join(root, safeArtifactPathSegment(artifact.name), String(artifact.id || '')); } @@ -56,12 +77,7 @@ function buildInitialManifest(selection = {}, options = {}) { schema: DOWNLOAD_MANIFEST_SCHEMA, status: selection.status === 'error' ? 'error' : 'pending', generated_at: generatedAt, - selection: { - path: selectionPath, - schema: cleanString(selection.schema), - status: cleanString(selection.status || 'pass'), - selected_count: selected.length, - }, + selection: compactSelectionDetails(selection, selectionPath), stats: { selected_count: selected.length, download_pass_count: 0, @@ -313,6 +329,7 @@ if (require.main === module) { module.exports = { DOWNLOAD_MANIFEST_SCHEMA, buildInitialManifest, + compactSelectionDetails, finalizeManifest, formatMarkdown, safeArtifactPathSegment, diff --git a/tests/scripts/test_aggregate_agent_metrics.py b/tests/scripts/test_aggregate_agent_metrics.py index e0e45d934..c640535a2 100644 --- a/tests/scripts/test_aggregate_agent_metrics.py +++ b/tests/scripts/test_aggregate_agent_metrics.py @@ -1014,7 +1014,28 @@ def test_main_includes_artifact_download_manifest( { "schema": "workflows-weekly-metrics-artifact-download-manifest/v1", "status": "warning", - "selection": {"selected_count": 2}, + "selection": { + "selected_count": 2, + "selected_family_counts": {"keepalive-metrics": 1}, + "priority_family_statuses": [ + { + "family": "codex-cli-freshness", + "status": "selected", + "candidate_count": 1, + "selected_count": 1, + "selected_artifact": { + "id": 44, + "name": "codex-cli-freshness-24965031474", + }, + } + ], + "latest_candidate_by_family": { + "codex-cli-freshness": { + "id": 44, + "name": "codex-cli-freshness-24965031474", + } + }, + }, "stats": { "selected_count": 2, "download_pass_count": 1, @@ -1059,6 +1080,11 @@ def test_main_includes_artifact_download_manifest( downloads = summary_json["artifact_downloads"] assert downloads["schema"] == "workflows-weekly-metrics-artifact-download-manifest/v1" assert downloads["status"] == "warning" + assert downloads["selection"]["priority_family_statuses"][0]["family"] == ( + "codex-cli-freshness" + ) + assert downloads["selection"]["priority_family_statuses"][0]["status"] == "selected" + assert downloads["selection"]["latest_candidate_by_family"]["codex-cli-freshness"]["id"] == 44 assert downloads["stats"]["download_failed_count"] == 1 assert downloads["failed_artifacts"] == [ { diff --git a/tests/scripts/test_check_codex_cli_freshness.py b/tests/scripts/test_check_codex_cli_freshness.py index e2ae81a21..5d0e3f1cb 100644 --- a/tests/scripts/test_check_codex_cli_freshness.py +++ b/tests/scripts/test_check_codex_cli_freshness.py @@ -1,4 +1,5 @@ import json +import subprocess from pathlib import Path from scripts import check_codex_cli_freshness @@ -40,6 +41,25 @@ def test_build_contract_reports_outdated_pin() -> None: assert "@openai/codex@0.127.3" in markdown +def test_query_latest_npm_version_uses_isolated_cache(monkeypatch) -> None: + captured_env = {} + + def fake_run(command, **kwargs): + captured_env.update(kwargs["env"]) + assert command == ["npm", "view", "@openai/codex", "version", "--silent"] + return subprocess.CompletedProcess(command, 0, stdout="0.126.0\n", stderr="") + + monkeypatch.setenv("NPM_CONFIG_CACHE", "/bad/shared/cache") + monkeypatch.setattr(check_codex_cli_freshness.subprocess, "run", fake_run) + + latest, error = check_codex_cli_freshness.query_latest_npm_version() + + assert latest == "0.126.0" + assert error == "" + assert captured_env["NPM_CONFIG_CACHE"] != "/bad/shared/cache" + assert "codex-cli-freshness-npm-" in captured_env["NPM_CONFIG_CACHE"] + + def test_main_writes_machine_readable_outputs(tmp_path: Path) -> None: workflow = tmp_path / "workflow.yml" output_json = tmp_path / "freshness.json"