Skip to content

[codex] Preserve weekly freshness selection state #1915

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stranske merged 1 commit into from
Apr 26, 2026
Merged

[codex] Preserve weekly freshness selection state #1915

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 87 additions & 0 deletions .github/scripts/__tests__/weekly-metrics-download-manifest.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ const assert = require('node:assert/strict');
const {
DOWNLOAD_MANIFEST_SCHEMA,
buildInitialManifest,
compactSelectionDetails,
finalizeManifest,
formatMarkdown,
safeArtifactPathSegment,
Expand All @@ -13,6 +14,40 @@ const {
const selection = {
schema: 'workflows-weekly-metrics-artifact-selection/v1',
status: 'pass',
candidate_count: 3,
candidate_family_counts: {
'codex-cli-freshness': 1,
'keepalive-metrics': 1,
'review-thread-terminal-disposition': 1,
},
selected_family_counts: {
'keepalive-metrics': 1,
'review-thread-terminal-disposition': 1,
},
missing_priority_families: ['bot-comment-auth-coverage-reusable'],
latest_candidate_by_family: {
'codex-cli-freshness': {
id: 44,
name: 'codex-cli-freshness-24965031474',
created_at: '2026-04-26T01:01:00Z',
updated_at: '2026-04-26T01:01:00Z',
},
},
priority_family_statuses: [
{
family: 'codex-cli-freshness',
status: 'available',
candidate_count: 1,
selected_count: 0,
latest_candidate: {
id: 44,
name: 'codex-cli-freshness-24965031474',
created_at: '2026-04-26T01:01:00Z',
updated_at: '2026-04-26T01:01:00Z',
},
selected_artifact: null,
},
],
selected_artifacts: [
{
id: 42,
Expand Down Expand Up @@ -45,6 +80,40 @@ test('builds initial download manifest from selected artifacts', () => {
schema: 'workflows-weekly-metrics-artifact-selection/v1',
status: 'pass',
selected_count: 2,
candidate_count: 3,
candidate_family_counts: {
'codex-cli-freshness': 1,
'keepalive-metrics': 1,
'review-thread-terminal-disposition': 1,
},
selected_family_counts: {
'keepalive-metrics': 1,
'review-thread-terminal-disposition': 1,
},
missing_priority_families: ['bot-comment-auth-coverage-reusable'],
priority_family_statuses: [
{
family: 'codex-cli-freshness',
status: 'available',
candidate_count: 1,
selected_count: 0,
latest_candidate: {
id: 44,
name: 'codex-cli-freshness-24965031474',
created_at: '2026-04-26T01:01:00Z',
updated_at: '2026-04-26T01:01:00Z',
},
selected_artifact: null,
},
],
latest_candidate_by_family: {
'codex-cli-freshness': {
id: 44,
name: 'codex-cli-freshness-24965031474',
created_at: '2026-04-26T01:01:00Z',
updated_at: '2026-04-26T01:01:00Z',
},
},
});
assert.deepEqual(manifest.stats, {
selected_count: 2,
Expand All @@ -60,6 +129,24 @@ test('builds initial download manifest from selected artifacts', () => {
assert.equal(manifest.artifacts[0].unzip.status, 'pending');
});

test('preserves priority family state from the selector contract', () => {
const details = compactSelectionDetails(selection, 'artifacts/metric-artifacts-selection.json');

assert.equal(details.selected_count, 2);
assert.equal(details.candidate_count, 3);
assert.deepEqual(details.selected_family_counts, {
'keepalive-metrics': 1,
'review-thread-terminal-disposition': 1,
});
assert.deepEqual(details.latest_candidate_by_family['codex-cli-freshness'], {
id: 44,
name: 'codex-cli-freshness-24965031474',
created_at: '2026-04-26T01:01:00Z',
updated_at: '2026-04-26T01:01:00Z',
});
assert.equal(details.priority_family_statuses[0].status, 'available');
});

test('records download and unzip outcomes and finalizes warning status', () => {
const manifest = buildInitialManifest(selection);

Expand Down
29 changes: 23 additions & 6 deletions .github/scripts/weekly_metrics_download_manifest.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,27 @@ function selectedArtifactsFromSelection(selection = {}) {
return Array.isArray(selection.selected_artifacts) ? selection.selected_artifacts : [];
}

function compactSelectionDetails(selection = {}, selectionPath = '') {
return {
path: selectionPath,
schema: cleanString(selection.schema),
status: cleanString(selection.status || 'pass'),
selected_count: selectedArtifactsFromSelection(selection).length,
candidate_count: Number.isFinite(Number(selection.candidate_count))
? Number(selection.candidate_count)
: 0,
candidate_family_counts: selection.candidate_family_counts || {},
selected_family_counts: selection.selected_family_counts || {},
missing_priority_families: Array.isArray(selection.missing_priority_families)
Comment on lines +51 to +53

Copilot AI Apr 26, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

candidate_family_counts and selected_family_counts are passed through with || {}, which still allows non-object truthy values (e.g., arrays/strings) to leak into the manifest. Since this is part of a JSON contract, please validate these fields are plain objects (non-null, non-array) and fall back to {} (optionally shallow-clone) when the selector report is malformed; apply the same guard to latest_candidate_by_family.

Copilot uses AI. Check for mistakes.
? selection.missing_priority_families
: [],
priority_family_statuses: Array.isArray(selection.priority_family_statuses)
? selection.priority_family_statuses
: [],
latest_candidate_by_family: selection.latest_candidate_by_family || {},
};
}

function defaultArtifactDir(root, artifact) {
return path.posix.join(root, safeArtifactPathSegment(artifact.name), String(artifact.id || ''));
}
Expand All @@ -56,12 +77,7 @@ function buildInitialManifest(selection = {}, options = {}) {
schema: DOWNLOAD_MANIFEST_SCHEMA,
status: selection.status === 'error' ? 'error' : 'pending',
generated_at: generatedAt,
selection: {
path: selectionPath,
schema: cleanString(selection.schema),
status: cleanString(selection.status || 'pass'),
selected_count: selected.length,
},
selection: compactSelectionDetails(selection, selectionPath),
stats: {
selected_count: selected.length,
download_pass_count: 0,
Expand Down Expand Up @@ -313,6 +329,7 @@ if (require.main === module) {
module.exports = {
DOWNLOAD_MANIFEST_SCHEMA,
buildInitialManifest,
compactSelectionDetails,
finalizeManifest,
formatMarkdown,
safeArtifactPathSegment,
Expand Down
2 changes: 1 addition & 1 deletion scripts/check_codex_cli_freshness.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ def query_latest_npm_version(package: str = DEFAULT_PACKAGE, timeout: int = 30)
try:
with tempfile.TemporaryDirectory(prefix="codex-cli-freshness-npm-") as cache_dir:
env = os.environ.copy()
env.setdefault("NPM_CONFIG_CACHE", cache_dir)
env["NPM_CONFIG_CACHE"] = cache_dir
completed = subprocess.run(
command,
check=True,
Expand Down
29 changes: 23 additions & 6 deletions templates/consumer-repo/.github/scripts/weekly_metrics_download_manifest.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,27 @@ function selectedArtifactsFromSelection(selection = {}) {
return Array.isArray(selection.selected_artifacts) ? selection.selected_artifacts : [];
}

function compactSelectionDetails(selection = {}, selectionPath = '') {
return {
path: selectionPath,
schema: cleanString(selection.schema),
status: cleanString(selection.status || 'pass'),
selected_count: selectedArtifactsFromSelection(selection).length,
candidate_count: Number.isFinite(Number(selection.candidate_count))
? Number(selection.candidate_count)
: 0,
candidate_family_counts: selection.candidate_family_counts || {},
selected_family_counts: selection.selected_family_counts || {},
missing_priority_families: Array.isArray(selection.missing_priority_families)
Comment on lines +51 to +53

Copilot AI Apr 26, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

candidate_family_counts and selected_family_counts are passed through with || {}, which still allows non-object truthy values (e.g., arrays/strings) to leak into the manifest. Since this is part of a JSON contract, please validate these fields are plain objects (non-null, non-array) and fall back to {} (optionally shallow-clone) when the selector report is malformed; apply the same guard to latest_candidate_by_family.

Copilot uses AI. Check for mistakes.
? selection.missing_priority_families
: [],
priority_family_statuses: Array.isArray(selection.priority_family_statuses)
? selection.priority_family_statuses
: [],
latest_candidate_by_family: selection.latest_candidate_by_family || {},
};
}

function defaultArtifactDir(root, artifact) {
return path.posix.join(root, safeArtifactPathSegment(artifact.name), String(artifact.id || ''));
}
Expand All @@ -56,12 +77,7 @@ function buildInitialManifest(selection = {}, options = {}) {
schema: DOWNLOAD_MANIFEST_SCHEMA,
status: selection.status === 'error' ? 'error' : 'pending',
generated_at: generatedAt,
selection: {
path: selectionPath,
schema: cleanString(selection.schema),
status: cleanString(selection.status || 'pass'),
selected_count: selected.length,
},
selection: compactSelectionDetails(selection, selectionPath),
stats: {
selected_count: selected.length,
download_pass_count: 0,
Expand Down Expand Up @@ -313,6 +329,7 @@ if (require.main === module) {
module.exports = {
DOWNLOAD_MANIFEST_SCHEMA,
buildInitialManifest,
compactSelectionDetails,
finalizeManifest,
formatMarkdown,
safeArtifactPathSegment,
Expand Down
28 changes: 27 additions & 1 deletion tests/scripts/test_aggregate_agent_metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1014,7 +1014,28 @@ def test_main_includes_artifact_download_manifest(
{
"schema": "workflows-weekly-metrics-artifact-download-manifest/v1",
"status": "warning",
"selection": {"selected_count": 2},
"selection": {
"selected_count": 2,
"selected_family_counts": {"keepalive-metrics": 1},
"priority_family_statuses": [
{
"family": "codex-cli-freshness",
"status": "selected",
"candidate_count": 1,
"selected_count": 1,
"selected_artifact": {
"id": 44,
"name": "codex-cli-freshness-24965031474",
},
}
],
"latest_candidate_by_family": {
"codex-cli-freshness": {
"id": 44,
"name": "codex-cli-freshness-24965031474",
}
},
},
"stats": {
"selected_count": 2,
"download_pass_count": 1,
Expand Down Expand Up @@ -1059,6 +1080,11 @@ def test_main_includes_artifact_download_manifest(
downloads = summary_json["artifact_downloads"]
assert downloads["schema"] == "workflows-weekly-metrics-artifact-download-manifest/v1"
assert downloads["status"] == "warning"
assert downloads["selection"]["priority_family_statuses"][0]["family"] == (
"codex-cli-freshness"
)
assert downloads["selection"]["priority_family_statuses"][0]["status"] == "selected"
assert downloads["selection"]["latest_candidate_by_family"]["codex-cli-freshness"]["id"] == 44
assert downloads["stats"]["download_failed_count"] == 1
assert downloads["failed_artifacts"] == [
{
Expand Down
20 changes: 20 additions & 0 deletions tests/scripts/test_check_codex_cli_freshness.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import json
import subprocess
from pathlib import Path

from scripts import check_codex_cli_freshness
Expand Down Expand Up @@ -40,6 +41,25 @@ def test_build_contract_reports_outdated_pin() -> None:
assert "@openai/codex@0.127.3" in markdown


def test_query_latest_npm_version_uses_isolated_cache(monkeypatch) -> None:
captured_env = {}

def fake_run(command, **kwargs):
captured_env.update(kwargs["env"])
assert command == ["npm", "view", "@openai/codex", "version", "--silent"]
return subprocess.CompletedProcess(command, 0, stdout="0.126.0\n", stderr="")

monkeypatch.setenv("NPM_CONFIG_CACHE", "/bad/shared/cache")
monkeypatch.setattr(check_codex_cli_freshness.subprocess, "run", fake_run)

latest, error = check_codex_cli_freshness.query_latest_npm_version()

assert latest == "0.126.0"
assert error == ""
assert captured_env["NPM_CONFIG_CACHE"] != "/bad/shared/cache"
assert "codex-cli-freshness-npm-" in captured_env["NPM_CONFIG_CACHE"]


def test_main_writes_machine_readable_outputs(tmp_path: Path) -> None:
workflow = tmp_path / "workflow.yml"
output_json = tmp_path / "freshness.json"
Expand Down
Loading