Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions .github/workflows/statistical-studies.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ jobs:
shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
with:
python-version: "3.12"
Expand All @@ -40,13 +42,16 @@ jobs:
--skip mlsirm-core/test/literature_true_parameter_recovery::kang_jeon_2025_minimum_cell_recovers_true_parameters
--skip mlsirm-core/test/literature_true_parameter_recovery::gpu_recovery_matches_cpu_on_paper_design
--skip mlsirm-core/test/higher_order_mc_recovery::higher_order_dina_recovery_respects_monte_carlo_tolerance
--skip mlsirm-core/lib/mlsirm_core::grm::tests::mc_grm_recovery_500

rust-pyo3-ignored:
name: Ignored PyO3 studies
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30
- run: cargo test --release --manifest-path crates/fast-mlsirm-py/Cargo.toml -- --ignored --test-threads=1 --nocapture

Expand All @@ -56,6 +61,8 @@ jobs:
timeout-minutes: 120
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30
- name: Run Kang-Jeon paper-design recovery
run: >-
Expand All @@ -70,6 +77,31 @@ jobs:
higher_order_dina_recovery_respects_monte_carlo_tolerance
-- --ignored --exact --nocapture

grm-recovery:
name: GRM 500-rep recovery study
runs-on: ubuntu-latest
timeout-minutes: 120
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30
- name: Run multidimensional GRM Monte Carlo recovery
run: |
set -euo pipefail
cargo test --release -p mlsirm-core --lib \
grm::tests::mc_grm_recovery_500 \
-- --ignored --exact --nocapture --test-threads=1 \
2>&1 | tee grm-recovery-study.log
- name: Publish GRM recovery study log
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
with:
name: grm-recovery-study-${{ github.run_id }}
path: grm-recovery-study.log
if-no-files-found: error
retention-days: 90

gpu-recovery:
name: Literature GPU recovery parity
runs-on: ubuntu-latest
Expand All @@ -79,6 +111,8 @@ jobs:
XDG_RUNTIME_DIR: /tmp/fast-mlsirm-xdg-runtime
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30
- name: Install software Vulkan adapter
run: |
Expand Down
10 changes: 10 additions & 0 deletions docs/changelog.d/grm-recovery-evidence-retention.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# Dedicated GRM recovery evidence retention

## Changed

- Kept the 500-replication multidimensional Graded Response Model recovery
study out of pull-request CI and out of the generic 1,800-second ignored-shard
budget, then published its printed bias, RMSE, convergence, and theta
correlation lines as a 90-day Actions artifact.
- Withheld checkout credentials from every Statistical Studies job so
repository-controlled `cargo test` cannot reuse the Actions token.
49 changes: 49 additions & 0 deletions tests/test_statistical_studies_workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,11 @@ def test_pull_request_ci_keeps_exhaustive_studies_out_of_the_queue():
assert "rust-ignored:" not in text
assert "rust-pyo3-ignored:" not in text
assert "rust-recovery:" not in text
assert "grm-recovery:" not in text
assert "gpu-recovery:" not in text
assert "gpu_recovery_matches_cpu_on_paper_design" not in text
assert "kang_jeon_2025_minimum_cell_recovers_true_parameters" not in text
assert "mc_grm_recovery_500" not in text


def test_exhaustive_studies_are_scheduled_manual_and_release_triggered():
Expand All @@ -32,6 +35,7 @@ def test_exhaustive_studies_are_scheduled_manual_and_release_triggered():
assert "rust-ignored:" in text
assert "rust-pyo3-ignored:" in text
assert "rust-recovery:" in text
assert "grm-recovery:" in text
assert "gpu-recovery:" in text


Expand All @@ -45,6 +49,49 @@ def test_statistical_studies_are_read_only_and_never_rewrite_source():
assert "--exact" in text


def test_grm_recovery_checkout_does_not_persist_credentials():
"""The dedicated GRM study keeps checkout credentials away from Rust tests."""
text = _STUDIES.read_text(encoding="utf-8")
_, grm_block = text.split("\n grm-recovery:\n", maxsplit=1)
grm_block, _ = grm_block.split("\n gpu-recovery:\n", maxsplit=1)
assert "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1" in grm_block
assert "persist-credentials: false" in grm_block


def test_statistical_studies_checkouts_do_not_persist_credentials():
"""Every scheduled study checkout withholds the Actions token from cargo test."""
text = _STUDIES.read_text(encoding="utf-8")
checkout = "uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1"
checkout_tails = text.split(checkout)[1:]
assert len(checkout_tails) == 5
for checkout_tail in checkout_tails:
checkout_block = checkout_tail.split("\n - ", maxsplit=1)[0]
assert "\n with:\n" in checkout_block
assert "\n persist-credentials: false" in checkout_block


def test_grm_recovery_publishes_a_durable_study_log():
"""Buyers can download bias/RMSE/convergence lines after the job log expires."""
text = _STUDIES.read_text(encoding="utf-8")
_, grm_block = text.split("\n grm-recovery:\n", maxsplit=1)
grm_block, _ = grm_block.split("\n gpu-recovery:\n", maxsplit=1)
assert "timeout-minutes: 120" in grm_block
assert "grm::tests::mc_grm_recovery_500" in grm_block
assert "--ignored" in grm_block
assert "--exact" in grm_block
assert "--test-threads=1" in grm_block
assert "set -euo pipefail" in grm_block
assert "tee grm-recovery-study.log" in grm_block
assert (
"actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"
in grm_block
)
assert "path: grm-recovery-study.log" in grm_block
assert "if-no-files-found: error" in grm_block
assert "retention-days: 90" in grm_block
assert "if: always()" in grm_block


def test_general_and_pyo3_jobs_follow_the_declared_workspace_boundary():
"""The general inventory uses workspace metadata; excluded PyO3 is separate."""
workflow = _STUDIES.read_text(encoding="utf-8")
Expand Down Expand Up @@ -72,11 +119,13 @@ def test_dedicated_study_exclusions_are_target_qualified_and_executed_elsewhere(
"gpu_recovery_matches_cpu_on_paper_design",
"mlsirm-core/test/higher_order_mc_recovery::"
"higher_order_dina_recovery_respects_monte_carlo_tolerance",
"mlsirm-core/lib/mlsirm_core::grm::tests::mc_grm_recovery_500",
)
raw_names = (
"kang_jeon_2025_minimum_cell_recovers_true_parameters",
"gpu_recovery_matches_cpu_on_paper_design",
"higher_order_dina_recovery_respects_monte_carlo_tolerance",
"mc_grm_recovery_500",
)
for identifier in identifiers:
assert f"--skip {identifier}" in text
Expand Down
Loading