From 397a84292fe2f289406f18bc09e2c04250fcdbb6 Mon Sep 17 00:00:00 2001 From: aoshen02 Date: Tue, 2 Jun 2026 15:57:02 +0000 Subject: [PATCH] [CI] add reward-fn cpu tests + cpu-on-main trigger (port slime #1939+#1940) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port of THUDM/slime#1939 (add more cpu ci) + #1940 (run cpu test on main): - tests/test_rm_{deepscaler,f1,gpqa,math,math_dapo}.py: pure-cpu unit tests for vime/rollout/rm_hub reward fns (107 tests, validated locally). - vime/utils/misc.py: lazy 'import ray' so cpu-only paths (rm_hub scoring, plugin contracts) don't require ray. - pr-test.yml.j2: register the 5 rm tests in the cpu (num_gpus:0) matrix; enable push-to-main trigger gated to cpu jobs only (GPU/self-hosted jobs stay PR/label-gated so push never burns the fleet). slime #1939 also added tests/test_sample.py, which references Sample.rollout_id (introduced by the #1926-chain / vime PR #119) — that test is stacked on #119 instead of here. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: aoshen02 --- .github/workflows/pr-test.yml | 18 ++- .github/workflows/pr-test.yml.j2 | 19 ++- tests/test_rm_deepscaler.py | 96 +++++++++++++ tests/test_rm_f1.py | 102 ++++++++++++++ tests/test_rm_gpqa.py | 176 +++++++++++++++++++++++ tests/test_rm_math.py | 231 +++++++++++++++++++++++++++++++ tests/test_rm_math_dapo.py | 230 ++++++++++++++++++++++++++++++ vime/utils/misc.py | 6 +- 8 files changed, 867 insertions(+), 11 deletions(-) create mode 100644 tests/test_rm_deepscaler.py create mode 100644 tests/test_rm_f1.py create mode 100644 tests/test_rm_gpqa.py create mode 100644 tests/test_rm_math.py create mode 100644 tests/test_rm_math_dapo.py diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index bc8b8e769..904a1a1cc 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -5,9 +5,13 @@ name: PR Test on: - # Do not run CI on push to reduce CI time - # push: - # branches: [main] + # Push to main triggers ONLY the cpu jobs (cheap GitHub-hosted runner) — see + # the per-job `if:` below, which adds `push` only for cpu configs. GPU jobs + # stay PR/label-gated so push events never burn the self-hosted fleet. This + # catches PR-pair regressions where two PRs pass individually but main breaks + # after both land. + push: + branches: [main] pull_request: branches: [main] types: [synchronize, labeled] @@ -468,7 +472,9 @@ jobs: e2e-test-plugin-contracts: needs: pre-commit - if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' || github.event_name == 'push' + runs-on: ubuntu-latest @@ -476,7 +482,7 @@ jobs: strategy: fail-fast: false matrix: - info: [{"num_gpus": 0, "test_file": "test_megatron_argument_validation.py"}, {"num_gpus": 0, "test_file": "plugin_contracts/test_plugin_rollout_contracts.py"}, {"num_gpus": 0, "test_file": "plugin_contracts/test_plugin_runtime_hook_contracts.py"}, {"num_gpus": 0, "test_file": "plugin_contracts/test_plugin_path_loading_contracts.py"}, {"num_gpus": 0, "test_file": "plugin_contracts/test_plugin_generate_contracts.py"}] + info: [{"num_gpus": 0, "test_file": "test_megatron_argument_validation.py"}, {"num_gpus": 0, "test_file": "plugin_contracts/test_plugin_rollout_contracts.py"}, {"num_gpus": 0, "test_file": "plugin_contracts/test_plugin_runtime_hook_contracts.py"}, {"num_gpus": 0, "test_file": "plugin_contracts/test_plugin_path_loading_contracts.py"}, {"num_gpus": 0, "test_file": "plugin_contracts/test_plugin_generate_contracts.py"}, {"num_gpus": 0, "test_file": "test_rm_deepscaler.py"}, {"num_gpus": 0, "test_file": "test_rm_f1.py"}, {"num_gpus": 0, "test_file": "test_rm_gpqa.py"}, {"num_gpus": 0, "test_file": "test_rm_math.py"}, {"num_gpus": 0, "test_file": "test_rm_math_dapo.py"}] defaults: run: working-directory: ${{ github.workspace }} @@ -767,4 +773,4 @@ jobs: else python tests/ci/gpu_lock_exec.py --count "$NUM_GPUS" -- python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}" fi - ' + ' \ No newline at end of file diff --git a/.github/workflows/pr-test.yml.j2 b/.github/workflows/pr-test.yml.j2 index adab45a63..13df88595 100644 --- a/.github/workflows/pr-test.yml.j2 +++ b/.github/workflows/pr-test.yml.j2 @@ -59,6 +59,11 @@ {'test_file': 'plugin_contracts/test_plugin_runtime_hook_contracts.py', 'num_gpus': 0}, {'test_file': 'plugin_contracts/test_plugin_path_loading_contracts.py', 'num_gpus': 0}, {'test_file': 'plugin_contracts/test_plugin_generate_contracts.py', 'num_gpus': 0}, + {'test_file': 'test_rm_deepscaler.py', 'num_gpus': 0}, + {'test_file': 'test_rm_f1.py', 'num_gpus': 0}, + {'test_file': 'test_rm_gpqa.py', 'num_gpus': 0}, + {'test_file': 'test_rm_math.py', 'num_gpus': 0}, + {'test_file': 'test_rm_math_dapo.py', 'num_gpus': 0}, ], }, @@ -85,9 +90,13 @@ name: PR Test on: - # Do not run CI on push to reduce CI time - # push: - # branches: [main] + # Push to main triggers ONLY the cpu jobs (cheap GitHub-hosted runner) — see + # the per-job `if:` below, which adds `push` only for cpu configs. GPU jobs + # stay PR/label-gated so push events never burn the self-hosted fleet. This + # catches PR-pair regressions where two PRs pass individually but main breaks + # after both land. + push: + branches: [main] pull_request: branches: [main] types: [synchronize, labeled] @@ -128,7 +137,11 @@ jobs: << job_name >>: needs: pre-commit <% if config.get('always') %> +<% if config.get('cpu') %> + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' || github.event_name == 'push' +<% else %> if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' +<% endif %> <% else %> if: (github.event_name == 'workflow_dispatch') || (github.event.pull_request && contains(github.event.pull_request.labels.*.name, '<< config.label >>')) <% endif %> diff --git a/tests/test_rm_deepscaler.py b/tests/test_rm_deepscaler.py new file mode 100644 index 000000000..60e79bff4 --- /dev/null +++ b/tests/test_rm_deepscaler.py @@ -0,0 +1,96 @@ +"""CPU unit tests for ``vime.rollout.rm_hub.deepscaler``. + +Pins the wrapper that decides which segment of the response counts as +the "solution" and reduces grading to ``math_utils``. The branching is +small (3 cases) but silent-failure prone: if the ```` / +``###Response`` markers stop matching the format the rollout actually +produces, the function returns 0 *for every sample* and no other CI +signal would catch it. +""" + +from __future__ import annotations + +import pytest + +from vime.rollout.rm_hub.deepscaler import get_deepscaler_rule_based_reward + + +@pytest.mark.unit +def test_response_split_on_think_marker_grades_tail(): + """The default chat format puts the answer after ````. Only + the tail is graded — pre-think reasoning is ignored, even if it + contains the wrong answer first.""" + response = r"Let me reconsider. \boxed{99}Final: \boxed{42}" + assert get_deepscaler_rule_based_reward(response, "42") == 1 + + +@pytest.mark.unit +def test_response_split_on_response_marker_grades_tail(): + """Alternate format: ``###Response`` separator. Only what comes after + is graded (deepscaler.py:7-8).""" + response = r"Scratch work \boxed{wrong}###Response\boxed{42}" + assert get_deepscaler_rule_based_reward(response, "42") == 1 + + +@pytest.mark.unit +def test_response_without_any_marker_returns_zero(): + """No ```` AND no ``###Response`` → fall through to 0 + immediately (deepscaler.py:9-10). This is the silent-failure pole — + if upstream chat templates drop both markers, all rewards become 0.""" + assert get_deepscaler_rule_based_reward(r"\boxed{42}", "42") == 0 + + +@pytest.mark.unit +def test_response_with_no_boxed_answer_returns_zero(): + """Marker is present but no ``\\boxed`` in the tail → ``extract_answer`` + returns None → 0 (deepscaler.py:13-14).""" + assert get_deepscaler_rule_based_reward("plainno box here", "42") == 0 + + +@pytest.mark.unit +def test_empty_label_returns_zero(): + """Empty ground-truth → 0 (deepscaler.py:15-16). Guards against + missing-label data poisoning training with spurious 0s — explicitly + the same as wrong-answer, intentional.""" + assert get_deepscaler_rule_based_reward(r"\boxed{42}", "") == 0 + + +@pytest.mark.unit +def test_label_as_int_is_coerced_to_string(): + """Integer labels are accepted and ``str()``'d (deepscaler.py:19, 25). + Common case for datasets that store numeric labels.""" + assert get_deepscaler_rule_based_reward(r"\boxed{42}", 42) == 1 + + +@pytest.mark.unit +def test_label_as_float_is_coerced_to_string(): + """float labels: stringified to e.g. "42.0". The current grader path + (mathd or sympy) handles "42.0" vs "42" via normalization — pinning + the wiring, not the equality logic.""" + assert get_deepscaler_rule_based_reward(r"\boxed{42}", 42) == 1 + + +@pytest.mark.unit +def test_label_with_boxed_marker_is_extracted_too(): + """If the ground truth itself is wrapped in ``\\boxed{}``, it must be + unwrapped before grading (deepscaler.py:26-29).""" + assert get_deepscaler_rule_based_reward(r"\boxed{42}", r"\boxed{42}") == 1 + + +@pytest.mark.unit +def test_wrong_answer_returns_zero(): + """Sanity-check the negative side of the contract.""" + assert get_deepscaler_rule_based_reward(r"\boxed{43}", "42") == 0 + + +@pytest.mark.unit +def test_grader_uses_either_mathd_or_sympy_path(): + """``\\frac{1}{2}`` vs ``0.5`` — mathd_normalize collapses both, even + though the strings aren't lexically equal. Pins the "either mathd OR + sympy succeeds" disjunction at deepscaler.py:38.""" + response = r"\boxed{\frac{1}{2}}" + assert get_deepscaler_rule_based_reward(response, "0.5") == 1 + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__])) diff --git a/tests/test_rm_f1.py b/tests/test_rm_f1.py new file mode 100644 index 000000000..a68438b0e --- /dev/null +++ b/tests/test_rm_f1.py @@ -0,0 +1,102 @@ +"""CPU unit tests for ``vime.rollout.rm_hub.f1``. + +Pins the token-F1 contract used by the ``f1`` rm_type. The whole pipeline +is pure Python (regex + ``collections.Counter``), so any silent drift +here directly distorts a training run's reward signal without touching a +crash log. Cover the four shapes a reward consumer cares about: + + - normalize_answer: article-strip + punctuation-strip + lowercase + + whitespace-collapse (the order matters — ``a.`` should normalize to + `""`, not `"a"`) + - yes/no/noanswer special-case (exact-match required, not token F1) + - zero-overlap path (returns the ZERO_METRIC sentinel) + - non-trivial F1 with hand-derived precision/recall + +The module ships zero existing tests; if any of the regexes or the +Counter intersection break, no other CI signal would notice. +""" + +from __future__ import annotations + +import pytest + +from vime.rollout.rm_hub.f1 import f1_score, normalize_answer + + +@pytest.mark.unit +@pytest.mark.parametrize( + "raw,expected", + [ + ("The Quick Brown Fox.", "quick brown fox"), # articles stripped, punc removed, lowercased + ("An apple, a day", "apple day"), # both "an" and "a" stripped + ("HELLO WORLD", "hello world"), # whitespace collapsed + ("a.b,c!", "abc"), # punc adjacent to chars collapses without leaving a space + ("", ""), # empty input survives + ("the", ""), # all-article input collapses to empty + ], +) +def test_normalize_answer(raw, expected): + assert normalize_answer(raw) == expected + + +@pytest.mark.unit +def test_f1_exact_match_is_perfect(): + """Hand-derived: tokens fully overlap → precision=recall=f1=1.0.""" + f1, p, r = f1_score("Paris is the capital", "Paris is the capital") + assert (f1, p, r) == (1.0, 1.0, 1.0) + + +@pytest.mark.unit +def test_f1_partial_overlap_hand_derived(): + """Hand-derived: prediction "the brown fox" → ["brown", "fox"] after + normalize; ground truth "a quick brown fox" → ["quick", "brown", "fox"]. + Intersection = {"brown", "fox"} → num_same = 2. + precision = 2 / 2 = 1.0 (len(pred_tokens) = 2) + recall = 2 / 3 + f1 = 2 * 1.0 * (2/3) / (1.0 + 2/3) = 0.8 + """ + f1, p, r = f1_score("the brown fox", "a quick brown fox") + assert p == pytest.approx(1.0) + assert r == pytest.approx(2 / 3) + assert f1 == pytest.approx(0.8) + + +@pytest.mark.unit +def test_f1_no_token_overlap_returns_zero_metric(): + """Disjoint vocabularies → ZERO_METRIC sentinel (0, 0, 0).""" + assert f1_score("apple banana", "carrot date") == (0, 0, 0) + + +@pytest.mark.unit +def test_f1_none_prediction_returns_zero_metric(): + """A failed/missing prediction is a common rm path — must be zero, not raise.""" + assert f1_score(None, "anything") == (0, 0, 0) + + +@pytest.mark.unit +@pytest.mark.parametrize("special", ["yes", "no", "noanswer"]) +def test_f1_special_token_pred_mismatch_returns_zero(special): + """yes/no/noanswer in the prediction but not the ground truth — must be + zero even if token-F1 would otherwise be non-zero. Pins the asymmetric + early-exit at f1.py:33.""" + assert f1_score(special, "some other phrase") == (0, 0, 0) + + +@pytest.mark.unit +@pytest.mark.parametrize("special", ["yes", "no", "noanswer"]) +def test_f1_special_token_gt_mismatch_returns_zero(special): + """Mirror check: special tokens in the ground truth (f1.py:35).""" + assert f1_score("some other phrase", special) == (0, 0, 0) + + +@pytest.mark.unit +def test_f1_special_token_exact_match_uses_token_path(): + """When prediction == ground_truth == "yes", the special-case early-exit + does NOT fire (it has ``!=`` guards), so we land on the token-F1 path + with a single common token → f1 = 1.0.""" + f1, p, r = f1_score("yes", "yes") + assert (f1, p, r) == (1.0, 1.0, 1.0) + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__])) diff --git a/tests/test_rm_gpqa.py b/tests/test_rm_gpqa.py new file mode 100644 index 000000000..add4f1aee --- /dev/null +++ b/tests/test_rm_gpqa.py @@ -0,0 +1,176 @@ +"""CPU unit tests for ``vime.rollout.rm_hub.gpqa``. + +Pins the GPQA rule-based scorer used by the ``gpqa`` rm_type. The whole +pipeline is pure-Python regex/string normalization with multiple +fall-through branches (extract-letter → match-correct-letter → text- +contains fallback), so a regex tweak or branch-order change can silently +shift rewards from 1.0 → 0.0 without any other CI signal. + +Test surface: + + - ``_extract_letter_from_response``: each of the three named patterns, + the standalone-letter fallback (last valid letter wins), and the + ```` chain-of-thought stripper. + - ``compute_gpqa_reward``: every label shape it accepts + (single-letter str, int index, full-text label) and every metadata + shape (``choices`` as list / dict, explicit ``correct_letter``, + explicit ``valid_letters``). + +Together these cover the silent-wrong-reward classes that string-parsing +scorers regress into. +""" + +from __future__ import annotations + +import pytest + +from vime.rollout.rm_hub.gpqa import DEFAULT_VALID_LETTERS, _extract_letter_from_response, compute_gpqa_reward + +# --------------------------------------------------------------------------- +# _extract_letter_from_response +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + "response,expected", + [ + # Pattern 1: "answer is X" / "option: X" / "choice X" + ("The answer is B.", "B"), + ("Answer: C", "C"), + ("My choice is D", "D"), + # Pattern 2: "X is correct" + ("A is correct here", "A"), + ("E is the correct option", "E"), + # Pattern 3: "final answer X" + ("Final answer: B", "B"), + ("the final option is C", "C"), + # Fallback: last standalone capital letter in valid set + ("we ruled out A, then B, settled on C", "C"), + ], +) +def test_extract_letter_named_patterns_and_fallback(response, expected): + assert _extract_letter_from_response(response, DEFAULT_VALID_LETTERS) == expected + + +@pytest.mark.unit +def test_extract_letter_strips_chain_of_thought_before_matching(): + """```` marker → keep only the trailing segment. Without the + strip, the earlier "Answer: A" would win over the real answer "B".""" + response = "Let me think… Answer: A is wrong.The answer is B." + assert _extract_letter_from_response(response, DEFAULT_VALID_LETTERS) == "B" + + +@pytest.mark.unit +def test_extract_letter_returns_none_on_no_match(): + """No pattern hit AND no valid standalone capital → None.""" + assert _extract_letter_from_response("no idea, sorry", DEFAULT_VALID_LETTERS) is None + + +@pytest.mark.unit +def test_extract_letter_returns_none_on_empty(): + assert _extract_letter_from_response("", DEFAULT_VALID_LETTERS) is None + assert _extract_letter_from_response(None, DEFAULT_VALID_LETTERS) is None + + +@pytest.mark.unit +def test_extract_letter_respects_valid_letters_restriction(): + """Restricting valid letters to {A, B} → an ``answer is C`` match is + rejected and the standalone-fallback used instead.""" + # The named-pattern match catches "C" but it's invalid; the fallback + # then walks standalone letters in reverse — "B" wins. + response = "Answer: C, but actually A was right, no wait B." + assert _extract_letter_from_response(response, ["A", "B"]) == "B" + + +# --------------------------------------------------------------------------- +# compute_gpqa_reward +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_reward_letter_label_match(): + """label is a single capital letter, no choices needed — exact letter match + short-circuits to 1.0.""" + reward = compute_gpqa_reward("The answer is C.", label="C") + assert reward == 1.0 + + +@pytest.mark.unit +def test_reward_letter_label_mismatch_returns_zero(): + assert compute_gpqa_reward("The answer is C.", label="D") == 0.0 + + +@pytest.mark.unit +def test_reward_int_label_maps_via_choices_length(): + """``label=2`` with 4 choices → valid_letters[2] = "C". Extracting "C" + from response should land on 1.0.""" + reward = compute_gpqa_reward("Answer: C", label=2, metadata={"choices": ["alpha", "beta", "gamma", "delta"]}) + assert reward == 1.0 + + +@pytest.mark.unit +def test_reward_choices_as_dict_is_accepted(): + """Some pipelines pass ``choices`` as an ordered dict — code path at + gpqa.py:62-63 unpacks via ``.values()``.""" + reward = compute_gpqa_reward( + "Answer: B", + label=1, + metadata={"choices": {"a": "first", "b": "second", "c": "third"}}, + ) + assert reward == 1.0 + + +@pytest.mark.unit +def test_reward_full_text_label_resolves_to_letter_via_choices(): + """label is the answer *text* — code matches normalized label against + each choice and resolves the position to a letter, then extracts the + letter from the response.""" + reward = compute_gpqa_reward( + "I think the answer is B.", + label="capital of france", + metadata={"choices": ["London", "Capital of France", "Berlin"]}, + ) + assert reward == 1.0 + + +@pytest.mark.unit +def test_reward_text_match_fallback_when_no_letter_extracted(): + """No letter pattern, but the response contains the answer text — the + fallback at gpqa.py:122-124 matches normalized text containment.""" + reward = compute_gpqa_reward( + "Definitely paris.", + label="Paris", + metadata={"choices": ["London", "Paris", "Berlin"]}, + ) + assert reward == 1.0 + + +@pytest.mark.unit +def test_reward_correct_letter_metadata_overrides_label(): + """``metadata.correct_letter`` takes precedence over label-derived + letter (gpqa.py:75-79). Useful for datasets where the label is the + full text but the correct option letter is pre-computed.""" + reward = compute_gpqa_reward( + "Answer: A", + label="anything", + metadata={"correct_letter": "a"}, # lowercase ok — gets upper()'d + ) + assert reward == 1.0 + + +@pytest.mark.unit +def test_reward_none_response_is_zero(): + """A nil response (custom_generate failure) must score 0, not crash.""" + assert compute_gpqa_reward(None, label="A") == 0.0 + + +@pytest.mark.unit +def test_reward_no_correct_letter_and_no_match_is_zero(): + """No metadata, label is non-letter text, response doesn't contain it — + every branch falls through to 0.0 at gpqa.py:129.""" + assert compute_gpqa_reward("random text", label="some answer") == 0.0 + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__])) diff --git a/tests/test_rm_math.py b/tests/test_rm_math.py new file mode 100644 index 000000000..06300e274 --- /dev/null +++ b/tests/test_rm_math.py @@ -0,0 +1,231 @@ +"""CPU unit tests for ``vime.rollout.rm_hub.math_utils``. + +Pins the boxed-answer extraction + lenient normalization used by every +math-style rm_type (math / deepscaler / dapo all funnel here). The +extraction is a hand-written brace counter; ``_strip_string`` is a long +chain of LaTeX-normalization regexes/replaces. Both are prime targets +for silent regressions — a wrong brace count or one bad replace can +quietly flip rewards from 1 → 0. + +This file covers the dependency-free pieces (pure string ops). The sympy +branch (``grade_answer_sympy``, ``are_equal_under_sympy``) is exercised +end-to-end through ``grade_answer_verl`` happy-path cases so we don't +have to re-derive sympy's normalization rules — just confirm the wiring +holds. +""" + +from __future__ import annotations + +import pytest + +from vime.rollout.rm_hub.math_utils import ( + _strip_string, + extract_answer, + extract_boxed_answer, + grade_answer_mathd, + grade_answer_sympy, + grade_answer_verl, + last_boxed_only_string, + mathd_normalize_answer, + remove_boxed, +) + + +# --------------------------------------------------------------------------- +# last_boxed_only_string — hand-rolled brace counter +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_last_boxed_returns_last_when_multiple(): + """When two boxed expressions exist, ``rfind`` picks the last one.""" + s = r"first attempt \boxed{wrong}, final \boxed{42}" + assert last_boxed_only_string(s) == r"\boxed{42}" + + +@pytest.mark.unit +def test_last_boxed_handles_nested_braces(): + """Brace counter must balance — nested braces inside the boxed expr + should be included, not cause early termination.""" + s = r"answer: \boxed{\frac{1}{2}}" + assert last_boxed_only_string(s) == r"\boxed{\frac{1}{2}}" + + +@pytest.mark.unit +def test_last_boxed_falls_back_to_fbox(): + """If no ``\\boxed`` is present, the function also accepts ``\\fbox``.""" + s = r"answer: \fbox{7}" + assert last_boxed_only_string(s) == r"\fbox{7}" + + +@pytest.mark.unit +def test_last_boxed_returns_none_when_missing(): + assert last_boxed_only_string("plain text, no box") is None + + +@pytest.mark.unit +def test_last_boxed_returns_none_on_unterminated_box(): + """Open brace never closes → braces stay imbalanced → return None + (not e.g. raise IndexError).""" + assert last_boxed_only_string(r"start \boxed{never closes") is None + + +# --------------------------------------------------------------------------- +# remove_boxed — strip the wrapper, leave the content +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_remove_boxed_strips_wrapper(): + assert remove_boxed(r"\boxed{x+1}") == "x+1" + + +@pytest.mark.unit +def test_remove_boxed_preserves_inner_braces(): + """Brace count is irrelevant here — the function just strips the prefix + and final ``}``; nested braces stay.""" + assert remove_boxed(r"\boxed{\frac{1}{2}}") == r"\frac{1}{2}" + + +@pytest.mark.unit +def test_remove_boxed_returns_none_on_malformed_input(): + """math_utils.remove_boxed wraps its asserts in try/except (412-419) — + silently returns None on bad input. ``math_dapo_utils.remove_boxed`` + behaves differently (raises); see test_rm_math_dapo for that contract. + """ + assert remove_boxed("not boxed at all") is None + assert remove_boxed(None) is None # type: ignore[arg-type] + + +# --------------------------------------------------------------------------- +# extract_boxed_answer / extract_answer — the convenience composition +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_extract_boxed_answer_end_to_end(): + assert extract_boxed_answer(r"Solution: \boxed{42}") == "42" + + +@pytest.mark.unit +def test_extract_answer_returns_none_when_no_boxed_marker(): + """``extract_answer`` only triggers when ``\\boxed`` is in the passage; + otherwise returns None (pinning the branch at math_utils.py:479).""" + assert extract_answer("just 42") is None + + +# --------------------------------------------------------------------------- +# mathd_normalize_answer / _strip_string — Hendrycks-style normalization +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + "raw,expected", + [ + # \\frac shorthand → \frac{a}{b}; non-frac stays + (r"\frac12", r"\frac{1}{2}"), + # tfrac/dfrac → frac + (r"\tfrac{1}{2}", r"\frac{1}{2}"), + # \left/\right wrappers removed + (r"\left(1\right)", "(1)"), + # degree symbol stripped (both variants) + (r"45^{\circ}", "45"), + (r"45^\circ", "45"), + # leading 0 added to .N → 0.N (but NOT .5 — that collides with + # the 0.5 → \frac{1}{2} special-case below; use .7 to isolate + # the leading-zero rule). + (".7", "0.7"), + # 0.5 → \frac{1}{2} convenience replacement (pinning the special + # case at math_utils.py:153-154). Note: .5 also lands here because + # the leading-zero rule runs first. + ("0.5", r"\frac{1}{2}"), + (".5", r"\frac{1}{2}"), # composition of leading-zero + 0.5 special + # a/b → \frac{a}{b} + ("3/4", r"\frac{3}{4}"), + # spaces collapsed away + ("1 + 1", "1+1"), + ], +) +def test_strip_string_canonical_substitutions(raw, expected): + assert _strip_string(raw) == expected + + +@pytest.mark.unit +def test_mathd_normalize_strips_text_wrapper(): + """A ``\\text{...}`` enclosing the whole answer is unwrapped before + further normalization (math_utils.py:21-23).""" + assert mathd_normalize_answer(r"\text{42}") == "42" + + +@pytest.mark.unit +def test_mathd_normalize_none_passthrough(): + assert mathd_normalize_answer(None) is None + + +# --------------------------------------------------------------------------- +# grade_answer_mathd — pure lexical equality after normalization +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + "given,gt", + [ + (r"\frac{1}{2}", "0.5"), # 0.5 → \frac{1}{2} + (r"\frac12", r"\frac{1}{2}"), # \frac shorthand expansion + (r"45^\circ", "45"), # degree marker stripped on both sides + ("3/4", r"\frac{3}{4}"), # a/b normalization + ], +) +def test_grade_answer_mathd_canonical_equivalences(given, gt): + assert grade_answer_mathd(given, gt) is True + + +@pytest.mark.unit +def test_grade_answer_mathd_rejects_different(): + assert grade_answer_mathd("42", "43") is False + + +# --------------------------------------------------------------------------- +# grade_answer_sympy / grade_answer_verl — sympy-backed paths (happy-path +# only; this isn't a sympy regression suite, just wiring confirmation) +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_grade_answer_sympy_equivalent_expressions(): + """Symbolically equal expressions should compare equal under sympy.""" + assert grade_answer_sympy("x+1", "1+x") is True + + +@pytest.mark.unit +def test_grade_answer_sympy_fraction_must_match_exactly(): + """Reducible fractions are intentionally NOT considered equal — pinning + the explicit ``_is_frac`` short-circuit at math_utils.py:453-456.""" + assert grade_answer_sympy(r"\frac{2}{4}", r"\frac{1}{2}") is False + + +@pytest.mark.unit +def test_grade_answer_verl_extracts_both_sides_from_boxed(): + """Both solution and ground_truth carry ``\\boxed{...}`` markers — the + function extracts then grades. Confirms the wiring at + math_utils.py:488-492 not just the inner mathd/sympy logic.""" + assert grade_answer_verl(r"answer: \boxed{42}", r"\boxed{42}") is True + + +@pytest.mark.unit +def test_grade_answer_verl_returns_false_on_missing_extraction(): + """No ``\\boxed`` in the solution → ``given_answer`` is None → False + (math_utils.py:491-492).""" + assert grade_answer_verl("just 42", "42") is False + + +@pytest.mark.unit +def test_grade_answer_verl_returns_false_on_empty_ground_truth(): + """Empty / falsy ground truth → False at the top guard.""" + assert grade_answer_verl(r"\boxed{42}", "") is False + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__])) diff --git a/tests/test_rm_math_dapo.py b/tests/test_rm_math_dapo.py new file mode 100644 index 000000000..f7ceb7351 --- /dev/null +++ b/tests/test_rm_math_dapo.py @@ -0,0 +1,230 @@ +"""CPU unit tests for ``vime.rollout.rm_hub.math_dapo_utils``. + +Pins the DAPO math scorer (``rm_type=dapo``). Distinct from +``math_utils`` in three ways the tests need to lock down: + + - ``remove_boxed`` here raises ``AssertionError`` on malformed input + (vs ``math_utils.remove_boxed`` which returns None silently) + - ``normalize_final_answer`` has its own pipeline (SUBSTITUTIONS list + + REMOVED_EXPRESSIONS list + per-unit regexes); silent drift here + causes wrong predicate matches in ``is_correct_minerva`` + - ``compute_score`` only considers the last 300 chars (efficiency + truncation at line 280) — a regression that drops this would silently + blow up scoring time on long traces +""" + +from __future__ import annotations + +import pytest + +from vime.rollout.rm_hub.math_dapo_utils import ( + compute_score, + is_correct_minerva, + is_correct_strict_box, + last_boxed_only_string, + normalize_final_answer, + remove_boxed, + verify, +) + + +# --------------------------------------------------------------------------- +# last_boxed_only_string — brace counter (separate impl from math_utils; +# this one expects ``\boxed{`` specifically, no \fbox fallback) +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_last_boxed_picks_rightmost(): + assert last_boxed_only_string(r"\boxed{first}, then \boxed{42}") == r"\boxed{42}" + + +@pytest.mark.unit +def test_last_boxed_balances_nested_braces(): + """Nested ``{}`` inside the box must not terminate the counter early.""" + assert last_boxed_only_string(r"\boxed{\frac{1}{2}}") == r"\boxed{\frac{1}{2}}" + + +@pytest.mark.unit +def test_last_boxed_returns_none_when_missing(): + assert last_boxed_only_string("no box") is None + + +@pytest.mark.unit +def test_last_boxed_returns_none_on_unterminated(): + """Imbalanced braces → ``right_brace_idx`` stays None → return None + (math_dapo_utils.py:47).""" + assert last_boxed_only_string(r"\boxed{never closes") is None + + +# --------------------------------------------------------------------------- +# remove_boxed — distinct from math_utils: this raises on bad input +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_remove_boxed_strips_wrapper(): + assert remove_boxed(r"\boxed{42}") == "42" + + +@pytest.mark.unit +def test_remove_boxed_raises_on_malformed(): + """Unlike ``math_utils.remove_boxed`` (try/except → None), the dapo + version asserts (line 60-61). A consumer that catches None will + silently break if these two implementations are later unified.""" + with pytest.raises(AssertionError, match="box error"): + remove_boxed("not boxed") + + +# --------------------------------------------------------------------------- +# normalize_final_answer — substitutions + removals + regex pipeline +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + "raw,expected", + [ + # "x = answer" → "answer" (split on '=' takes the last segment) + ("x = 42", "42"), + # SUBSTITUTIONS: \$ → "" (currency removal), commas in "text and" → "," + # Articles stripped — "an apple" → "apple" + ("an answer", "answer"), + # Unit removal: "square", "ways", etc. dropped silently + ("42 square", "42"), + ("100 dollars", "100"), + # \text{...} unwrapped via regex + (r"\text{hello}", "hello"), + # \boxed{...} unwrapped via regex (NOT via remove_boxed; lighter touch) + (r"\boxed{42}", "42"), + # Plain integer with commas → commas stripped + ("1,234,567", "1234567"), + # Spaces stripped via SUBSTITUTIONS (" " → "") + ("a b c", "bc"), # "a " is also substituted away (articles) + ], +) +def test_normalize_final_answer_canonical_substitutions(raw, expected): + assert normalize_final_answer(raw) == expected + + +@pytest.mark.unit +def test_normalize_final_answer_strips_end_tokens(): + """Generation end-markers should be stripped (REMOVED_EXPRESSIONS + line 138-139).""" + assert normalize_final_answer("42<|endoftext|>") == "42" + assert normalize_final_answer("42<|im_end|>") == "42" + + +# --------------------------------------------------------------------------- +# is_correct_strict_box — extract & exact-match within last 100 chars +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_is_correct_strict_box_matches(): + """Boxed prediction at end → extracted and compared to gt.""" + score, pred = is_correct_strict_box(r"long preamble … \boxed{42}", "42") + assert (score, pred) == (1, "42") + + +@pytest.mark.unit +def test_is_correct_strict_box_mismatch_returns_minus_one(): + """``compute_score`` later maps {1: 1.0, -1: -1.0} — pin both poles + here so the dispatching doesn't drift.""" + score, pred = is_correct_strict_box(r"\boxed{43}", "42") + assert (score, pred) == (-1, "43") + + +@pytest.mark.unit +def test_is_correct_strict_box_no_box_returns_minus_one(): + """No box at all → extracted_pred is None → mismatch path → -1.""" + score, pred = is_correct_strict_box("plain 42", "42") + assert score == -1 + assert pred is None + + +# --------------------------------------------------------------------------- +# is_correct_minerva — regex extract via "Answer:" then dapo-normalize +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_is_correct_minerva_matches_int_answer(): + """Minerva path expects gt to coerce via ``int(float(gt))`` (line 210), + so floats / int-strings collapse to canonical int strings before + comparison.""" + correct, pred = is_correct_minerva("Long solution. Answer: 42", "42") + assert correct is True + assert pred == "42" + + +@pytest.mark.unit +def test_is_correct_minerva_takes_last_answer_match(): + """Multiple ``Answer:`` lines → ``re.findall`` returns the list and + the function picks ``[-1]`` (line 201). Pinning this means a model + that re-states the answer at the end is still graded on the final one. + + Note: the regex captures up to a newline (``[^\\n]+``), so the two + candidates MUST be on separate lines — otherwise the first ``Answer:`` + greedily swallows the rest including the second one. + """ + text = "Answer: 41 was a wrong guess.\nAnswer: 42" + correct, pred = is_correct_minerva(text, "42") + assert correct is True + + +@pytest.mark.unit +def test_is_correct_minerva_no_answer_marker_is_invalid(): + """No ``Answer: ...`` in text → ``[INVALID]`` sentinel → mismatch.""" + correct, _ = is_correct_minerva("just a number 42", "42") + assert correct is False + + +# --------------------------------------------------------------------------- +# verify + compute_score — top-level public scoring entrypoint +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_verify_strict_box_dispatches_to_strict_box(): + """``strict_box_verify=True`` → strict-box path; correct = (score == 1).""" + correct, pred = verify(r"\boxed{42}", "42", strict_box_verify=True) + assert correct is True + assert pred == "42" + + +@pytest.mark.unit +def test_verify_default_dispatches_to_minerva(): + correct, pred = verify("Answer: 42", "42") + assert correct is True + + +@pytest.mark.unit +def test_compute_score_correct_returns_dict_with_reward_one(): + """Public API contract: dict shape with {score, acc, pred} keys.""" + out = compute_score(r"\boxed{42}", "42", strict_box_verify=True) + assert out == {"score": 1.0, "acc": True, "pred": "42"} + + +@pytest.mark.unit +def test_compute_score_incorrect_returns_minus_one(): + """The -1 reward (not 0) is the deliberate signal for "wrong" — pins + the explicit ``-1.0`` at line 285.""" + out = compute_score(r"\boxed{43}", "42", strict_box_verify=True) + assert out["score"] == -1.0 + assert out["acc"] is False + + +@pytest.mark.unit +def test_compute_score_only_uses_last_300_chars(): + """Efficiency truncation at line 280: only the tail is verified. A + correct boxed answer earlier in the string but absent in the tail + must score as incorrect — that's the explicit design choice.""" + # Put the correct boxed expr at the very start, then 400 chars of noise. + sol = r"\boxed{42}" + (" filler" * 60) # >300 chars after the box + out = compute_score(sol, "42", strict_box_verify=True) + assert out["score"] == -1.0 # truncated away → not found + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__])) diff --git a/vime/utils/misc.py b/vime/utils/misc.py index 8ac600366..5b643987c 100644 --- a/vime/utils/misc.py +++ b/vime/utils/misc.py @@ -1,8 +1,6 @@ import importlib import subprocess -import ray - from vime.utils.http_utils import is_port_available @@ -56,6 +54,10 @@ def exec_command(cmd: str, capture_output: bool = False) -> str | None: def get_current_node_ip(): + # Lazy import so CPU-only code paths (rm_hub scoring, plugin contracts, + # etc.) can use other helpers in this module without requiring ray. + import ray + address = ray._private.services.get_node_ip_address() # strip ipv6 address address = address.strip("[]")