docs: add checkpointing, metrics tracking, and workflow guides

areal/api/cli_args.py

@@ -1538,17 +1538,24 @@ class RecoverConfig(_Timer):
         metadata={
             "help": "Recovery mode for the launcher. "
             "Options: "
-            "'disabled': Never recover from previous runs. "
-            "'auto': Automatically recover from previous runs if recover info and checkpoints are available. "
-            "'fault': Only recover from previous runs if the new run fails. "
-            "'resume': Force to resume, raise an error if no recover info was found. Never resume if failed again."
+            "'on' or 'auto': Automatically recover from previous runs if recover info and checkpoints are available. "
+            "'off' or 'disabled': Never recover from previous runs."
         },
     )
     retries: int = field(
         default=3,
-        metadata={"help": "Number of recovery retries (auto/fault modes only)."},
+        metadata={"help": "Number of recovery retries when recovery is enabled."},
     )
 
+    def __post_init__(self):
+        valid_modes = {"on", "off", "auto", "disabled"}
+        if self.mode not in valid_modes:
+            raise ValueError(
+                f"Invalid recover mode '{self.mode}'. "
+                f"Valid options: {valid_modes}. "
+                f"Note: 'fault' and 'resume' modes have been removed."
+            )
+
 
 @dataclass
 class WandBConfig:

areal/launcher/local.py

@@ -367,7 +367,6 @@ def local_main(config, run_id: int = 0):
         gpu = nprocs = alloc_mode.train.world_size
         _env_vars = dict(
             AREAL_LLM_SERVER_ADDRS=",".join(server_addrs),
-            AREAL_RECOVER_RUN=str(int(is_recover_run)),
         )
         if alloc_mode.gen_backend == "sglang":
             # Required by NCCL weight update group.
@@ -414,7 +413,7 @@ def local_main(config, run_id: int = 0):
             recover_this = (
                 e.reason in recover_states
                 and run_id < config.recover.retries
-                and config.recover.mode in ["auto", "fault"]
+                and config.recover.mode in ("on", "auto")
             )
             if recover_this:
                 time.sleep(RECOVER_TIME_INTERVAL)

areal/launcher/ray.py

@@ -564,7 +564,6 @@ def torch_env_hook(n_tasks: int, placement_group: PlacementGroup) -> list[dict]:
 
         _env_vars = dict(
             AREAL_LLM_SERVER_ADDRS=",".join(llm_addrs),
-            AREAL_RECOVER_RUN=str(int(is_recover_run)),
         )
         if allocation_mode.gen_backend == "sglang":
             # Required by NCCL weight update group.
@@ -617,7 +616,7 @@ def torch_env_hook(n_tasks: int, placement_group: PlacementGroup) -> list[dict]:
             recover_this = (
                 e.reason in recover_states
                 and run_id < config.recover.retries
-                and config.recover.mode in ["auto", "fault"]
+                and config.recover.mode in ("on", "auto")
             )
             if recover_this:
                 time.sleep(RECOVER_TIME_INTERVAL)

areal/launcher/slurm.py

@@ -609,7 +609,6 @@ def _build_trainer_cmds(
         # launch trainers
         _env_vars = dict(
             AREAL_LLM_SERVER_ADDRS=",".join(llm_addrs),
-            AREAL_RECOVER_RUN=str(int(is_recover_run)),
         )
         if allocation_mode.gen_backend == "sglang":
             # Required by NCCL weight update group.
@@ -666,7 +665,7 @@ def _build_trainer_cmds(
             recover_this = (
                 e.reason in recover_states
                 and run_id < config.recover.retries
-                and config.recover.mode in ["auto", "fault"]
+                and config.recover.mode in ("on", "auto")
             )
             if recover_this:
                 time.sleep(RECOVER_TIME_INTERVAL)

areal/scheduler/slurm.py

@@ -1,6 +1,5 @@
 import asyncio
 import getpass
-import os
 import re
 import shlex
 import subprocess
@@ -401,7 +400,6 @@ def _prepare_worker_specs(
         # Amend environment variables
         for sch in schedulings:
             # AReaL env var forwarding
-            sch.env_vars["AREAL_RECOVER_RUN"] = os.getenv("AREAL_RECOVER_RUN", str(0))
             if self.enable_tms_offload:
                 sch.env_vars.update(get_tms_env_vars())
             sch.env_vars.update(get_env_vars())

areal/tests/test_recover.py

@@ -0,0 +1,169 @@
+"""Tests for the recovery configuration and functionality."""
+
+import tempfile
+
+import pytest
+
+from areal.api.cli_args import RecoverConfig
+from areal.utils.recover import check_if_auto_recover, check_if_recover
+
+
+class TestRecoverConfig:
+    """Tests for RecoverConfig dataclass validation."""
+
+    def test_default_values(self):
+        """Test that default values are set correctly."""
+        config = RecoverConfig(
+            experiment_name="test_exp",
+            trial_name="test_trial",
+            fileroot="/tmp",
+        )
+        assert config.mode == "disabled"
+        assert config.retries == 3
+
+    @pytest.mark.parametrize("mode", ["on", "off", "auto", "disabled"])
+    def test_valid_modes(self, mode):
+        """Test that all valid modes are accepted."""
+        config = RecoverConfig(
+            experiment_name="test_exp",
+            trial_name="test_trial",
+            fileroot="/tmp",
+            mode=mode,
+        )
+        assert config.mode == mode
+
+    @pytest.mark.parametrize("mode", ["fault", "resume", "invalid", "ON", "OFF", ""])
+    def test_invalid_modes(self, mode):
+        """Test that invalid modes raise ValueError with helpful message."""
+        with pytest.raises(ValueError) as exc_info:
+            RecoverConfig(
+                experiment_name="test_exp",
+                trial_name="test_trial",
+                fileroot="/tmp",
+                mode=mode,
+            )
+        error_msg = str(exc_info.value)
+        assert f"Invalid recover mode '{mode}'" in error_msg
+        assert "fault" in error_msg and "resume" in error_msg  # Migration hint
+
+
+class TestCheckIfRecover:
+    """Tests for the check_if_recover function."""
+
+    @pytest.mark.parametrize("mode", ["disabled", "off"])
+    def test_disabled_modes_return_false(self, mode):
+        """Test that disabled modes always return False."""
+        config = RecoverConfig(
+            experiment_name="test_exp",
+            trial_name="test_trial",
+            fileroot="/tmp",
+            mode=mode,
+        )
+        # Should return False regardless of run_id
+        assert check_if_recover(config, 0) is False
+        assert check_if_recover(config, 1) is False
+        assert check_if_recover(config, 10) is False
+
+    @pytest.mark.parametrize("mode", ["on", "auto"])
+    def test_enabled_modes_check_for_checkpoint(self, mode):
+        """Test that enabled modes check for existing checkpoints."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            config = RecoverConfig(
+                experiment_name="test_exp",
+                trial_name="test_trial",
+                fileroot=tmpdir,
+                mode=mode,
+            )
+            # No checkpoint exists, should return False
+            assert check_if_recover(config, 0) is False
+
+    @pytest.mark.parametrize("run_id", [0, 1, 5, 100])
+    def test_run_id_parameter_unused(self, run_id):
+        """Test that run_id parameter doesn't affect the result."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            # Test with disabled mode
+            config_disabled = RecoverConfig(
+                experiment_name="test_exp",
+                trial_name="test_trial",
+                fileroot=tmpdir,
+                mode="disabled",
+            )
+            assert check_if_recover(config_disabled, run_id) is False
+
+            # Test with enabled mode (no checkpoint)
+            config_enabled = RecoverConfig(
+                experiment_name="test_exp",
+                trial_name="test_trial",
+                fileroot=tmpdir,
+                mode="on",
+            )
+            # Result should be the same regardless of run_id
+            result = check_if_recover(config_enabled, run_id)
+            assert result == check_if_recover(config_enabled, 0)
+
+
+class TestCheckIfAutoRecover:
+    """Tests for the check_if_auto_recover function."""
+
+    def test_no_checkpoint_returns_false(self):
+        """Test that missing checkpoint returns False."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            config = RecoverConfig(
+                experiment_name="test_exp",
+                trial_name="test_trial",
+                fileroot=tmpdir,
+                mode="on",
+            )
+            assert check_if_auto_recover(config) is False
+
+    def test_empty_directory_returns_false(self):
+        """Test that empty directory (no checkpoint) returns False."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            config = RecoverConfig(
+                experiment_name="test_exp",
+                trial_name="test_trial",
+                fileroot=tmpdir,
+                mode="on",
+            )
+            assert check_if_auto_recover(config) is False
+
+
+class TestModeEquivalence:
+    """Tests to verify mode equivalences (on=auto, off=disabled)."""
+
+    def test_on_equals_auto(self):
+        """Test that 'on' and 'auto' modes behave identically."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            config_on = RecoverConfig(
+                experiment_name="test_exp",
+                trial_name="test_trial",
+                fileroot=tmpdir,
+                mode="on",
+            )
+            config_auto = RecoverConfig(
+                experiment_name="test_exp",
+                trial_name="test_trial",
+                fileroot=tmpdir,
+                mode="auto",
+            )
+            # Both should return the same result
+            assert check_if_recover(config_on, 0) == check_if_recover(config_auto, 0)
+
+    def test_off_equals_disabled(self):
+        """Test that 'off' and 'disabled' modes behave identically."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            config_off = RecoverConfig(
+                experiment_name="test_exp",
+                trial_name="test_trial",
+                fileroot=tmpdir,
+                mode="off",
+            )
+            config_disabled = RecoverConfig(
+                experiment_name="test_exp",
+                trial_name="test_trial",
+                fileroot=tmpdir,
+                mode="disabled",
+            )
+            # Both should return False
+            assert check_if_recover(config_off, 0) is False
+            assert check_if_recover(config_disabled, 0) is False

areal/utils/recover.py

@@ -176,7 +176,7 @@ def dump(
         processor: AutoProcessor | None = None,
         base_model_path: str | None = None,
     ):
-        if self.config.mode == "disabled":
+        if self.config.mode in ("disabled", "off"):
             return
         # currently only support recover on one engine
         if not self.freq_ctl.check(
@@ -223,9 +223,7 @@ def load(
         weight_update_meta: WeightUpdateMeta | None = None,
         inference_engine_update_from: str = "default",
     ) -> RecoverInfo | None:
-        if self.config.mode == "disabled":
-            return
-        if os.environ.get("AREAL_RECOVER_RUN", "0") != "1":
+        if self.config.mode in ("disabled", "off"):
             return
         if inference_engine is not None and weight_update_meta is None:
             raise ValueError("Weight update meta is required for recovery.")
@@ -324,8 +322,8 @@ def _load_checkpoint(
 
 
 def check_if_auto_recover(config: RecoverConfig) -> bool:
-    # This method is called only by launchers to check if the experiment should be a recover run
-    # when "recover_mode" is auto.
+    # This method is called by check_if_recover to check if the experiment should
+    # recover from a previous run when recovery is enabled ("on" or "auto" mode).
     experiment_name = config.experiment_name
     trial_name = config.trial_name
     fileroot = config.fileroot
@@ -363,16 +361,20 @@ def check_if_auto_recover(config: RecoverConfig) -> bool:
     return False
 
 
-def check_if_recover(config: RecoverConfig, run_id: int) -> bool:
-    # This method is called by the launcher to check if the experiment should be a recover run
-    # when "recover_mode" is not disabled.
-    if config.mode == "disabled":
+def check_if_recover(config: RecoverConfig, _run_id: int) -> bool:
+    """Check if the experiment should be a recover run.
+
+    When recovery is enabled ('on' or 'auto'), this checks if valid recover
+    info and checkpoints are available for automatic recovery.
+
+    Args:
+        config: Recovery configuration.
+        _run_id: Unused. Kept for API compatibility.
+
+    Returns:
+        True if the experiment should recover from a previous run.
+    """
+    if config.mode in ("disabled", "off"):
         return False
-    elif config.mode == "auto":
-        return check_if_auto_recover(config)
-    elif config.mode == "fault":
-        return run_id > 0
-    elif config.mode == "resume":
-        return True
-    else:
-        raise ValueError(f"Unknown recover mode: {config.mode}")
+    # Both "on" and "auto" use auto-recovery behavior
+    return check_if_auto_recover(config)

docs/_toc.yml

@@ -7,22 +7,22 @@ parts:
   - caption: Version History
     chapters:
     - file: version_history
-  - caption: Getting Started
-    chapters:
-    - file: tutorial/gsm8k_grpo
   - caption: Tutorial
     chapters:
     - file: tutorial/installation
     - file: tutorial/installation_npu
     - file: tutorial/quickstart
-    - file: tutorial/openai_workflows
     - file: tutorial/agentic_rl
     - file: tutorial/eval
     - file: tutorial/megatron
     - file: cli_reference
+  - caption: Code Walkthrough
+    chapters:
+    - file: tutorial/gsm8k_grpo
   - caption: Best Practices
     chapters:
     - file: best_practices/algo_perf
+    - file: best_practices/workflow
     - file: best_practices/debugging
     - file: best_practices/handling_oom
     - file: best_practices/perf_profiling
@@ -37,3 +37,9 @@ parts:
     - file: algorithms/grpo_series
     - file: algorithms/m2po
     - file: algorithms/prox_approx
+  - caption: Reference
+    chapters:
+    - file: reference/checkpointing
+    - file: reference/metrics_tracking
+    - file: reference/alloc_mode
+    - file: reference/rollout_workflow