docs: add checkpointing, metrics tracking, and workflow guides

.claude/rules/testing.md

@@ -33,7 +33,7 @@ def test_<what>_<condition>_<expected>():
 
 - Use `torch.distributed.fake_pg` for unit tests
 - Mock `dist.get_rank()` and `dist.get_world_size()` explicitly
-- Don't mock internals of FSDP/DTensor — use integration tests
+- Don't mock internals of FSDP/DTensor - use integration tests
 
 ## GPU Test Constraints
 
@@ -58,4 +58,4 @@ def test_<what>_<condition>_<expected>():
 
 - Use `torch.testing.assert_close()` for tensor comparison
 - Specify `rtol`/`atol` explicitly for numerical tests
-- Avoid bare `assert tensor.equal()` — no useful error message
+- Avoid bare `assert tensor.equal()` - no useful error message

.claude/skills/add-unit-tests/SKILL.md

@@ -150,7 +150,7 @@ def test_gpu_function():
 
 - Use `torch.testing.assert_close()` for tensor comparison
 - Specify `rtol`/`atol` explicitly for numerical tests
-- Avoid bare `assert tensor.equal()` — no useful error message
+- Avoid bare `assert tensor.equal()` - no useful error message
 
 ## Reference Implementations
 

AGENTS.md

@@ -1,6 +1,6 @@
 <!-- Think of this file as the go-to brief for AI coding agents working on AReaL. -->
 
-# AGENTS.md — AReaL Agent Operations Guide
+# AGENTS.md - AReaL Agent Operations Guide
 
 ## TL;DR for coding agents
 
@@ -22,54 +22,54 @@ When unsure, leave a `TODO(agent)` comment and note the constraint in your respo
 
 ## Repository map
 
-- `.claude/` — Configuration directory for AI coding agents:
-  - `.claude/agents/` — Specialized agent definitions (planner, code-verifier,
+- `.claude/` - Configuration directory for AI coding agents:
+  - `.claude/agents/` - Specialized agent definitions (planner, code-verifier,
     simple-code-reviewer, engine experts, algorithm-expert, launcher-scheduler-expert)
-  - `.claude/commands/` — User-invocable commands (create-pr, gen-commit-msg, pr-review)
-  - `.claude/rules/` — Project-wide code quality standards (api-config, code-style,
+  - `.claude/commands/` - User-invocable commands (create-pr, gen-commit-msg, pr-review)
+  - `.claude/rules/` - Project-wide code quality standards (api-config, code-style,
     distributed, testing)
-  - `.claude/skills/` — Guided development workflows (add-dataset, add-workflow,
+  - `.claude/skills/` - Guided development workflows (add-dataset, add-workflow,
     add-reward, add-unit-tests, debug-distributed)
-  - `.claude/hooks/` — Event-driven automation hooks
-  - `.claude/data/` — Agent working data and caches
-  - `.claude/plans/` — Implementation plans generated by planner agent
-- `areal/` — Core Python package housing APIs, controllers, engines, workflows, and
+  - `.claude/hooks/` - Event-driven automation hooks
+  - `.claude/data/` - Agent working data and caches
+  - `.claude/plans/` - Implementation plans generated by planner agent
+- `areal/` - Core Python package housing APIs, controllers, engines, workflows, and
   shared utilities:
-  - `areal/api/` — Contracts for workflows, engines, schedulers, IO structs, and
+  - `areal/api/` - Contracts for workflows, engines, schedulers, IO structs, and
     CLI/config dataclasses.
-  - `areal/infra/` — Single controller implementation, async orchestration primitives,
+  - `areal/infra/` - Single controller implementation, async orchestration primitives,
     and hardware/platform abstractions for CPU/GPU/NPU runtimes.
-  - `areal/dataset/` — Stateful dataset loaders (GSM8K, Geometry3K, CLEVR, HH-RLHF,
+  - `areal/dataset/` - Stateful dataset loaders (GSM8K, Geometry3K, CLEVR, HH-RLHF,
     TORL, etc.) and utilities that feed rollout jobs safely.
-  - `areal/engine/` — Training backends (FSDP2, Megatron, PPO, SFT, reward modeling) and
+  - `areal/engine/` - Training backends (FSDP2, Megatron, PPO, SFT, reward modeling) and
     inference adapters (SGLang, vLLM remote engines).
-  - `areal/experimental/` — Prototype engines/workflows that evolve quickly; expect
+  - `areal/experimental/` - Prototype engines/workflows that evolve quickly; expect
     breaking changes.
-  - `areal/launcher/` — Launch specs for local, Ray, and Slurm clusters, plus
+  - `areal/launcher/` - Launch specs for local, Ray, and Slurm clusters, plus
     SGLang/vLLM inference server launchers and container guidance.
-  - `areal/models/` — Model-specific adapters (Megatron-Core layers, Transformers
+  - `areal/models/` - Model-specific adapters (Megatron-Core layers, Transformers
     wrappers, custom heads).
-  - `areal/reward/` — Built-in reward functions (GSM8K, Geometry3K, CLEVR, etc.), math
+  - `areal/reward/` - Built-in reward functions (GSM8K, Geometry3K, CLEVR, etc.), math
     parsers, and helpers; wrap slow logic with `AsyncRewardWrapper`.
-  - `areal/scheduler/` — Placement and allocation policies aligned with launcher
+  - `areal/scheduler/` - Placement and allocation policies aligned with launcher
     configs.
-  - `areal/tests/` — Focused unit/integration suites (many require GPUs or mocked
+  - `areal/tests/` - Focused unit/integration suites (many require GPUs or mocked
     distributed backends).
-  - `areal/tools/` — Developer utilities and maintenance scripts tied to the core
+  - `areal/tools/` - Developer utilities and maintenance scripts tied to the core
     package.
-  - `areal/utils/` — Cross-cutting helpers for logging, tensor ops, stats tracking,
+  - `areal/utils/` - Cross-cutting helpers for logging, tensor ops, stats tracking,
     checkpoints, and recovery.
-  - `areal/workflow/` — Concrete rollout agents implementing `RolloutWorkflow`:
+  - `areal/workflow/` - Concrete rollout agents implementing `RolloutWorkflow`:
     multi-turn, RLVR, vision RLVR workflows, plus `openai_agent/` for OpenAI Agent-style
     implementations.
-- `assets/` — Figures and other static assets referenced across docs and blogs.
-- `blog/` — Release notes and update write-ups documenting project progress.
-- `docs/` — Jupyter Book source for https://inclusionai.github.io/AReaL/ plus CLI
+- `assets/` - Figures and other static assets referenced across docs and blogs.
+- `blog/` - Release notes and update write-ups documenting project progress.
+- `docs/` - Jupyter Book source for https://inclusionai.github.io/AReaL/ plus CLI
   reference generators.
-- `examples/` — End-to-end training scripts and launcher recipes for math reasoning,
+- `examples/` - End-to-end training scripts and launcher recipes for math reasoning,
   multi-turn conversations, VLM, RLHF alignment, agent-based workflows, LoRA
   fine-tuning, SkyPilot deployment, and more.
-- `notebook/` — Reference notebooks (outputs stripped via pre-commit) for quick
+- `notebook/` - Reference notebooks (outputs stripped via pre-commit) for quick
   experimentation.
 
 ## Distributed operations & tooling

CLAUDE.md

@@ -1,4 +1,4 @@
-# CLAUDE.md — AReaL
+# CLAUDE.md - AReaL
 
 ## WHAT: Project Overview
 
@@ -9,15 +9,15 @@ learning.
 
 **Core Directories**:
 
-- `areal/` — Core package
-  - `api/` — Config dataclasses, workflow/engine contracts
-  - `engine/` — FSDP2, Megatron, SGLang/vLLM adapters
-  - `workflow/` — RolloutWorkflow implementations
-  - `reward/` — Reward functions
-  - `dataset/` — Dataset loaders
-  - `utils/` — Logging, tensor ops, checkpoints
-- `examples/` — Training scripts and configs
-- `docs/` — Jupyter Book source
+- `areal/` - Core package
+  - `api/` - Config dataclasses, workflow/engine contracts
+  - `engine/` - FSDP2, Megatron, SGLang/vLLM adapters
+  - `workflow/` - RolloutWorkflow implementations
+  - `reward/` - Reward functions
+  - `dataset/` - Dataset loaders
+  - `utils/` - Logging, tensor ops, checkpoints
+- `examples/` - Training scripts and configs
+- `docs/` - Jupyter Book source
 
 ## WHY: Purpose
 

areal/README.md

@@ -23,17 +23,17 @@ principles:
 To achieve an *algorithm-first* and *lightweight* design while maintaining efficiency,
 AReaL is guided by seven core principles:
 
-1. **Native asynchronous RL training** — Built from the ground up for decoupled
+1. **Native asynchronous RL training** - Built from the ground up for decoupled
    generation and training
-1. **System-abstracted design** — Minimize exposure to low-level system concepts like
+1. **System-abstracted design** - Minimize exposure to low-level system concepts like
    "PlacementGroup"
-1. **PyTorch-centric approach** — Use native PyTorch types without unnecessary
+1. **PyTorch-centric approach** - Use native PyTorch types without unnecessary
    abstractions
-1. **Transparent orchestration** — Make the flow of operations clear and understandable
-1. **Developer-friendly navigation** — Enable easy access to implementation details via
+1. **Transparent orchestration** - Make the flow of operations clear and understandable
+1. **Developer-friendly navigation** - Enable easy access to implementation details via
    IDE features (Ctrl+click)
-1. **Ecosystem compatibility** — Integrate seamlessly with existing ML/RL tools
-1. **Single-file customization** — Support RL pipeline modifications within a single
+1. **Ecosystem compatibility** - Integrate seamlessly with existing ML/RL tools
+1. **Single-file customization** - Support RL pipeline modifications within a single
    file
 
 ## Architecture

areal/api/cli_args.py

@@ -1538,17 +1538,24 @@ class RecoverConfig(_Timer):
         metadata={
             "help": "Recovery mode for the launcher. "
             "Options: "
-            "'disabled': Never recover from previous runs. "
-            "'auto': Automatically recover from previous runs if recover info and checkpoints are available. "
-            "'fault': Only recover from previous runs if the new run fails. "
-            "'resume': Force to resume, raise an error if no recover info was found. Never resume if failed again."
+            "'on' or 'auto': Automatically recover from previous runs if recover info and checkpoints are available. "
+            "'off' or 'disabled': Never recover from previous runs."
         },
     )
     retries: int = field(
         default=3,
-        metadata={"help": "Number of recovery retries (auto/fault modes only)."},
+        metadata={"help": "Number of recovery retries when recovery is enabled."},
     )
 
+    def __post_init__(self):
+        valid_modes = {"on", "off", "auto", "disabled"}
+        if self.mode not in valid_modes:
+            raise ValueError(
+                f"Invalid recover mode '{self.mode}'. "
+                f"Valid options: {valid_modes}. "
+                f"Note: 'fault' and 'resume' modes have been removed."
+            )
+
 
 @dataclass
 class WandBConfig:

areal/launcher/local.py

@@ -367,7 +367,6 @@ def local_main(config, run_id: int = 0):
         gpu = nprocs = alloc_mode.train.world_size
         _env_vars = dict(
             AREAL_LLM_SERVER_ADDRS=",".join(server_addrs),
-            AREAL_RECOVER_RUN=str(int(is_recover_run)),
         )
         if alloc_mode.gen_backend == "sglang":
             # Required by NCCL weight update group.
@@ -414,7 +413,7 @@ def local_main(config, run_id: int = 0):
             recover_this = (
                 e.reason in recover_states
                 and run_id < config.recover.retries
-                and config.recover.mode in ["auto", "fault"]
+                and config.recover.mode in ("on", "auto")
             )
             if recover_this:
                 time.sleep(RECOVER_TIME_INTERVAL)

areal/launcher/ray.py

@@ -564,7 +564,6 @@ def torch_env_hook(n_tasks: int, placement_group: PlacementGroup) -> list[dict]:
 
         _env_vars = dict(
             AREAL_LLM_SERVER_ADDRS=",".join(llm_addrs),
-            AREAL_RECOVER_RUN=str(int(is_recover_run)),
         )
         if allocation_mode.gen_backend == "sglang":
             # Required by NCCL weight update group.
@@ -617,7 +616,7 @@ def torch_env_hook(n_tasks: int, placement_group: PlacementGroup) -> list[dict]:
             recover_this = (
                 e.reason in recover_states
                 and run_id < config.recover.retries
-                and config.recover.mode in ["auto", "fault"]
+                and config.recover.mode in ("on", "auto")
             )
             if recover_this:
                 time.sleep(RECOVER_TIME_INTERVAL)

areal/launcher/slurm.py

@@ -609,7 +609,6 @@ def _build_trainer_cmds(
         # launch trainers
         _env_vars = dict(
             AREAL_LLM_SERVER_ADDRS=",".join(llm_addrs),
-            AREAL_RECOVER_RUN=str(int(is_recover_run)),
         )
         if allocation_mode.gen_backend == "sglang":
             # Required by NCCL weight update group.
@@ -666,7 +665,7 @@ def _build_trainer_cmds(
             recover_this = (
                 e.reason in recover_states
                 and run_id < config.recover.retries
-                and config.recover.mode in ["auto", "fault"]
+                and config.recover.mode in ("on", "auto")
             )
             if recover_this:
                 time.sleep(RECOVER_TIME_INTERVAL)

areal/scheduler/slurm.py

@@ -1,6 +1,5 @@
 import asyncio
 import getpass
-import os
 import re
 import shlex
 import subprocess
@@ -401,7 +400,6 @@ def _prepare_worker_specs(
         # Amend environment variables
         for sch in schedulings:
             # AReaL env var forwarding
-            sch.env_vars["AREAL_RECOVER_RUN"] = os.getenv("AREAL_RECOVER_RUN", str(0))
             if self.enable_tms_offload:
                 sch.env_vars.update(get_tms_env_vars())
             sch.env_vars.update(get_env_vars())