Add missing experimental autodoc classes to docs

docs/source/bema_for_reference_model.md

@@ -29,3 +29,14 @@ trainer = DPOTrainer(
 
 trainer.train()
 ```
+
+## DPOTrainer
+
+[[autodoc]] experimental.bema_for_ref_model.DPOTrainer
+    - train
+    - save_model
+    - push_to_hub
+
+## BEMACallback
+
+[[autodoc]] experimental.bema_for_ref_model.BEMACallback

docs/source/gfpo.md

@@ -37,3 +37,14 @@ trainer = GFPOTrainer(
 )
 trainer.train()
 ```
+
+## GFPOTrainer
+
+[[autodoc]] experimental.gfpo.GFPOTrainer
+    - train
+    - save_model
+    - push_to_hub
+
+## GFPOConfig
+
+[[autodoc]] experimental.gfpo.GFPOConfig

docs/source/grpo_with_replay_buffer.md

@@ -39,3 +39,18 @@ previous_trainable_params = {n: param.clone() for n, param in trainer.model.name
 
 trainer.train()
 ```
+
+## GRPOWithReplayBufferTrainer
+
+[[autodoc]] experimental.grpo_with_replay_buffer.GRPOWithReplayBufferTrainer
+    - train
+    - save_model
+    - push_to_hub
+
+## GRPOWithReplayBufferConfig
+
+[[autodoc]] experimental.grpo_with_replay_buffer.GRPOWithReplayBufferConfig
+
+## ReplayBuffer
+
+[[autodoc]] experimental.grpo_with_replay_buffer.ReplayBuffer

docs/source/gspo_token.md

@@ -16,3 +16,10 @@ training_args = GRPOConfig(
 
 > [!WARNING]
 > To leverage GSPO-token, the user will need to provide the per-token advantage  \\( \hat{A_{i,t}} \\) for each token  \\( t \\) in the sequence  \\( i \\) (i.e., make  \\( \hat{A_{i,t}} \\) varies with  \\( t \\)—which isn't the case here,  \\( \hat{A_{i,t}}=\hat{A_{i}} \\)). Otherwise, GSPO-Token gradient is just equivalent to the original GSPO implementation.
+
+## GRPOTrainer
+
+[[autodoc]] experimental.gspo_token.GRPOTrainer
+    - train
+    - save_model
+    - push_to_hub

docs/source/merge_model_callback.md

@@ -1,3 +1,3 @@
 # MergeModelCallback
 
-[[autodoc]] MergeModelCallback
+[[autodoc]] experimental.merge_model_callback.MergeModelCallback

trl/experimental/merge_model_callback.py

@@ -268,12 +268,12 @@ def create(self) -> "MergeConfiguration":
             return self.create_merge_config_slerp()
 
 
-def merge_models(config: MergeConfig, out_path: str):
+def merge_models(config: "MergeConfiguration", out_path: str):
     """
     Merge two models using mergekit
 
     Args:
-        config ([`MergeConfig`]): The merge configuration.
+        config (`MergeConfiguration`): The merge configuration.
         out_path (`str`): The output path for the merged model.
     """
     if not is_mergekit_available():
@@ -297,8 +297,8 @@ class MergeModelCallback(TrainerCallback):
     on a merge configuration.
 
     Args:
-        merge_config ([`MergeConfig`], *optional*):
-            Configuration used for the merging process. If not provided, the default [`MergeConfig`] is used.
+        merge_config ([`experimental.merge_model_callback.MergeConfig`], *optional*):
+            Configuration used for the merging process. If not provided, the default [`~experimental.merge_model_callback.MergeConfig`] is used.
         merge_at_every_checkpoint (`bool`, *optional*, defaults to `False`):
             Whether to merge the model at every checkpoint.
         push_to_hub (`bool`, *optional*, defaults to `False`):
@@ -307,7 +307,7 @@ class MergeModelCallback(TrainerCallback):
     Example:
 
     ```python
-    from trl.experiemental.merge_model_callback import MergeConfig, MergeModelCallback
+    from trl.experimental.merge_model_callback import MergeConfig, MergeModelCallback
 
     config = MergeConfig()
     merge_callback = MergeModelCallback(config)