diff --git a/trl/trainer/ppov2_config.py b/trl/trainer/ppov2_config.py
index 1761d16e304..3cb33d0e5c6 100644
--- a/trl/trainer/ppov2_config.py
+++ b/trl/trainer/ppov2_config.py
@@ -12,7 +12,7 @@
 
 
 @dataclass
-class PPOv2Config(TrainingArguments, OnpolicyRuntimeConfig):
+class PPOv2Config(OnpolicyRuntimeConfig, TrainingArguments):
     # common config
     exp_name: str = os.path.basename(__file__)[: -len(".py")]
     """the name of this experiment"""
diff --git a/trl/trainer/rloo_config.py b/trl/trainer/rloo_config.py
index 20e353a33fc..dd1039c3f47 100644
--- a/trl/trainer/rloo_config.py
+++ b/trl/trainer/rloo_config.py
@@ -13,7 +13,7 @@
 
 
 @dataclass
-class RLOOConfig(TrainingArguments, OnpolicyRuntimeConfig):
+class RLOOConfig(OnpolicyRuntimeConfig, TrainingArguments):
     # common config
     exp_name: str = os.path.basename(__file__)[: -len(".py")]
     """the name of this experiment"""