diff --git a/python/ray/rllib/agents/qmix/qmix_policy.py b/python/ray/rllib/agents/qmix/qmix_policy.py
index 26ec387de004..99045899684b 100644
--- a/python/ray/rllib/agents/qmix/qmix_policy.py
+++ b/python/ray/rllib/agents/qmix/qmix_policy.py
@@ -204,6 +204,8 @@ def __init__(self, obs_space, action_space, config):
 
         # Setup optimizer
         self.params = list(self.model.parameters())
+        if self.mixer:
+            self.params += list(self.mixer.parameters())
         self.loss = QMixLoss(self.model, self.target_model, self.mixer,
                              self.target_mixer, self.n_agents, self.n_actions,
                              self.config["double_q"], self.config["gamma"])