optimizer state values should be taken as mean absolute otherwise you could underestimate changes

Author: grantbuster

sup3r/models/abstract.py

@@ -1071,7 +1071,8 @@ def get_optimizer_state(cls, optimizer):
         state = {'learning_rate': lr}
         for var in optimizer.variables:
             name = var.name
-            var = var.numpy().flatten().mean()  # collapse ndarrays
+            var = var.numpy().flatten()
+            var = np.abs(var).mean()  # collapse ndarrays into mean absolute
             state[name] = float(var)
         return state